deepline 0.1.199 → 0.1.200

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts

@@ -0,0 +1,573 @@
+import { randomUUID } from 'node:crypto';
+import { spawn } from 'node:child_process';
+import { cp, mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises';
+import { dirname, join } from 'node:path';
+import type { SpawnOptionsWithoutStdio } from 'node:child_process';
+import {
+  setSpanAttributes,
+  withActiveSpan,
+} from '@shared_libs/observability/tracing';
+import type { PlayRunnerBackend } from '../types';
+import { buildPlayRunnerBundle } from '../bundle';
+import {
+  findPlayRunnerResult,
+  parsePlayRunnerEventLine,
+  parsePlayRunnerEvents,
+} from '../runner-events';
+import type { PlayRunnerExecutionConfig } from '@shared_libs/play-runtime/protocol';
+import { resolveDaytonaSandboxComputeItem } from '@shared_libs/play-runtime/worker-api-types';
+import {
+  recordComputeBillingItemViaAppRuntime,
+  type WorkerRuntimeApiContext,
+} from '@shared_libs/play-runtime/app-runtime-api';
+import {
+  STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
+  STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
+} from '@shared_libs/play-runtime/runtime-constants';
+
+interface CommandExecutionResult {
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+}
+
+type StreamedCommandCallbacks = {
+  onStdoutLine?: (line: string) => void;
+  /** Called once when the result event is found — avoids storing full stdout. */
+  onResultLine?: (line: string) => void;
+  cancellationSignal?: AbortSignal;
+};
+
+const LOCAL_PROCESS_COMPUTE_SOURCE = 'local_process';
+const LOCAL_PROCESS_COMPUTE_CPU = 1;
+const LOCAL_PROCESS_COMPUTE_MEMORY_GIB = 1;
+const LOCAL_PROCESS_COMPUTE_DISK_GIB = 3;
+export const LOCAL_PROCESS_CANCEL_GRACE_MS = 2_000;
+const LOCAL_PROCESS_CANCELLED_ERROR = 'Local play runner cancelled';
+const STANDARD_PLAY_RUNTIME_LIMIT_ERROR = `Based on this plan, max runtime is ${STANDARD_PLAY_RUNTIME_LIMIT_LABEL}. Use smaller batches; ask for runtime.`;
+
+export const LOCAL_PROCESS_COMPUTE_PROFILE = {
+  source: LOCAL_PROCESS_COMPUTE_SOURCE,
+  cpu: LOCAL_PROCESS_COMPUTE_CPU,
+  memoryGiB: LOCAL_PROCESS_COMPUTE_MEMORY_GIB,
+  diskGiB: LOCAL_PROCESS_COMPUTE_DISK_GIB,
+} as const;
+
+function logLocalRunnerPerf(
+  workflowId: string | null | undefined,
+  phase: string,
+  startedAt: number,
+  extra: Record<string, unknown> = {},
+) {
+  console.info('[perf][play-runner.local_process]', {
+    workflowId,
+    phase,
+    ms: Date.now() - startedAt,
+    ...extra,
+  });
+}
+
+async function runCommand(
+  command: string,
+  args: string[],
+  options?: SpawnOptionsWithoutStdio,
+  callbacks?: StreamedCommandCallbacks,
+): Promise<CommandExecutionResult> {
+  return await new Promise((resolve, reject) => {
+    if (callbacks?.cancellationSignal?.aborted) {
+      reject(new Error(LOCAL_PROCESS_CANCELLED_ERROR));
+      return;
+    }
+
+    const child = spawn(command, args, {
+      stdio: ['ignore', 'pipe', 'pipe'],
+      ...options,
+    });
+
+    let stderr = '';
+    let stdoutBuffer = '';
+    let stdoutLineCount = 0;
+    // Only keep the last line that looks like a result event (avoids storing full stdout).
+    let resultLine = '';
+    let settled = false;
+    let cancellationRequested = false;
+    let cancellationErrorMessage = LOCAL_PROCESS_CANCELLED_ERROR;
+    let cancelKillTimer: NodeJS.Timeout | null = null;
+
+    const cleanupCancellation = () => {
+      callbacks?.cancellationSignal?.removeEventListener('abort', onCancel);
+      if (cancelKillTimer) {
+        clearTimeout(cancelKillTimer);
+        cancelKillTimer = null;
+      }
+    };
+
+    const settleRejected = (error: unknown) => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+      cleanupCancellation();
+      reject(error);
+    };
+
+    const settleResolved = (result: CommandExecutionResult) => {
+      if (settled) {
+        return;
+      }
+      settled = true;
+      cleanupCancellation();
+      resolve(result);
+    };
+
+    function onCancel() {
+      if (settled) {
+        return;
+      }
+      cancellationRequested = true;
+      const reason = callbacks?.cancellationSignal?.reason;
+      cancellationErrorMessage =
+        typeof reason === 'string' && reason.trim()
+          ? reason.trim()
+          : LOCAL_PROCESS_CANCELLED_ERROR;
+      child.kill('SIGTERM');
+      cancelKillTimer = setTimeout(() => {
+        if (child.exitCode === null && child.signalCode === null) {
+          child.kill('SIGKILL');
+        }
+      }, LOCAL_PROCESS_CANCEL_GRACE_MS);
+      cancelKillTimer.unref?.();
+    }
+
+    callbacks?.cancellationSignal?.addEventListener('abort', onCancel, {
+      once: true,
+    });
+
+    child.stdout?.on('data', (chunk) => {
+      const text = String(chunk);
+      stdoutBuffer += text;
+      const lines = stdoutBuffer.split(/\r?\n/);
+      stdoutBuffer = lines.pop() ?? '';
+      for (const line of lines) {
+        const trimmed = line.trim();
+        if (!trimmed) {
+          continue;
+        }
+        stdoutLineCount++;
+        // Capture result lines so we can extract the result without storing all stdout.
+        if (trimmed.includes('"type":"result"')) {
+          resultLine = trimmed;
+        }
+        callbacks?.onStdoutLine?.(trimmed);
+      }
+    });
+    child.stdout?.on('error', () => {
+      // Absorb pipe errors — we still get partial stdout
+    });
+    child.stderr?.on('data', (chunk) => {
+      stderr += String(chunk);
+    });
+    child.stderr?.on('error', () => {});
+
+    child.on('error', settleRejected);
+    child.on('close', (code) => {
+      const trailing = stdoutBuffer.trim();
+      if (trailing) {
+        stdoutLineCount++;
+        if (trailing.includes('"type":"result"')) {
+          resultLine = trailing;
+        }
+        callbacks?.onStdoutLine?.(trailing);
+      }
+      if (cancellationRequested) {
+        settleRejected(new Error(cancellationErrorMessage));
+        return;
+      }
+      settleResolved({
+        exitCode: code ?? 0,
+        stdout: resultLine,
+        stderr,
+      });
+    });
+  });
+}
+
+async function runCommandWithStandardRuntimeLimit(
+  command: string,
+  args: string[],
+  options?: SpawnOptionsWithoutStdio,
+  callbacks?: StreamedCommandCallbacks,
+): Promise<CommandExecutionResult> {
+  const runtimeLimitController = new AbortController();
+  const runtimeDeadlineTimer = setTimeout(() => {
+    runtimeLimitController.abort(STANDARD_PLAY_RUNTIME_LIMIT_ERROR);
+  }, STANDARD_PLAY_RUNTIME_LIMIT_SECONDS * 1000);
+  runtimeDeadlineTimer.unref?.();
+
+  const onExternalCancel = () => {
+    if (!runtimeLimitController.signal.aborted) {
+      runtimeLimitController.abort(
+        callbacks?.cancellationSignal?.reason ?? LOCAL_PROCESS_CANCELLED_ERROR,
+      );
+    }
+  };
+
+  if (callbacks?.cancellationSignal?.aborted) {
+    onExternalCancel();
+  } else {
+    callbacks?.cancellationSignal?.addEventListener('abort', onExternalCancel, {
+      once: true,
+    });
+  }
+
+  try {
+    return await runCommand(command, args, options, {
+      ...callbacks,
+      cancellationSignal: runtimeLimitController.signal,
+    });
+  } finally {
+    clearTimeout(runtimeDeadlineTimer);
+    callbacks?.cancellationSignal?.removeEventListener(
+      'abort',
+      onExternalCancel,
+    );
+  }
+}
+
+function createFailedResult(config: PlayRunnerExecutionConfig, error: string) {
+  return {
+    status: 'failed' as const,
+    error,
+    logs: [],
+    stats: {},
+    steps: [],
+    checkpoint: config.checkpoint ?? null,
+    tableNamespace: null,
+  };
+}
+
+function runtimeApiContextForConfig(
+  config: PlayRunnerExecutionConfig,
+): WorkerRuntimeApiContext {
+  const baseUrl = config.context.baseUrl?.trim();
+  const executorToken = config.context.executorToken?.trim();
+  if (!baseUrl || !executorToken) {
+    throw new Error(
+      'Missing runtime API context for local-process compute billing.',
+    );
+  }
+  return { baseUrl, executorToken };
+}
+
+async function prepareLocalProcessWorkspace(
+  config: PlayRunnerExecutionConfig,
+): Promise<{
+  workspaceRoot: string;
+  runnerPath: string;
+  configPath: string;
+  localConfig: PlayRunnerExecutionConfig;
+}> {
+  const workspaceRoot = await mkdtemp(
+    join(config.workspaceRoot, '.deepline-local-play-runner-'),
+  );
+  const filesRoot = join(workspaceRoot, 'files');
+  await mkdir(filesRoot, { recursive: true });
+
+  const localMaterializedFiles: Record<string, string> = {};
+  const copiedFiles = new Map<string, string>();
+
+  for (const [logicalPath, localPath] of Object.entries(
+    config.materializedFiles,
+  )) {
+    if (!copiedFiles.has(localPath)) {
+      const targetPath = join(filesRoot, randomUUID());
+      await mkdir(dirname(targetPath), { recursive: true });
+      await cp(localPath, targetPath, { force: true });
+      copiedFiles.set(localPath, targetPath);
+    }
+    localMaterializedFiles[logicalPath] = copiedFiles.get(localPath)!;
+  }
+
+  let localCsvSourcePath: string | null = null;
+  if (config.csvSourcePath) {
+    if (!copiedFiles.has(config.csvSourcePath)) {
+      const targetPath = join(filesRoot, randomUUID());
+      await mkdir(dirname(targetPath), { recursive: true });
+      await cp(config.csvSourcePath, targetPath, { force: true });
+      copiedFiles.set(config.csvSourcePath, targetPath);
+    }
+    localCsvSourcePath = copiedFiles.get(config.csvSourcePath)!;
+  }
+
+  const runnerPath = join(workspaceRoot, 'play-runner.cjs');
+  const configPath = join(workspaceRoot, 'runner-config.json');
+  const localConfig: PlayRunnerExecutionConfig = {
+    ...config,
+    workspaceRoot,
+    csvSourcePath: localCsvSourcePath,
+    materializedFiles: localMaterializedFiles,
+  };
+
+  return {
+    workspaceRoot,
+    runnerPath,
+    configPath,
+    localConfig,
+  };
+}
+
+export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
+  async execute(config, callbacks) {
+    return await withActiveSpan(
+      'plays.runner.local_process',
+      {
+        tracer: 'deepline.plays',
+        attributes: {
+          'plays.workflow_id': config.context.workflowId,
+          'plays.play_name': config.context.playName,
+          'plays.has_csv_source': Boolean(config.csvSourcePath),
+          'plays.materialized_file_count': Object.keys(config.materializedFiles)
+            .length,
+        },
+      },
+      async (backendSpan) => {
+        const localComputeStartedAt = Date.now();
+        let phaseStartedAt = Date.now();
+        const bundle = await withActiveSpan(
+          'plays.runner.bundle_ready',
+          {
+            tracer: 'deepline.plays',
+            attributes: {
+              'plays.workflow_id': config.context.workflowId,
+              'plays.play_name': config.context.playName,
+            },
+          },
+          async () => await buildPlayRunnerBundle(),
+        );
+        logLocalRunnerPerf(
+          config.context.workflowId,
+          'bundle_ready',
+          localComputeStartedAt,
+          { bundleBytes: bundle.length },
+        );
+        phaseStartedAt = Date.now();
+        const { workspaceRoot, runnerPath, configPath, localConfig } =
+          await withActiveSpan(
+            'plays.runner.prepare_workspace',
+            {
+              tracer: 'deepline.plays',
+              attributes: {
+                'plays.workflow_id': config.context.workflowId,
+                'plays.play_name': config.context.playName,
+              },
+            },
+            async (span) => {
+              const prepared = await prepareLocalProcessWorkspace(config);
+              setSpanAttributes(span, {
+                'plays.workspace_materialized_file_count': Object.keys(
+                  prepared.localConfig.materializedFiles,
+                ).length,
+                'plays.workspace_has_csv_source': Boolean(
+                  prepared.localConfig.csvSourcePath,
+                ),
+              });
+              return prepared;
+            },
+          );
+        logLocalRunnerPerf(
+          config.context.workflowId,
+          'prepare_workspace',
+          phaseStartedAt,
+        );
+
+        try {
+          phaseStartedAt = Date.now();
+          await withActiveSpan(
+            'plays.runner.write_files',
+            {
+              tracer: 'deepline.plays',
+              attributes: {
+                'plays.workflow_id': config.context.workflowId,
+                'plays.play_name': config.context.playName,
+              },
+            },
+            async () => {
+              await writeFile(runnerPath, bundle, 'utf-8');
+              await writeFile(configPath, JSON.stringify(localConfig), 'utf-8');
+            },
+          );
+          logLocalRunnerPerf(
+            config.context.workflowId,
+            'write_files',
+            phaseStartedAt,
+          );
+
+          phaseStartedAt = Date.now();
+          const execution = await withActiveSpan(
+            'plays.runner.child_process',
+            {
+              tracer: 'deepline.plays',
+              attributes: {
+                'plays.workflow_id': config.context.workflowId,
+                'plays.play_name': config.context.playName,
+              },
+            },
+            async (span) => {
+              const result = await runCommandWithStandardRuntimeLimit(
+                'node',
+                [runnerPath, configPath],
+                {
+                  cwd: workspaceRoot,
+                  env: {
+                    ...process.env,
+                    PWD: workspaceRoot,
+                    TMPDIR: workspaceRoot,
+                    TEMP: workspaceRoot,
+                    TMP: workspaceRoot,
+                  },
+                },
+                {
+                  cancellationSignal: callbacks?.cancellationSignal,
+                  onStdoutLine: (line) => {
+                    const event = parsePlayRunnerEventLine(line);
+                    if (!event) {
+                      return;
+                    }
+                    if (event.type === 'log') {
+                      callbacks?.onLog?.(event);
+                      return;
+                    }
+                    if (event.type === 'checkpoint') {
+                      callbacks?.onCheckpoint?.(event.checkpoint);
+                      return;
+                    }
+                    if (event.type === 'row_update') {
+                      callbacks?.onRowUpdate?.(event.update);
+                      return;
+                    }
+                    if (event.type === 'execution_event') {
+                      callbacks?.onExecutionEvent?.(event.event);
+                    }
+                  },
+                },
+              );
+              setSpanAttributes(span, {
+                'plays.child_exit_code': result.exitCode,
+                'plays.stdout_bytes': result.stdout.length,
+                'plays.stderr_bytes': result.stderr.length,
+              });
+              return result;
+            },
+          );
+          logLocalRunnerPerf(
+            config.context.workflowId,
+            'child_process',
+            phaseStartedAt,
+            {
+              exitCode: execution.exitCode,
+              stdoutBytes: execution.stdout.length,
+              stderrBytes: execution.stderr.length,
+            },
+          );
+
+          phaseStartedAt = Date.now();
+          const events = await withActiveSpan(
+            'plays.runner.parse_events',
+            {
+              tracer: 'deepline.plays',
+              attributes: {
+                'plays.workflow_id': config.context.workflowId,
+                'plays.play_name': config.context.playName,
+              },
+            },
+            async (span) => {
+              const parsed = parsePlayRunnerEvents(execution.stdout);
+              setSpanAttributes(span, {
+                'plays.event_count': parsed.length,
+              });
+              return parsed;
+            },
+          );
+          logLocalRunnerPerf(
+            config.context.workflowId,
+            'parse_events',
+            phaseStartedAt,
+            { eventCount: events.length },
+          );
+          const result = findPlayRunnerResult(events);
+          if (result) {
+            setSpanAttributes(backendSpan, {
+              'plays.runner_status': result.status,
+            });
+            return result;
+          }
+
+          return createFailedResult(
+            config,
+            execution.exitCode !== 0
+              ? `Local play runner exited with code ${execution.exitCode}.${execution.stderr ? ` ${execution.stderr.trim()}` : ''}`.trim()
+              : 'Local play runner produced no result.',
+          );
+        } catch (error) {
+          return createFailedResult(
+            config,
+            error instanceof Error ? error.message : String(error),
+          );
+        } finally {
+          const localComputeEndedAt = Date.now();
+          try {
+            await recordComputeBillingItemViaAppRuntime(
+              runtimeApiContextForConfig(config),
+              {
+                sessionId:
+                  config.context.workflowId ??
+                  config.context.runId ??
+                  workspaceRoot,
+                orgId: config.context.orgId ?? 'unknown',
+                userId: null,
+                operation: 'workflow_run',
+                item: resolveDaytonaSandboxComputeItem({
+                  itemId: `${LOCAL_PROCESS_COMPUTE_SOURCE}:${config.context.workflowId ?? workspaceRoot}`,
+                  source: LOCAL_PROCESS_COMPUTE_SOURCE,
+                  sandboxId: workspaceRoot,
+                  wallTimeSeconds:
+                    (localComputeEndedAt - localComputeStartedAt) / 1000,
+                  cpu: LOCAL_PROCESS_COMPUTE_PROFILE.cpu,
+                  memoryGiB: LOCAL_PROCESS_COMPUTE_PROFILE.memoryGiB,
+                  diskGiB: LOCAL_PROCESS_COMPUTE_PROFILE.diskGiB,
+                  startedAt: localComputeStartedAt,
+                  endedAt: localComputeEndedAt,
+                }),
+              },
+            );
+          } catch (billingError) {
+            console.error(
+              '[play-runner.local_process.compute_billing_failed]',
+              {
+                workflowId: config.context.workflowId,
+                workspaceRoot,
+                error:
+                  billingError instanceof Error
+                    ? billingError.message
+                    : String(billingError),
+              },
+            );
+          }
+          await withActiveSpan(
+            'plays.runner.cleanup',
+            {
+              tracer: 'deepline.plays',
+              attributes: {
+                'plays.workflow_id': config.context.workflowId,
+                'plays.play_name': config.context.playName,
+              },
+            },
+            async () => {
+              await rm(workspaceRoot, { recursive: true, force: true });
+            },
+          );
+        }
+      },
+    );
+  },
+};