@donkeylabs/server 2.0.22 → 2.0.24

package/src/core/workflow-executor.ts

@@ -1,18 +1,13 @@
 #!/usr/bin/env bun
 // Workflow Executor - Subprocess Entry Point
-// Thin shell that creates a WorkflowStateMachine with IPC event bridge.
-// The state machine owns all execution logic and persistence.
+// Bootstraps core services and plugins locally for isolated workflows.
 
 import { connect } from "node:net";
 import type { Socket } from "node:net";
-import { Kysely } from "kysely";
-import { BunSqliteDialect } from "kysely-bun-sqlite";
-import Database from "bun:sqlite";
 import type { WorkflowEvent } from "./workflow-socket";
-import { WorkflowProxyConnection, createPluginsProxy, createCoreServicesProxy } from "./workflow-proxy";
 import type { WorkflowDefinition } from "./workflows";
-import { KyselyWorkflowAdapter } from "./workflow-adapter-kysely";
 import { WorkflowStateMachine, type StateMachineEvents } from "./workflow-state-machine";
+import { bootstrapSubprocess } from "./subprocess-bootstrap";
 
 // ============================================
 // Types
@@ -26,6 +21,10 @@ interface ExecutorConfig {
   tcpPort?: number;
   modulePath: string;
   dbPath: string;
+  pluginNames: string[];
+  pluginModulePaths: Record<string, string>;
+  pluginConfigs: Record<string, any>;
+  coreConfig?: Record<string, any>;
 }
 
 // ============================================
@@ -37,19 +36,23 @@ async function main(): Promise<void> {
   const stdin = await Bun.stdin.text();
   const config: ExecutorConfig = JSON.parse(stdin);
 
-  const { instanceId, workflowName, socketPath, tcpPort, modulePath, dbPath } = config;
+  const {
+    instanceId,
+    workflowName,
+    socketPath,
+    tcpPort,
+    modulePath,
+    dbPath,
+    pluginNames,
+    pluginModulePaths,
+    pluginConfigs,
+    coreConfig,
+  } = config;
 
-  // Connect to IPC socket
   const socket = await connectToSocket(socketPath, tcpPort);
-  const proxyConnection = new WorkflowProxyConnection(socket);
 
-  // Create database connection + adapter (subprocess owns its own persistence)
-  const sqlite = new Database(dbPath);
-  sqlite.run("PRAGMA busy_timeout = 5000");
-  const db = new Kysely<any>({
-    dialect: new BunSqliteDialect({ database: sqlite }),
-  });
-  const adapter = new KyselyWorkflowAdapter(db, { cleanupDays: 0 });
+  let cleanup: (() => Promise<void>) | undefined;
+  let exitCode = 0;
 
   // Start heartbeat
   const heartbeatInterval = setInterval(() => {
@@ -61,39 +64,41 @@ async function main(): Promise<void> {
   }, 5000);
 
   try {
-    // Send started event
-    sendEvent(socket, {
-      type: "started",
-      instanceId,
-      timestamp: Date.now(),
-    });
-
     // Import the workflow module to get the definition
     const module = await import(modulePath);
     const definition = findWorkflowDefinition(module, workflowName, modulePath);
 
-    // Create proxy objects for plugin/core access via IPC
-    const plugins = createPluginsProxy(proxyConnection);
-    const coreServices = createCoreServicesProxy(proxyConnection);
-
-    // Wrap coreServices proxy so that `db` resolves locally instead of via IPC.
-    // Spreading a Proxy with no ownKeys trap loses all proxied properties.
-    const coreWithDb = new Proxy(coreServices, {
-      get(target, prop, receiver) {
-        if (prop === "db") return db;
-        return Reflect.get(target, prop, receiver);
+    const bootstrap = await bootstrapSubprocess({
+      dbPath,
+      coreConfig,
+      pluginMetadata: {
+        names: pluginNames,
+        modulePaths: pluginModulePaths,
+        configs: pluginConfigs,
       },
     });
+    cleanup = bootstrap.cleanup;
+
+    sendEvent(socket, {
+      type: "ready",
+      instanceId,
+      timestamp: Date.now(),
+    });
 
-    // Create state machine with IPC event bridge
     const sm = new WorkflowStateMachine({
-      adapter,
-      core: coreWithDb as any,
-      plugins,
+      adapter: bootstrap.workflowAdapter,
+      core: bootstrap.core as any,
+      plugins: bootstrap.manager.getServices(),
       events: createIpcEventBridge(socket, instanceId),
       pollInterval: 1000,
     });
 
+    sendEvent(socket, {
+      type: "started",
+      instanceId,
+      timestamp: Date.now(),
+    });
+
     // Run the state machine to completion
     const result = await sm.run(instanceId, definition);
 
@@ -112,16 +117,16 @@ async function main(): Promise<void> {
       timestamp: Date.now(),
       error: error instanceof Error ? error.message : String(error),
     });
-    process.exit(1);
+    exitCode = 1;
   } finally {
     clearInterval(heartbeatInterval);
-    proxyConnection.close();
     socket.end();
-    adapter.stop();
-    await db.destroy();
+    if (cleanup) {
+      await cleanup();
+    }
   }
 
-  process.exit(0);
+  process.exit(exitCode);
 }
 
 // ============================================

package/src/core/workflow-socket.ts

@@ -12,6 +12,7 @@ import { createServer as createNetServer } from "node:net";
 // ============================================
 
 export type WorkflowEventType =
+  | "ready"
   | "started"
   | "heartbeat"
   | "step.started"

package/src/core/workflow-state-machine.ts

@@ -3,6 +3,7 @@
 // Communicates through an event callback interface - no knowledge of IPC, SSE, or process management.
 
 import type { CoreServices } from "../core";
+import type { LogLevel } from "./logger";
 import type { Jobs } from "./jobs";
 import type {
   WorkflowAdapter,
@@ -482,6 +483,35 @@ export class WorkflowStateMachine {
     const adapter = this.adapter;
     const instanceId = instance.id;
 
+    const scopedLogger = this.core?.logger?.scoped("workflow", instance.id);
+    const emit = this.core?.events
+      ? async (event: string, data?: Record<string, any>) => {
+          const payload = {
+            instanceId: instance.id,
+            workflowName: instance.workflowName,
+            event,
+            data,
+          };
+
+          await this.core!.events.emit("workflow.event", payload);
+          await this.core!.events.emit(`workflow.${instance.workflowName}.event`, payload);
+          await this.core!.events.emit(`workflow.${instance.id}.event`, payload);
+        }
+      : undefined;
+
+    const log = scopedLogger
+      ? (level: LogLevel, message: string, data?: Record<string, any>) => {
+          scopedLogger[level](message, data);
+        }
+      : undefined;
+
+    const core = this.core
+      ? {
+          ...this.core,
+          logger: scopedLogger ?? this.core.logger,
+        }
+      : this.core;
+
     return {
       input: instance.input,
       steps,
@@ -490,7 +520,10 @@ export class WorkflowStateMachine {
       getStepResult: <T = any>(stepName: string): T | undefined => {
         return steps[stepName] as T | undefined;
       },
-      core: this.core!,
+      core: core!,
+      logger: scopedLogger,
+      emit,
+      log,
       plugins: this.plugins,
       metadata,
       setMetadata: async (key: string, value: any): Promise<void> => {

package/src/core/workflows.ts

@@ -14,6 +14,7 @@ import type { SSE } from "./sse";
 import type { z } from "zod";
 import { sql } from "kysely";
 import type { CoreServices } from "../core";
+import type { Logger, LogLevel } from "./logger";
 import { dirname, join, resolve } from "node:path";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import {
@@ -243,6 +244,12 @@ export interface WorkflowContext {
   getStepResult<T = any>(stepName: string): T | undefined;
   /** Core services (logger, events, cache, etc.) */
   core: CoreServices;
+  /** Scoped logger for this workflow instance (source=workflow, sourceId=instanceId) */
+  logger?: Logger;
+  /** Emit a workflow-scoped custom event */
+  emit?: (event: string, data?: Record<string, any>) => Promise<void>;
+  /** Write a scoped log entry for this workflow instance */
+  log?: (level: LogLevel, message: string, data?: Record<string, any>) => void;
   /** Plugin services - available for business logic in workflow handlers */
   plugins: Record<string, any>;
   /**
@@ -640,6 +647,8 @@ export interface WorkflowsConfig {
   dbPath?: string;
   /** Heartbeat timeout in ms (default: 60000) */
   heartbeatTimeout?: number;
+  /** Timeout waiting for isolated subprocess readiness (ms, default: 10000) */
+  readyTimeout?: number;
 }
 
 /** Options for registering a workflow */
@@ -689,6 +698,16 @@ export interface Workflows {
   setPlugins(plugins: Record<string, any>): void;
   /** Update metadata for a workflow instance (used by isolated workflows) */
   updateMetadata(instanceId: string, key: string, value: any): Promise<void>;
+  /** Set plugin metadata for local instantiation in isolated workflows */
+  setPluginMetadata(metadata: PluginMetadata): void;
+}
+
+export interface PluginMetadata {
+  names: string[];
+  modulePaths: Record<string, string>;
+  configs: Record<string, any>;
+  dependencies: Record<string, string[]>;
+  customErrors: Record<string, Record<string, any>>;
 }
 
 // ============================================
@@ -719,8 +738,25 @@ class WorkflowsImpl implements Workflows {
   private tcpPortRange: [number, number];
   private dbPath?: string;
   private heartbeatTimeoutMs: number;
+  private readyTimeoutMs: number;
   private workflowModulePaths = new Map<string, string>();
   private isolatedProcesses = new Map<string, IsolatedProcessInfo>();
+  private readyWaiters = new Map<
+    string,
+    {
+      promise: Promise<void>;
+      resolve: () => void;
+      reject: (error: Error) => void;
+      timeout: ReturnType<typeof setTimeout>;
+    }
+  >();
+
+  // Plugin metadata for local instantiation in isolated workflows
+  private pluginNames: string[] = [];
+  private pluginModulePaths: Record<string, string> = {};
+  private pluginConfigs: Record<string, any> = {};
+  private pluginDependencies: Record<string, string[]> = {};
+  private pluginCustomErrors: Record<string, Record<string, any>> = {};
 
   constructor(config: WorkflowsConfig = {}) {
     this.adapter = config.adapter ?? new MemoryWorkflowAdapter();
@@ -735,6 +771,7 @@ class WorkflowsImpl implements Workflows {
     this.tcpPortRange = config.tcpPortRange ?? [49152, 65535];
     this.dbPath = config.dbPath;
     this.heartbeatTimeoutMs = config.heartbeatTimeout ?? 60000;
+    this.readyTimeoutMs = config.readyTimeout ?? 10000;
   }
 
   private getSocketServer(): WorkflowSocketServer {
@@ -786,6 +823,14 @@ class WorkflowsImpl implements Workflows {
     this.plugins = plugins;
   }
 
+  setPluginMetadata(metadata: PluginMetadata): void {
+    this.pluginNames = metadata.names;
+    this.pluginModulePaths = metadata.modulePaths;
+    this.pluginConfigs = metadata.configs;
+    this.pluginDependencies = metadata.dependencies;
+    this.pluginCustomErrors = metadata.customErrors;
+  }
+
   async updateMetadata(instanceId: string, key: string, value: any): Promise<void> {
     const instance = await this.adapter.getInstance(instanceId);
     if (!instance) return;
@@ -855,7 +900,7 @@ class WorkflowsImpl implements Workflows {
 
     if (isIsolated && modulePath && this.dbPath) {
       // Execute in isolated subprocess
-      this.executeIsolatedWorkflow(instance.id, definition, input, modulePath);
+      await this.executeIsolatedWorkflow(instance.id, definition, input, modulePath);
     } else {
       // Execute inline using state machine
       if (isIsolated && !modulePath) {
@@ -952,7 +997,14 @@ class WorkflowsImpl implements Workflows {
       const modulePath = this.workflowModulePaths.get(instance.workflowName);
 
       if (isIsolated && modulePath && this.dbPath) {
-        this.executeIsolatedWorkflow(instance.id, definition, instance.input, modulePath);
+        try {
+          await this.executeIsolatedWorkflow(instance.id, definition, instance.input, modulePath);
+        } catch (error) {
+          console.error(
+            `[Workflows] Failed to resume isolated workflow ${instance.id}:`,
+            error instanceof Error ? error.message : String(error)
+          );
+        }
       } else {
         this.startInlineWorkflow(instance.id, definition);
       }
@@ -989,6 +1041,12 @@ class WorkflowsImpl implements Workflows {
     }
     this.running.clear();
 
+    for (const [instanceId, waiter] of this.readyWaiters) {
+      clearTimeout(waiter.timeout);
+      waiter.reject(new Error(`Workflows stopped before ready: ${instanceId}`));
+    }
+    this.readyWaiters.clear();
+
     // Stop adapter (cleanup timers and prevent further DB access)
     if (this.adapter && typeof (this.adapter as any).stop === "function") {
       (this.adapter as any).stop();
@@ -1182,20 +1240,36 @@ class WorkflowsImpl implements Workflows {
   ): Promise<void> {
     const socketServer = this.getSocketServer();
 
+    const pluginNames = this.pluginNames.length > 0
+      ? this.pluginNames
+      : Object.keys(this.pluginModulePaths);
+
+    if (pluginNames.length === 0 && Object.keys(this.plugins).length > 0) {
+      throw new Error(
+        "[Workflows] Plugin metadata is required for isolated workflows. " +
+          "Call workflows.setPluginMetadata() during server initialization."
+      );
+    }
+
+    const missingModulePaths = pluginNames.filter((name) => !this.pluginModulePaths[name]);
+    if (missingModulePaths.length > 0) {
+      throw new Error(
+        `[Workflows] Missing module paths for plugins: ${missingModulePaths.join(", ")}. ` +
+          `Ensure plugins are created with createPlugin.define() and registered before workflows start.`
+      );
+    }
+
+    const pluginConfigs = serializePluginConfigsOrThrow(this.pluginConfigs, pluginNames);
+    const coreConfig = serializeCoreConfigOrThrow(this.core?.config);
+
     // Create socket for this workflow instance
     const { socketPath, tcpPort } = await socketServer.createSocket(instanceId);
 
-    // Mark workflow as running
-    await this.adapter.updateInstance(instanceId, {
-      status: "running",
-      startedAt: new Date(),
-    });
-
     // Get the executor path
     const currentDir = dirname(fileURLToPath(import.meta.url));
     const executorPath = join(currentDir, "workflow-executor.ts");
 
-    // Prepare config for the executor
+    // Prepare config for the executor, including plugin metadata for local instantiation
     const config = {
       instanceId,
       workflowName: definition.name,
@@ -1204,6 +1278,10 @@ class WorkflowsImpl implements Workflows {
       tcpPort,
       modulePath,
       dbPath: this.dbPath,
+      pluginNames,
+      pluginModulePaths: this.pluginModulePaths,
+      pluginConfigs,
+      coreConfig,
     };
 
     // Spawn the subprocess
@@ -1240,6 +1318,19 @@ class WorkflowsImpl implements Workflows {
     // Set up heartbeat timeout
     this.resetHeartbeatTimeout(instanceId, proc.pid);
 
+    const exitBeforeReady = proc.exited.then((exitCode) => {
+      throw new Error(`Subprocess exited before ready (code ${exitCode})`);
+    });
+
+    try {
+      await Promise.race([this.waitForIsolatedReady(instanceId), exitBeforeReady]);
+    } catch (error) {
+      await this.handleIsolatedStartFailure(instanceId, proc.pid, error);
+      exitBeforeReady.catch(() => undefined);
+      throw error;
+    }
+    exitBeforeReady.catch(() => undefined);
+
     // Handle process exit
     proc.exited.then(async (exitCode) => {
       const info = this.isolatedProcesses.get(instanceId);
@@ -1250,9 +1341,9 @@ class WorkflowsImpl implements Workflows {
       }
       await socketServer.closeSocket(instanceId);
 
-      // Check if workflow is still running (crashed before completion)
+      // Check if workflow is still pending/running (crashed before completion)
       const instance = await this.adapter.getInstance(instanceId);
-      if (instance && instance.status === "running") {
+      if (instance && (instance.status === "running" || instance.status === "pending")) {
         console.error(`[Workflows] Isolated workflow ${instanceId} crashed with exit code ${exitCode}`);
         await this.adapter.updateInstance(instanceId, {
           status: "failed",
@@ -1293,6 +1384,11 @@ class WorkflowsImpl implements Workflows {
     }
 
     switch (type) {
+      case "ready": {
+        this.resolveIsolatedReady(instanceId);
+        break;
+      }
+
       case "started":
       case "heartbeat":
         // No-op: heartbeat handled above, started is handled by executeIsolatedWorkflow
@@ -1382,6 +1478,7 @@ class WorkflowsImpl implements Workflows {
       case "completed": {
         // Clean up isolated process tracking
         this.cleanupIsolatedProcess(instanceId);
+        this.resolveIsolatedReady(instanceId);
 
         // Subprocess already persisted state - just emit events
         await this.emitEvent("workflow.completed", {
@@ -1398,6 +1495,7 @@ class WorkflowsImpl implements Workflows {
       case "failed": {
         // Clean up isolated process tracking
         this.cleanupIsolatedProcess(instanceId);
+        this.rejectIsolatedReady(instanceId, new Error(event.error ?? "Isolated workflow failed"));
 
         // Subprocess already persisted state - just emit events
         await this.emitEvent("workflow.failed", {
@@ -1416,6 +1514,91 @@ class WorkflowsImpl implements Workflows {
     }
   }
 
+  private waitForIsolatedReady(instanceId: string): Promise<void> {
+    const existing = this.readyWaiters.get(instanceId);
+    if (existing) {
+      return existing.promise;
+    }
+
+    let resolveFn!: () => void;
+    let rejectFn!: (error: Error) => void;
+    const promise = new Promise<void>((resolve, reject) => {
+      resolveFn = resolve;
+      rejectFn = reject;
+    });
+
+    const timeout = setTimeout(() => {
+      this.readyWaiters.delete(instanceId);
+      rejectFn(new Error(`Timed out waiting for isolated workflow ${instanceId} readiness`));
+    }, this.readyTimeoutMs);
+
+    this.readyWaiters.set(instanceId, {
+      promise,
+      resolve: () => resolveFn(),
+      reject: (error) => rejectFn(error),
+      timeout,
+    });
+
+    return promise;
+  }
+
+  private resolveIsolatedReady(instanceId: string): void {
+    const waiter = this.readyWaiters.get(instanceId);
+    if (!waiter) return;
+    clearTimeout(waiter.timeout);
+    this.readyWaiters.delete(instanceId);
+    waiter.resolve();
+  }
+
+  private rejectIsolatedReady(instanceId: string, error: Error): void {
+    const waiter = this.readyWaiters.get(instanceId);
+    if (!waiter) return;
+    clearTimeout(waiter.timeout);
+    this.readyWaiters.delete(instanceId);
+    waiter.reject(error);
+  }
+
+  private async handleIsolatedStartFailure(
+    instanceId: string,
+    pid: number,
+    error: unknown
+  ): Promise<void> {
+    const errorMessage = error instanceof Error ? error.message : String(error);
+
+    try {
+      process.kill(pid, "SIGTERM");
+    } catch {
+      // Process might already be dead
+    }
+
+    this.cleanupIsolatedProcess(instanceId);
+    await this.getSocketServer().closeSocket(instanceId);
+
+    const instance = await this.adapter.getInstance(instanceId);
+    if (instance && (instance.status === "pending" || instance.status === "running")) {
+      await this.adapter.updateInstance(instanceId, {
+        status: "failed",
+        error: errorMessage,
+        completedAt: new Date(),
+      });
+
+      await this.emitEvent("workflow.failed", {
+        instanceId,
+        workflowName: instance.workflowName,
+        error: errorMessage,
+      });
+
+      if (this.sse) {
+        this.sse.broadcast(`workflow:${instanceId}`, "failed", { error: errorMessage });
+        this.sse.broadcast("workflows:all", "workflow.failed", {
+          instanceId,
+          workflowName: instance.workflowName,
+          error: errorMessage,
+        });
+      }
+    }
+  }
+
   /**
    * Handle proxy calls from isolated subprocess
    */
@@ -1460,6 +1643,7 @@ class WorkflowsImpl implements Workflows {
       if (info.heartbeatTimeout) clearTimeout(info.heartbeatTimeout);
       this.isolatedProcesses.delete(instanceId);
     }
+    this.rejectIsolatedReady(instanceId, new Error("Isolated workflow cleaned up"));
   }
 
   /**
@@ -1532,6 +1716,102 @@ class WorkflowsImpl implements Workflows {
   }
 }
 
+// ============================================
+// Helpers
+// ============================================
+
+function serializePluginConfigsOrThrow(
+  configs: Record<string, any>,
+  pluginNames: string[]
+): Record<string, any> {
+  const result: Record<string, any> = {};
+  const failures: string[] = [];
+
+  for (const name of pluginNames) {
+    if (!(name in configs)) continue;
+    try {
+      assertJsonSerializable(configs[name], `pluginConfigs.${name}`);
+      const serialized = JSON.stringify(configs[name]);
+      result[name] = JSON.parse(serialized);
+    } catch {
+      failures.push(name);
+    }
+  }
+
+  if (failures.length > 0) {
+    throw new Error(
+      `[Workflows] Non-serializable plugin config(s): ${failures.join(", ")}. ` +
+        `Provide JSON-serializable configs for isolated workflows.`
+    );
+  }
+
+  return result;
+}
+
+function serializeCoreConfigOrThrow(config?: Record<string, any>): Record<string, any> | undefined {
+  if (!config) return undefined;
+  try {
+    assertJsonSerializable(config, "coreConfig");
+    const serialized = JSON.stringify(config);
+    return JSON.parse(serialized);
+  } catch {
+    throw new Error(
+      "[Workflows] Core config is not JSON-serializable. Provide JSON-serializable values for isolated workflows."
+    );
+  }
+}
+
+function assertJsonSerializable(value: any, path: string, seen = new WeakSet<object>()): void {
+  if (
+    value === null ||
+    typeof value === "string" ||
+    typeof value === "number" ||
+    typeof value === "boolean"
+  ) {
+    return;
+  }
+
+  if (typeof value === "undefined" || typeof value === "function" || typeof value === "symbol") {
+    throw new Error(`[Workflows] Non-serializable value at ${path}`);
+  }
+
+  if (typeof value === "bigint") {
+    throw new Error(`[Workflows] Non-serializable bigint at ${path}`);
+  }
+
+  if (value instanceof Date) {
+    return;
+  }
+
+  if (Array.isArray(value)) {
+    for (let i = 0; i < value.length; i++) {
+      assertJsonSerializable(value[i], `${path}[${i}]`, seen);
+    }
+    return;
+  }
+
+  if (typeof value === "object") {
+    if (seen.has(value)) {
+      throw new Error(`[Workflows] Circular reference at ${path}`);
+    }
+    seen.add(value);
+
+    if (!isPlainObject(value)) {
+      throw new Error(`[Workflows] Non-serializable object at ${path}`);
+    }
+
+    for (const [key, nested] of Object.entries(value)) {
+      assertJsonSerializable(nested, `${path}.${key}`, seen);
+    }
+    return;
+  }
+}
+
+function isPlainObject(value: Record<string, any>): boolean {
+  const proto = Object.getPrototypeOf(value);
+  return proto === Object.prototype || proto === null;
+}
+
 // ============================================
 // Factory Function
 // ============================================