@donkeylabs/server 2.0.19 → 2.0.21

package/src/core/workflow-adapter-kysely.ts

@@ -31,6 +31,7 @@ interface WorkflowInstancesTable {
   completed_at: string | null;
   parent_id: string | null;
   branch_name: string | null;
+  metadata: string | null;
 }
 
 interface Database {
@@ -41,6 +42,7 @@ export class KyselyWorkflowAdapter implements WorkflowAdapter {
   private db: Kysely<Database>;
   private cleanupTimer?: ReturnType<typeof setInterval>;
   private cleanupDays: number;
+  private stopped = false;
 
   constructor(db: Kysely<any>, config: KyselyWorkflowAdapterConfig = {}) {
     this.db = db as Kysely<Database>;
@@ -53,7 +55,16 @@ export class KyselyWorkflowAdapter implements WorkflowAdapter {
     }
   }
 
+  /** Check if adapter is stopped (for safe database access) */
+  private checkStopped(): boolean {
+    return this.stopped;
+  }
+
   async createInstance(instance: Omit<WorkflowInstance, "id">): Promise<WorkflowInstance> {
+    if (this.checkStopped()) {
+      throw new Error("WorkflowAdapter has been stopped");
+    }
+
     const id = `wf_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`;
 
     await this.db
@@ -75,6 +86,7 @@ export class KyselyWorkflowAdapter implements WorkflowAdapter {
         completed_at: instance.completedAt?.toISOString() ?? null,
         parent_id: instance.parentId ?? null,
         branch_name: instance.branchName ?? null,
+        metadata: instance.metadata ? JSON.stringify(instance.metadata) : null,
       })
       .execute();
 
@@ -82,17 +94,27 @@ export class KyselyWorkflowAdapter implements WorkflowAdapter {
   }
 
   async getInstance(instanceId: string): Promise<WorkflowInstance | null> {
-    const row = await this.db
-      .selectFrom("__donkeylabs_workflow_instances__")
-      .selectAll()
-      .where("id", "=", instanceId)
-      .executeTakeFirst();
+    if (this.checkStopped()) return null;
 
-    if (!row) return null;
-    return this.rowToInstance(row);
+    try {
+      const row = await this.db
+        .selectFrom("__donkeylabs_workflow_instances__")
+        .selectAll()
+        .where("id", "=", instanceId)
+        .executeTakeFirst();
+
+      if (!row) return null;
+      return this.rowToInstance(row);
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return null;
+      throw err;
+    }
   }
 
   async updateInstance(instanceId: string, updates: Partial<WorkflowInstance>): Promise<void> {
+    if (this.checkStopped()) return;
+
     const updateData: Partial<WorkflowInstancesTable> = {};
 
     if (updates.status !== undefined) {
@@ -121,17 +143,28 @@ export class KyselyWorkflowAdapter implements WorkflowAdapter {
     if (updates.completedAt !== undefined) {
       updateData.completed_at = updates.completedAt?.toISOString() ?? null;
     }
+    if (updates.metadata !== undefined) {
+      updateData.metadata = updates.metadata ? JSON.stringify(updates.metadata) : null;
+    }
 
     if (Object.keys(updateData).length === 0) return;
 
-    await this.db
-      .updateTable("__donkeylabs_workflow_instances__")
-      .set(updateData)
-      .where("id", "=", instanceId)
-      .execute();
+    try {
+      await this.db
+        .updateTable("__donkeylabs_workflow_instances__")
+        .set(updateData)
+        .where("id", "=", instanceId)
+        .execute();
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return;
+      throw err;
+    }
   }
 
   async deleteInstance(instanceId: string): Promise<boolean> {
+    if (this.checkStopped()) return false;
+
     // Check if exists first since BunSqliteDialect doesn't report numDeletedRows properly
     const exists = await this.db
       .selectFrom("__donkeylabs_workflow_instances__")
@@ -153,6 +186,8 @@ export class KyselyWorkflowAdapter implements WorkflowAdapter {
     workflowName: string,
     status?: WorkflowStatus
   ): Promise<WorkflowInstance[]> {
+    if (this.checkStopped()) return [];
+
     let query = this.db
       .selectFrom("__donkeylabs_workflow_instances__")
       .selectAll()
@@ -167,16 +202,26 @@ export class KyselyWorkflowAdapter implements WorkflowAdapter {
   }
 
   async getRunningInstances(): Promise<WorkflowInstance[]> {
-    const rows = await this.db
-      .selectFrom("__donkeylabs_workflow_instances__")
-      .selectAll()
-      .where("status", "=", "running")
-      .execute();
+    if (this.checkStopped()) return [];
 
-    return rows.map((r) => this.rowToInstance(r));
+    try {
+      const rows = await this.db
+        .selectFrom("__donkeylabs_workflow_instances__")
+        .selectAll()
+        .where("status", "=", "running")
+        .execute();
+
+      return rows.map((r) => this.rowToInstance(r));
+    } catch (err: any) {
+      // Silently ignore errors if adapter was stopped during query
+      if (this.stopped && err?.message?.includes("destroyed")) return [];
+      throw err;
+    }
   }
 
   async getAllInstances(options: GetAllWorkflowsOptions = {}): Promise<WorkflowInstance[]> {
+    if (this.checkStopped()) return [];
+
     const { status, workflowName, limit = 100, offset = 0 } = options;
 
     let query = this.db.selectFrom("__donkeylabs_workflow_instances__").selectAll();
@@ -231,12 +276,13 @@ export class KyselyWorkflowAdapter implements WorkflowAdapter {
       completedAt: row.completed_at ? new Date(row.completed_at) : undefined,
       parentId: row.parent_id ?? undefined,
       branchName: row.branch_name ?? undefined,
+      metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
     };
   }
 
   /** Clean up old completed/failed/cancelled workflows */
   private async cleanup(): Promise<void> {
-    if (this.cleanupDays <= 0) return;
+    if (this.cleanupDays <= 0 || this.checkStopped()) return;
 
     try {
       const cutoff = new Date();
@@ -268,6 +314,7 @@ export class KyselyWorkflowAdapter implements WorkflowAdapter {
 
   /** Stop the adapter and cleanup timer */
   stop(): void {
+    this.stopped = true;
     if (this.cleanupTimer) {
       clearInterval(this.cleanupTimer);
       this.cleanupTimer = undefined;

package/src/core/workflow-executor.ts

@@ -0,0 +1,239 @@
+#!/usr/bin/env bun
+// Workflow Executor - Subprocess Entry Point
+// Thin shell that creates a WorkflowStateMachine with IPC event bridge.
+// The state machine owns all execution logic and persistence.
+
+import { connect } from "node:net";
+import type { Socket } from "node:net";
+import { Kysely } from "kysely";
+import { BunSqliteDialect } from "kysely-bun-sqlite";
+import Database from "bun:sqlite";
+import type { WorkflowEvent } from "./workflow-socket";
+import { WorkflowProxyConnection, createPluginsProxy, createCoreServicesProxy } from "./workflow-proxy";
+import type { WorkflowDefinition } from "./workflows";
+import { KyselyWorkflowAdapter } from "./workflow-adapter-kysely";
+import { WorkflowStateMachine, type StateMachineEvents } from "./workflow-state-machine";
+
+// ============================================
+// Types
+// ============================================
+
+interface ExecutorConfig {
+  instanceId: string;
+  workflowName: string;
+  input: any;
+  socketPath?: string;
+  tcpPort?: number;
+  modulePath: string;
+  dbPath: string;
+}
+
+// ============================================
+// Main Executor
+// ============================================
+
+async function main(): Promise<void> {
+  // Read config from stdin
+  const stdin = await Bun.stdin.text();
+  const config: ExecutorConfig = JSON.parse(stdin);
+
+  const { instanceId, workflowName, socketPath, tcpPort, modulePath, dbPath } = config;
+
+  // Connect to IPC socket
+  const socket = await connectToSocket(socketPath, tcpPort);
+  const proxyConnection = new WorkflowProxyConnection(socket);
+
+  // Create database connection + adapter (subprocess owns its own persistence)
+  const db = new Kysely<any>({
+    dialect: new BunSqliteDialect({
+      database: new Database(dbPath),
+    }),
+  });
+  const adapter = new KyselyWorkflowAdapter(db, { cleanupDays: 0 });
+
+  // Start heartbeat
+  const heartbeatInterval = setInterval(() => {
+    sendEvent(socket, {
+      type: "heartbeat",
+      instanceId,
+      timestamp: Date.now(),
+    });
+  }, 5000);
+
+  try {
+    // Send started event
+    sendEvent(socket, {
+      type: "started",
+      instanceId,
+      timestamp: Date.now(),
+    });
+
+    // Import the workflow module to get the definition
+    const module = await import(modulePath);
+    const definition = findWorkflowDefinition(module, workflowName, modulePath);
+
+    // Create proxy objects for plugin/core access via IPC
+    const plugins = createPluginsProxy(proxyConnection);
+    const coreServices = createCoreServicesProxy(proxyConnection);
+
+    // Create state machine with IPC event bridge
+    const sm = new WorkflowStateMachine({
+      adapter,
+      core: { ...coreServices, db } as any,
+      plugins,
+      events: createIpcEventBridge(socket, instanceId),
+      pollInterval: 1000,
+    });
+
+    // Run the state machine to completion
+    const result = await sm.run(instanceId, definition);
+
+    // Send completed event
+    sendEvent(socket, {
+      type: "completed",
+      instanceId,
+      timestamp: Date.now(),
+      output: result,
+    });
+  } catch (error) {
+    // Send failed event
+    sendEvent(socket, {
+      type: "failed",
+      instanceId,
+      timestamp: Date.now(),
+      error: error instanceof Error ? error.message : String(error),
+    });
+    process.exit(1);
+  } finally {
+    clearInterval(heartbeatInterval);
+    proxyConnection.close();
+    socket.end();
+    adapter.stop();
+    await db.destroy();
+  }
+
+  process.exit(0);
+}
+
+// ============================================
+// IPC Event Bridge
+// ============================================
+
+function createIpcEventBridge(socket: Socket, instanceId: string): StateMachineEvents {
+  return {
+    onStepStarted: (id, stepName, stepType) => {
+      sendEvent(socket, {
+        type: "step.started",
+        instanceId: id,
+        timestamp: Date.now(),
+        stepName,
+        stepType,
+      });
+    },
+    onStepCompleted: (id, stepName, output, nextStep) => {
+      sendEvent(socket, {
+        type: "step.completed",
+        instanceId: id,
+        timestamp: Date.now(),
+        stepName,
+        output,
+        nextStep,
+      });
+    },
+    onStepFailed: (id, stepName, error, attempts) => {
+      sendEvent(socket, {
+        type: "step.failed",
+        instanceId: id,
+        timestamp: Date.now(),
+        stepName,
+        error,
+      });
+    },
+    onStepRetry: () => {
+      // Retry is internal to the state machine - no IPC event needed
+    },
+    onProgress: (id, progress, currentStep, completed, total) => {
+      sendEvent(socket, {
+        type: "progress",
+        instanceId: id,
+        timestamp: Date.now(),
+        progress,
+        completedSteps: completed,
+        totalSteps: total,
+      });
+    },
+    onCompleted: () => {
+      // Handled by the main try/catch after sm.run() returns
+    },
+    onFailed: () => {
+      // Handled by the main try/catch after sm.run() throws
+    },
+  };
+}
+
+// ============================================
+// Socket Connection
+// ============================================
+
+function connectToSocket(socketPath?: string, tcpPort?: number): Promise<Socket> {
+  return new Promise((resolve, reject) => {
+    let socket: Socket;
+
+    if (socketPath) {
+      socket = connect(socketPath);
+    } else if (tcpPort) {
+      socket = connect(tcpPort, "127.0.0.1");
+    } else {
+      reject(new Error("No socket path or TCP port provided"));
+      return;
+    }
+
+    socket.once("connect", () => resolve(socket));
+    socket.once("error", (err) => reject(err));
+  });
+}
+
+// ============================================
+// Helpers
+// ============================================
+
+function sendEvent(socket: Socket, event: WorkflowEvent): void {
+  socket.write(JSON.stringify(event) + "\n");
+}
+
+function findWorkflowDefinition(
+  module: any,
+  workflowName: string,
+  modulePath: string,
+): WorkflowDefinition {
+  // Try named exports
+  for (const key of Object.keys(module)) {
+    const exported = module[key];
+    if (isWorkflowDefinition(exported) && exported.name === workflowName) {
+      return exported;
+    }
+  }
+
+  // Try default export
+  if (module.default && isWorkflowDefinition(module.default) && module.default.name === workflowName) {
+    return module.default;
+  }
+
+  throw new Error(`Workflow "${workflowName}" not found in module ${modulePath}`);
+}
+
+function isWorkflowDefinition(obj: any): obj is WorkflowDefinition {
+  return (
+    obj &&
+    typeof obj === "object" &&
+    typeof obj.name === "string" &&
+    obj.steps instanceof Map &&
+    typeof obj.startAt === "string"
+  );
+}
+
+// Run main
+main().catch((err) => {
+  console.error("[WorkflowExecutor] Fatal error:", err);
+  process.exit(1);
+});

package/src/core/workflow-proxy.ts

@@ -0,0 +1,238 @@
+// Workflow Proxy Utilities
+// Provides transparent access to plugins and core services from isolated workflow subprocess via IPC
+
+import type { Socket } from "node:net";
+import type { ProxyRequest, ProxyResponse } from "./workflow-socket";
+
+// ============================================
+// Types
+// ============================================
+
+export interface ProxyConnection {
+  /** Send a proxy request and wait for response */
+  call(target: "plugin" | "core", service: string, method: string, args: any[]): Promise<any>;
+  /** Close the connection */
+  close(): void;
+}
+
+interface PendingRequest {
+  resolve: (value: any) => void;
+  reject: (error: Error) => void;
+  timeout: ReturnType<typeof setTimeout>;
+}
+
+// ============================================
+// Proxy Connection Implementation
+// ============================================
+
+/**
+ * Creates a proxy connection that sends requests over a socket and handles responses.
+ * Used by the workflow executor subprocess to communicate with the main process.
+ */
+export class WorkflowProxyConnection implements ProxyConnection {
+  private socket: Socket;
+  private pendingRequests = new Map<string, PendingRequest>();
+  private requestCounter = 0;
+  private timeoutMs: number;
+  private buffer = "";
+
+  constructor(socket: Socket, timeoutMs = 30000) {
+    this.socket = socket;
+    this.timeoutMs = timeoutMs;
+
+    // Handle incoming data (proxy responses)
+    socket.on("data", (data) => {
+      this.buffer += data.toString();
+      this.processBuffer();
+    });
+
+    socket.on("error", (err) => {
+      // Reject all pending requests on socket error
+      for (const [requestId, pending] of this.pendingRequests) {
+        clearTimeout(pending.timeout);
+        pending.reject(new Error(`Socket error: ${err.message}`));
+      }
+      this.pendingRequests.clear();
+    });
+
+    socket.on("close", () => {
+      // Reject all pending requests on socket close
+      for (const [requestId, pending] of this.pendingRequests) {
+        clearTimeout(pending.timeout);
+        pending.reject(new Error("Socket closed"));
+      }
+      this.pendingRequests.clear();
+    });
+  }
+
+  private processBuffer(): void {
+    const lines = this.buffer.split("\n");
+    this.buffer = lines.pop() || "";
+
+    for (const line of lines) {
+      if (!line.trim()) continue;
+
+      try {
+        const response = JSON.parse(line) as ProxyResponse;
+        this.handleResponse(response);
+      } catch {
+        // Ignore invalid JSON - might be other message types
+      }
+    }
+  }
+
+  private handleResponse(response: ProxyResponse): void {
+    const pending = this.pendingRequests.get(response.requestId);
+    if (!pending) return;
+
+    clearTimeout(pending.timeout);
+    this.pendingRequests.delete(response.requestId);
+
+    if (response.type === "proxy.result") {
+      pending.resolve(response.result);
+    } else {
+      pending.reject(new Error(response.error ?? "Proxy call failed"));
+    }
+  }
+
+  async call(target: "plugin" | "core", service: string, method: string, args: any[]): Promise<any> {
+    const requestId = `req_${++this.requestCounter}_${Date.now()}`;
+
+    const request: ProxyRequest = {
+      type: "proxy.call",
+      requestId,
+      target,
+      service,
+      method,
+      args,
+    };
+
+    return new Promise((resolve, reject) => {
+      const timeout = setTimeout(() => {
+        this.pendingRequests.delete(requestId);
+        reject(new Error(`Proxy call timed out: ${target}.${service}.${method}`));
+      }, this.timeoutMs);
+
+      this.pendingRequests.set(requestId, { resolve, reject, timeout });
+
+      // Send request
+      this.socket.write(JSON.stringify(request) + "\n", (err) => {
+        if (err) {
+          clearTimeout(timeout);
+          this.pendingRequests.delete(requestId);
+          reject(new Error(`Failed to send proxy request: ${err.message}`));
+        }
+      });
+    });
+  }
+
+  close(): void {
+    // Cancel all pending requests
+    for (const [requestId, pending] of this.pendingRequests) {
+      clearTimeout(pending.timeout);
+      pending.reject(new Error("Connection closed"));
+    }
+    this.pendingRequests.clear();
+  }
+}
+
+// ============================================
+// Proxy Factories
+// ============================================
+
+/**
+ * Creates a proxy object for accessing a plugin service.
+ * Method calls are intercepted and forwarded via IPC to the main process.
+ *
+ * @example
+ * const usersProxy = createPluginProxy(connection, "users");
+ * const user = await usersProxy.getById("user_123"); // Calls main process
+ */
+export function createPluginProxy<T = Record<string, any>>(
+  connection: ProxyConnection,
+  pluginName: string
+): T {
+  return new Proxy({}, {
+    get(_target, prop) {
+      if (typeof prop === "symbol") return undefined;
+
+      // Return a function that calls the method via IPC
+      return async (...args: any[]) => {
+        return connection.call("plugin", pluginName, prop as string, args);
+      };
+    },
+  }) as T;
+}
+
+/**
+ * Creates a proxy object for accessing a core service.
+ * Method calls are intercepted and forwarded via IPC to the main process.
+ *
+ * @example
+ * const cacheProxy = createCoreProxy(connection, "cache");
+ * await cacheProxy.set("key", "value"); // Calls main process
+ */
+export function createCoreProxy<T = Record<string, any>>(
+  connection: ProxyConnection,
+  serviceName: string
+): T {
+  return new Proxy({}, {
+    get(_target, prop) {
+      if (typeof prop === "symbol") return undefined;
+
+      // Return a function that calls the method via IPC
+      return async (...args: any[]) => {
+        return connection.call("core", serviceName, prop as string, args);
+      };
+    },
+  }) as T;
+}
+
+/**
+ * Creates a full plugins proxy that lazily creates plugin proxies on access.
+ *
+ * @example
+ * const plugins = createPluginsProxy(connection);
+ * const user = await plugins.users.getById("user_123");
+ * const order = await plugins.orders.create({ ... });
+ */
+export function createPluginsProxy(connection: ProxyConnection): Record<string, any> {
+  const cache = new Map<string, any>();
+
+  return new Proxy({}, {
+    get(_target, prop) {
+      if (typeof prop === "symbol") return undefined;
+
+      const pluginName = prop as string;
+      if (!cache.has(pluginName)) {
+        cache.set(pluginName, createPluginProxy(connection, pluginName));
+      }
+      return cache.get(pluginName);
+    },
+  });
+}
+
+/**
+ * Creates a full core services proxy that lazily creates service proxies on access.
+ * Note: Some services like db require special handling as they can't be fully proxied.
+ *
+ * @example
+ * const core = createCoreServicesProxy(connection);
+ * await core.cache.set("key", "value");
+ * await core.events.emit("user.created", { userId: "123" });
+ */
+export function createCoreServicesProxy(connection: ProxyConnection): Record<string, any> {
+  const cache = new Map<string, any>();
+
+  return new Proxy({}, {
+    get(_target, prop) {
+      if (typeof prop === "symbol") return undefined;
+
+      const serviceName = prop as string;
+      if (!cache.has(serviceName)) {
+        cache.set(serviceName, createCoreProxy(connection, serviceName));
+      }
+      return cache.get(serviceName);
+    },
+  });
+}