@donkeylabs/server 2.0.27 → 2.0.29

package/src/core/subprocess-bootstrap.ts

@@ -32,6 +32,11 @@ export interface SubprocessPluginMetadata {
 export interface SubprocessBootstrapOptions {
   dbPath: string;
   coreConfig?: Record<string, any>;
+  sqlitePragmas?: {
+    busyTimeout?: number;
+    synchronous?: "OFF" | "NORMAL" | "FULL" | "EXTRA";
+    journalMode?: "DELETE" | "TRUNCATE" | "PERSIST" | "MEMORY" | "WAL" | "OFF";
+  };
   pluginMetadata: SubprocessPluginMetadata;
   startServices?: {
     cron?: boolean;
@@ -53,7 +58,15 @@ export async function bootstrapSubprocess(
   options: SubprocessBootstrapOptions
 ): Promise<SubprocessBootstrapResult> {
   const sqlite = new Database(options.dbPath);
-  sqlite.run("PRAGMA busy_timeout = 5000");
+  const pragmas = options.sqlitePragmas;
+  const busyTimeout = pragmas?.busyTimeout ?? 5000;
+  sqlite.run(`PRAGMA busy_timeout = ${busyTimeout}`);
+  if (pragmas?.journalMode) {
+    sqlite.run(`PRAGMA journal_mode = ${pragmas.journalMode}`);
+  }
+  if (pragmas?.synchronous) {
+    sqlite.run(`PRAGMA synchronous = ${pragmas.synchronous}`);
+  }
 
   const db = new Kysely<any>({
     dialect: new BunSqliteDialect({ database: sqlite }),

package/src/core/workflow-adapter-kysely.ts

@@ -258,6 +258,11 @@ export class KyselyWorkflowAdapter implements WorkflowAdapter {
         startedAt: sr.startedAt ? new Date(sr.startedAt) : undefined,
         completedAt: sr.completedAt ? new Date(sr.completedAt) : undefined,
         attempts: sr.attempts,
+        pollCount: sr.pollCount,
+        lastPolledAt: sr.lastPolledAt ? new Date(sr.lastPolledAt) : undefined,
+        loopCount: sr.loopCount,
+        lastLoopedAt: sr.lastLoopedAt ? new Date(sr.lastLoopedAt) : undefined,
+        loopStartedAt: sr.loopStartedAt ? new Date(sr.loopStartedAt) : undefined,
       };
     }
 

package/src/core/workflow-executor.ts

@@ -25,6 +25,11 @@ interface ExecutorConfig {
   pluginModulePaths: Record<string, string>;
   pluginConfigs: Record<string, any>;
   coreConfig?: Record<string, any>;
+  sqlitePragmas?: {
+    busyTimeout?: number;
+    synchronous?: "OFF" | "NORMAL" | "FULL" | "EXTRA";
+    journalMode?: "DELETE" | "TRUNCATE" | "PERSIST" | "MEMORY" | "WAL" | "OFF";
+  };
 }
 
 // ============================================
@@ -47,6 +52,7 @@ async function main(): Promise<void> {
     pluginModulePaths,
     pluginConfigs,
     coreConfig,
+    sqlitePragmas,
   } = config;
 
   const socket = await connectToSocket(socketPath, tcpPort);
@@ -71,6 +77,7 @@ async function main(): Promise<void> {
     const bootstrap = await bootstrapSubprocess({
       dbPath,
       coreConfig,
+      sqlitePragmas,
       pluginMetadata: {
         names: pluginNames,
         modulePaths: pluginModulePaths,
@@ -184,6 +191,27 @@ function createIpcEventBridge(socket: Socket, instanceId: string): StateMachineE
         error,
       });
     },
+    onStepPoll: (id, stepName, pollCount, done, result) => {
+      sendEvent(socket, {
+        type: "step.poll",
+        instanceId: id,
+        timestamp: Date.now(),
+        stepName,
+        pollCount,
+        done,
+        result,
+      });
+    },
+    onStepLoop: (id, stepName, loopCount, target) => {
+      sendEvent(socket, {
+        type: "step.loop",
+        instanceId: id,
+        timestamp: Date.now(),
+        stepName,
+        loopCount,
+        target,
+      });
+    },
     onStepRetry: () => {
       // Retry is internal to the state machine - no IPC event needed
     },

package/src/core/workflow-socket.ts

@@ -20,6 +20,8 @@ export type WorkflowEventType =
   | "step.started"
   | "step.completed"
   | "step.failed"
+  | "step.poll"
+  | "step.loop"
   | "progress"
   | "completed"
   | "failed"
@@ -41,6 +43,11 @@ export interface WorkflowEvent {
   totalSteps?: number;
   /** Next step to execute (for step.completed events) */
   nextStep?: string;
+  pollCount?: number;
+  done?: boolean;
+  result?: any;
+  loopCount?: number;
+  target?: string;
   /** Custom event name (for event type) */
   event?: string;
   /** Custom event payload or log data */
@@ -241,7 +248,24 @@ export class WorkflowSocketServerImpl implements WorkflowSocketServer {
 
     let buffer = "";
 
-    socket.on("data", async (data) => {
+    const queue: WorkflowMessage[] = [];
+    let processing = false;
+
+    const processQueue = async () => {
+      if (processing) return;
+      processing = true;
+      while (queue.length > 0) {
+        const message = queue.shift()!;
+        try {
+          await this.handleMessage(instanceId, message);
+        } catch (err) {
+          this.onError?.(err instanceof Error ? err : new Error(String(err)), instanceId);
+        }
+      }
+      processing = false;
+    };
+
+    socket.on("data", (data) => {
       buffer += data.toString();
 
       // Process complete messages (newline-delimited JSON)
@@ -253,11 +277,13 @@ export class WorkflowSocketServerImpl implements WorkflowSocketServer {
 
         try {
           const message = JSON.parse(line) as WorkflowMessage;
-          await this.handleMessage(instanceId, message);
+          queue.push(message);
         } catch (err) {
           this.onError?.(new Error(`Invalid message: ${line}`), instanceId);
         }
       }
+
+      processQueue().catch(() => undefined);
     });
 
     socket.on("error", (err) => {

package/src/core/workflow-state-machine.ts

@@ -12,6 +12,8 @@ import type {
   WorkflowContext,
   StepDefinition,
   TaskStepDefinition,
+  LoopStepDefinition,
+  PollStepDefinition,
   ParallelStepDefinition,
   ChoiceStepDefinition,
   PassStepDefinition,
@@ -28,6 +30,8 @@ export interface StateMachineEvents {
   onStepCompleted(instanceId: string, stepName: string, output: any, nextStep?: string): void;
   onStepFailed(instanceId: string, stepName: string, error: string, attempts: number): void;
   onStepRetry(instanceId: string, stepName: string, attempt: number, max: number, delayMs: number): void;
+  onStepPoll(instanceId: string, stepName: string, pollCount: number, done: boolean, result?: any): void;
+  onStepLoop(instanceId: string, stepName: string, loopCount: number, target: string): void;
   onProgress(instanceId: string, progress: number, currentStep: string, completed: number, total: number): void;
   onCompleted(instanceId: string, output: any): void;
   onFailed(instanceId: string, error: string): void;
@@ -136,11 +140,17 @@ export class WorkflowStateMachine {
       this.events.onStepStarted(instanceId, stepName, step.type);
 
       // Update step result as running
+      const previousStep = freshInstance.stepResults[stepName];
       const stepResult: StepResult = {
         stepName,
         status: "running",
-        startedAt: new Date(),
-        attempts: (freshInstance.stepResults[stepName]?.attempts ?? 0) + 1,
+        startedAt: previousStep?.startedAt ?? new Date(),
+        attempts: (previousStep?.attempts ?? 0) + 1,
+        pollCount: previousStep?.pollCount,
+        lastPolledAt: previousStep?.lastPolledAt,
+        loopCount: previousStep?.loopCount,
+        lastLoopedAt: previousStep?.lastLoopedAt,
+        loopStartedAt: previousStep?.loopStartedAt,
       };
       await this.adapter.updateInstance(instanceId, {
         currentStep: stepName,
@@ -166,6 +176,12 @@ export class WorkflowStateMachine {
           case "pass":
             output = await this.executePassStep(step, ctx);
             break;
+          case "poll":
+            output = await this.executePollStep(instanceId, step, ctx, definition);
+            break;
+          case "loop":
+            output = await this.executeLoopStep(instanceId, step, ctx);
+            break;
         }
 
         // Persist step completion
@@ -176,6 +192,8 @@ export class WorkflowStateMachine {
         if (step.type === "choice") {
           // Choice step returns { chosen: "nextStepName" }
           currentStepName = output?.chosen;
+        } else if (step.type === "loop" && output?.loopTo) {
+          currentStepName = output.loopTo;
         } else if (step.end) {
           currentStepName = undefined;
         } else if (step.next) {
@@ -445,6 +463,87 @@ export class WorkflowStateMachine {
     return output;
   }
 
+  private async executePollStep(
+    instanceId: string,
+    step: PollStepDefinition,
+    ctx: WorkflowContext,
+    _definition: WorkflowDefinition,
+  ): Promise<any> {
+    let input: any;
+
+    if (step.inputSchema) {
+      if (typeof step.inputSchema === "function") {
+        input = step.inputSchema(ctx.prev, ctx.input);
+      } else {
+        const parseResult = step.inputSchema.safeParse(ctx.input);
+        if (!parseResult.success) {
+          throw new Error(`Input validation failed: ${parseResult.error.message}`);
+        }
+        input = parseResult.data;
+      }
+    } else {
+      input = ctx.input;
+    }
+
+    let instance = await this.adapter.getInstance(instanceId);
+    const stepResult = instance?.stepResults[step.name];
+    const startedAt = stepResult?.startedAt ?? new Date();
+
+    if (instance && stepResult) {
+      stepResult.input = stepResult.input ?? input;
+      stepResult.pollCount = stepResult.pollCount ?? 0;
+      await this.adapter.updateInstance(instanceId, {
+        stepResults: { ...instance.stepResults, [step.name]: stepResult },
+      });
+    }
+
+    while (true) {
+      if (step.timeout && Date.now() - startedAt.getTime() > step.timeout) {
+        throw new Error(`Poll step "${step.name}" timed out`);
+      }
+
+      instance = await this.adapter.getInstance(instanceId);
+      const sr = instance?.stepResults[step.name];
+      const pollCount = sr?.pollCount ?? 0;
+
+      if (step.maxAttempts && pollCount >= step.maxAttempts) {
+        throw new Error(`Poll step "${step.name}" exceeded maxAttempts`);
+      }
+
+      if (step.interval > 0) {
+        await new Promise((resolve) => setTimeout(resolve, step.interval));
+      }
+
+      const result = await step.check(input, ctx);
+      const nextPollCount = pollCount + 1;
+
+      if (instance && sr) {
+        sr.pollCount = nextPollCount;
+        sr.lastPolledAt = new Date();
+        sr.output = result;
+        await this.adapter.updateInstance(instanceId, {
+          stepResults: { ...instance.stepResults, [step.name]: sr },
+        });
+      }
+
+      this.events.onStepPoll(instanceId, step.name, nextPollCount, result.done, result.result);
+
+      if (result.done) {
+        let output = result.result;
+
+        if (step.outputSchema) {
+          const parseResult = step.outputSchema.safeParse(output);
+          if (!parseResult.success) {
+            throw new Error(`Output validation failed: ${parseResult.error.message}`);
+          }
+          output = parseResult.data;
+        }
+
+        return output;
+      }
+    }
+  }
+
   private async executePassStep(
     step: PassStepDefinition,
     ctx: WorkflowContext,
@@ -458,6 +557,52 @@ export class WorkflowStateMachine {
     return ctx.input;
   }
 
+  private async executeLoopStep(
+    instanceId: string,
+    step: LoopStepDefinition,
+    ctx: WorkflowContext,
+  ): Promise<{ loopTo?: string }> {
+    const instance = await this.adapter.getInstance(instanceId);
+    const stepResult = instance?.stepResults[step.name] ?? {
+      stepName: step.name,
+      status: "running" as const,
+      attempts: 0,
+      startedAt: new Date(),
+    };
+    const loopStartedAt = stepResult.loopStartedAt ?? stepResult.startedAt ?? new Date();
+    const loopCount = stepResult.loopCount ?? 0;
+
+    if (step.timeout && Date.now() - loopStartedAt.getTime() > step.timeout) {
+      throw new Error(`Loop step "${step.name}" timed out`);
+    }
+
+    if (step.maxIterations && loopCount >= step.maxIterations) {
+      throw new Error(`Loop step "${step.name}" exceeded maxIterations`);
+    }
+
+    const shouldLoop = step.condition(ctx);
+
+    if (instance) {
+      stepResult.loopCount = shouldLoop ? loopCount + 1 : loopCount;
+      stepResult.loopStartedAt = loopStartedAt;
+      stepResult.lastLoopedAt = shouldLoop ? new Date() : stepResult.lastLoopedAt;
+      stepResult.output = { looped: shouldLoop };
+      await this.adapter.updateInstance(instanceId, {
+        stepResults: { ...instance.stepResults, [step.name]: stepResult },
+      });
+    }
+
+    if (shouldLoop) {
+      this.events.onStepLoop(instanceId, step.name, loopCount + 1, step.target);
+      if (step.interval && step.interval > 0) {
+        await new Promise((resolve) => setTimeout(resolve, step.interval));
+      }
+      return { loopTo: step.target };
+    }
+
+    return {};
+  }
+
   // ============================================
   // Context Building
   // ============================================