@donkeylabs/server 0.6.4 → 1.0.0

package/src/core/workflow-adapter-kysely.ts

@@ -0,0 +1,255 @@
+/**
+ * Kysely Workflow Adapter
+ *
+ * Implements the WorkflowAdapter interface using Kysely for the shared app database.
+ * This provides persistence for workflows, which previously had NO persistence (in-memory only).
+ */
+
+import type { Kysely } from "kysely";
+import type { WorkflowAdapter, WorkflowInstance, WorkflowStatus, StepResult } from "./workflows";
+
+export interface KyselyWorkflowAdapterConfig {
+  /** Auto-cleanup completed workflows older than N days (default: 30, 0 to disable) */
+  cleanupDays?: number;
+  /** Cleanup interval in ms (default: 3600000 = 1 hour) */
+  cleanupInterval?: number;
+}
+
+// Table type for Kysely
+interface WorkflowInstancesTable {
+  id: string;
+  workflow_name: string;
+  status: string;
+  current_step: string | null;
+  input: string;
+  output: string | null;
+  error: string | null;
+  step_results: string;
+  branch_instances: string | null;
+  created_at: string;
+  started_at: string | null;
+  completed_at: string | null;
+  parent_id: string | null;
+  branch_name: string | null;
+}
+
+interface Database {
+  __donkeylabs_workflow_instances__: WorkflowInstancesTable;
+}
+
+export class KyselyWorkflowAdapter implements WorkflowAdapter {
+  private db: Kysely<Database>;
+  private cleanupTimer?: ReturnType<typeof setInterval>;
+  private cleanupDays: number;
+
+  constructor(db: Kysely<any>, config: KyselyWorkflowAdapterConfig = {}) {
+    this.db = db as Kysely<Database>;
+    this.cleanupDays = config.cleanupDays ?? 30;
+
+    // Start cleanup timer
+    if (this.cleanupDays > 0) {
+      const interval = config.cleanupInterval ?? 3600000; // 1 hour
+      this.cleanupTimer = setInterval(() => this.cleanup(), interval);
+      // Run cleanup on startup
+      this.cleanup();
+    }
+  }
+
+  async createInstance(instance: Omit<WorkflowInstance, "id">): Promise<WorkflowInstance> {
+    const id = `wf_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`;
+
+    await this.db
+      .insertInto("__donkeylabs_workflow_instances__")
+      .values({
+        id,
+        workflow_name: instance.workflowName,
+        status: instance.status,
+        current_step: instance.currentStep ?? null,
+        input: JSON.stringify(instance.input),
+        output: instance.output !== undefined ? JSON.stringify(instance.output) : null,
+        error: instance.error ?? null,
+        step_results: JSON.stringify(instance.stepResults),
+        branch_instances: instance.branchInstances
+          ? JSON.stringify(instance.branchInstances)
+          : null,
+        created_at: instance.createdAt.toISOString(),
+        started_at: instance.startedAt?.toISOString() ?? null,
+        completed_at: instance.completedAt?.toISOString() ?? null,
+        parent_id: instance.parentId ?? null,
+        branch_name: instance.branchName ?? null,
+      })
+      .execute();
+
+    return { ...instance, id };
+  }
+
+  async getInstance(instanceId: string): Promise<WorkflowInstance | null> {
+    const row = await this.db
+      .selectFrom("__donkeylabs_workflow_instances__")
+      .selectAll()
+      .where("id", "=", instanceId)
+      .executeTakeFirst();
+
+    if (!row) return null;
+    return this.rowToInstance(row);
+  }
+
+  async updateInstance(instanceId: string, updates: Partial<WorkflowInstance>): Promise<void> {
+    const updateData: Partial<WorkflowInstancesTable> = {};
+
+    if (updates.status !== undefined) {
+      updateData.status = updates.status;
+    }
+    if (updates.currentStep !== undefined) {
+      updateData.current_step = updates.currentStep ?? null;
+    }
+    if (updates.output !== undefined) {
+      updateData.output = JSON.stringify(updates.output);
+    }
+    if (updates.error !== undefined) {
+      updateData.error = updates.error;
+    }
+    if (updates.stepResults !== undefined) {
+      updateData.step_results = JSON.stringify(updates.stepResults);
+    }
+    if (updates.branchInstances !== undefined) {
+      updateData.branch_instances = updates.branchInstances
+        ? JSON.stringify(updates.branchInstances)
+        : null;
+    }
+    if (updates.startedAt !== undefined) {
+      updateData.started_at = updates.startedAt?.toISOString() ?? null;
+    }
+    if (updates.completedAt !== undefined) {
+      updateData.completed_at = updates.completedAt?.toISOString() ?? null;
+    }
+
+    if (Object.keys(updateData).length === 0) return;
+
+    await this.db
+      .updateTable("__donkeylabs_workflow_instances__")
+      .set(updateData)
+      .where("id", "=", instanceId)
+      .execute();
+  }
+
+  async deleteInstance(instanceId: string): Promise<boolean> {
+    // Check if exists first since BunSqliteDialect doesn't report numDeletedRows properly
+    const exists = await this.db
+      .selectFrom("__donkeylabs_workflow_instances__")
+      .select("id")
+      .where("id", "=", instanceId)
+      .executeTakeFirst();
+
+    if (!exists) return false;
+
+    await this.db
+      .deleteFrom("__donkeylabs_workflow_instances__")
+      .where("id", "=", instanceId)
+      .execute();
+
+    return true;
+  }
+
+  async getInstancesByWorkflow(
+    workflowName: string,
+    status?: WorkflowStatus
+  ): Promise<WorkflowInstance[]> {
+    let query = this.db
+      .selectFrom("__donkeylabs_workflow_instances__")
+      .selectAll()
+      .where("workflow_name", "=", workflowName);
+
+    if (status) {
+      query = query.where("status", "=", status);
+    }
+
+    const rows = await query.orderBy("created_at", "desc").execute();
+    return rows.map((r) => this.rowToInstance(r));
+  }
+
+  async getRunningInstances(): Promise<WorkflowInstance[]> {
+    const rows = await this.db
+      .selectFrom("__donkeylabs_workflow_instances__")
+      .selectAll()
+      .where("status", "=", "running")
+      .execute();
+
+    return rows.map((r) => this.rowToInstance(r));
+  }
+
+  private rowToInstance(row: WorkflowInstancesTable): WorkflowInstance {
+    // Parse step results with proper Date handling
+    const rawStepResults = JSON.parse(row.step_results);
+    const stepResults: Record<string, StepResult> = {};
+
+    for (const [key, value] of Object.entries(rawStepResults)) {
+      const sr = value as any;
+      stepResults[key] = {
+        stepName: sr.stepName,
+        status: sr.status,
+        input: sr.input,
+        output: sr.output,
+        error: sr.error,
+        startedAt: sr.startedAt ? new Date(sr.startedAt) : undefined,
+        completedAt: sr.completedAt ? new Date(sr.completedAt) : undefined,
+        attempts: sr.attempts,
+      };
+    }
+
+    return {
+      id: row.id,
+      workflowName: row.workflow_name,
+      status: row.status as WorkflowStatus,
+      currentStep: row.current_step ?? undefined,
+      input: JSON.parse(row.input),
+      output: row.output ? JSON.parse(row.output) : undefined,
+      error: row.error ?? undefined,
+      stepResults,
+      branchInstances: row.branch_instances ? JSON.parse(row.branch_instances) : undefined,
+      createdAt: new Date(row.created_at),
+      startedAt: row.started_at ? new Date(row.started_at) : undefined,
+      completedAt: row.completed_at ? new Date(row.completed_at) : undefined,
+      parentId: row.parent_id ?? undefined,
+      branchName: row.branch_name ?? undefined,
+    };
+  }
+
+  /** Clean up old completed/failed/cancelled workflows */
+  private async cleanup(): Promise<void> {
+    if (this.cleanupDays <= 0) return;
+
+    try {
+      const cutoff = new Date();
+      cutoff.setDate(cutoff.getDate() - this.cleanupDays);
+
+      const result = await this.db
+        .deleteFrom("__donkeylabs_workflow_instances__")
+        .where((eb) =>
+          eb.or([
+            eb("status", "=", "completed"),
+            eb("status", "=", "failed"),
+            eb("status", "=", "cancelled"),
+            eb("status", "=", "timed_out"),
+          ])
+        )
+        .where("completed_at", "<", cutoff.toISOString())
+        .execute();
+
+      const numDeleted = Number(result[0]?.numDeletedRows ?? 0);
+      if (numDeleted > 0) {
+        console.log(`[Workflows] Cleaned up ${numDeleted} old workflow instances`);
+      }
+    } catch (err) {
+      console.error("[Workflows] Cleanup error:", err);
+    }
+  }
+
+  /** Stop the adapter and cleanup timer */
+  stop(): void {
+    if (this.cleanupTimer) {
+      clearInterval(this.cleanupTimer);
+      this.cleanupTimer = undefined;
+    }
+  }
+}

package/src/core.ts

@@ -1,6 +1,8 @@
 import { sql, type Kysely } from "kysely";
+import { existsSync } from "node:fs";
 import { readdir } from "node:fs/promises";
-import { join } from "node:path";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
 import type { z } from "zod";
 import type { Logger } from "./core/logger";
 import type { Cache } from "./core/cache";
@@ -12,6 +14,8 @@ import type { RateLimiter } from "./core/rate-limiter";
 import type { Errors, CustomErrorRegistry } from "./core/errors";
 import type { Workflows } from "./core/workflows";
 import type { Processes } from "./core/processes";
+import type { Audit } from "./core/audit";
+import type { WebSocketService } from "./core/websocket";
 
 export interface PluginRegistry {}
 
@@ -56,6 +60,8 @@ export interface CoreServices {
   errors: Errors;
   workflows: Workflows;
   processes: Processes;
+  audit: Audit;
+  websocket: WebSocketService;
 }
 
 /**
@@ -398,12 +404,103 @@ export class PluginManager {
     `.execute(this.core.db);
   }
 
+  /**
+   * Get the directory where core migrations are stored.
+   * This handles both development (src) and production (dist) scenarios.
+   */
+  private getCoreMigrationsDir(): string {
+    // Try to find migrations relative to this file's location
+    const currentDir = dirname(fileURLToPath(import.meta.url));
+    const migrationsDir = join(currentDir, "core", "migrations");
+    return migrationsDir;
+  }
+
+  /**
+   * Run migrations for a specific plugin/service with the given migrations directory.
+   */
+  private async runMigrationsForPlugin(pluginName: string, migrationDir: string): Promise<void> {
+    try {
+      const files = await readdir(migrationDir);
+      const migrationFiles = files.filter((f) => f.endsWith(".ts") || f.endsWith(".js"));
+
+      if (migrationFiles.length === 0) return;
+
+      console.log(`[Migration] checking: ${pluginName} at ${migrationDir}`);
+
+      for (const file of migrationFiles.sort()) {
+        // Check if this migration has already been applied
+        const isApplied = await this.isMigrationApplied(pluginName, file);
+        if (isApplied) {
+          console.log(`  - Skipping (already applied): ${file}`);
+          continue;
+        }
+
+        console.log(`  - Executing migration: ${file}`);
+        const migrationPath = join(migrationDir, file);
+
+        let migration;
+        try {
+          migration = await import(migrationPath);
+        } catch (importError) {
+          const err = importError instanceof Error ? importError : new Error(String(importError));
+          throw new Error(`Failed to import migration ${file}: ${err.message}`);
+        }
+
+        if (migration.up) {
+          try {
+            await migration.up(this.core.db);
+            // Record successful migration
+            await this.recordMigration(pluginName, file);
+            console.log(`    Success`);
+          } catch (e) {
+            console.error(`    Failed to run ${file}:`, e);
+            throw e; // Stop on migration failure - don't continue with inconsistent state
+          }
+        }
+      }
+    } catch (e) {
+      // Re-throw migration execution errors (they've already been logged)
+      // Only silently catch directory read errors (ENOENT)
+      const isDirectoryError =
+        e instanceof Error &&
+        ((e as NodeJS.ErrnoException).code === "ENOENT" ||
+          (e as NodeJS.ErrnoException).code === "ENOTDIR");
+      if (!isDirectoryError) {
+        throw e;
+      }
+    }
+  }
+
+  /**
+   * Run core migrations for built-in services (jobs, processes, workflows, audit).
+   * Core migrations are tracked with @core/ prefix in the same migrations table.
+   * This runs BEFORE plugin migrations to ensure core tables exist.
+   */
+  async migrateCore(): Promise<void> {
+    console.log("Running core migrations...");
+
+    const coreMigrationsDir = this.getCoreMigrationsDir();
+    const coreServices = ["jobs", "processes", "workflows", "audit"];
+
+    for (const service of coreServices) {
+      const migrationDir = join(coreMigrationsDir, service);
+      if (existsSync(migrationDir)) {
+        await this.runMigrationsForPlugin(`@core/${service}`, migrationDir);
+      }
+    }
+
+    console.log("Core migrations complete.");
+  }
+
   async migrate(): Promise<void> {
     console.log("Running migrations (File-System Based)...");
 
     // Ensure the migrations tracking table exists
     await this.ensureMigrationsTable();
 
+    // Run core migrations FIRST (before plugin migrations)
+    await this.migrateCore();
+
     const sortedPlugins = this.resolveOrder();
 
     for (const plugin of sortedPlugins) {

package/src/harness.ts

@@ -13,6 +13,11 @@ import {
   createErrors,
   createWorkflows,
   createProcesses,
+  createAudit,
+  createWebSocket,
+  KyselyJobAdapter,
+  KyselyWorkflowAdapter,
+  MemoryAuditAdapter,
 } from "./core/index";
 
 /**
@@ -27,18 +32,38 @@ export async function createTestHarness(targetPlugin: Plugin, dependencies: Plug
     dialect: new BunSqliteDialect({ database: new Database(":memory:") }),
   });
 
-  // 2. Initialize Core Services
+  // 2. Initialize Core Services with Kysely adapters for in-memory testing
   const logger = createLogger({ level: "warn" }); // Less verbose in tests
   const cache = createCache();
   const events = createEvents();
   const cron = createCron();
-  const jobs = createJobs({ events });
   const sse = createSSE();
   const rateLimiter = createRateLimiter();
   const errors = createErrors();
-  const workflows = createWorkflows({ events, jobs, sse });
+
+  // Use Kysely adapters with in-memory DB for jobs and workflows
+  const jobAdapter = new KyselyJobAdapter(db, { cleanupDays: 0 }); // No cleanup in tests
+  const workflowAdapter = new KyselyWorkflowAdapter(db, { cleanupDays: 0 });
+
+  const jobs = createJobs({
+    events,
+    adapter: jobAdapter,
+    persist: false, // Using Kysely adapter
+  });
+
+  const workflows = createWorkflows({
+    events,
+    jobs,
+    sse,
+    adapter: workflowAdapter,
+  });
+
   const processes = createProcesses({ events, autoRecoverOrphans: false });
 
+  // Use in-memory adapter for audit in tests
+  const audit = createAudit({ adapter: new MemoryAuditAdapter() });
+  const websocket = createWebSocket();
+
   const core: CoreServices = {
     db,
     config: { env: "test" },
@@ -52,6 +77,8 @@ export async function createTestHarness(targetPlugin: Plugin, dependencies: Plug
     errors,
     workflows,
     processes,
+    audit,
+    websocket,
   };
 
   const manager = new PluginManager(core);
@@ -62,7 +89,7 @@ export async function createTestHarness(targetPlugin: Plugin, dependencies: Plug
   }
   manager.register(targetPlugin);
 
-  // 4. Run Migrations (Real Kysely Migrations!)
+  // 4. Run Migrations (Core + Plugin Migrations!)
   await manager.migrate();
 
   // 5. Init Plugins

package/src/server.ts

@@ -16,8 +16,14 @@ import {
   createErrors,
   createWorkflows,
   createProcesses,
+  createAudit,
+  createWebSocket,
   extractClientIP,
   HttpError,
+  KyselyJobAdapter,
+  KyselyProcessAdapter,
+  KyselyWorkflowAdapter,
+  KyselyAuditAdapter,
   type LoggerConfig,
   type CacheConfig,
   type EventsConfig,
@@ -28,6 +34,8 @@ import {
   type ErrorsConfig,
   type WorkflowsConfig,
   type ProcessesConfig,
+  type AuditConfig,
+  type WebSocketConfig,
 } from "./core/index";
 import { zodSchemaToTs } from "./generator/zod-to-ts";
 
@@ -63,6 +71,14 @@ export interface ServerConfig {
   errors?: ErrorsConfig;
   workflows?: WorkflowsConfig;
   processes?: ProcessesConfig;
+  audit?: AuditConfig;
+  websocket?: WebSocketConfig;
+  /**
+   * Use legacy separate databases for core services.
+   * Set to true to keep using .donkeylabs/*.db files instead of shared DB.
+   * @deprecated Migrate to shared DB for better consistency.
+   */
+  useLegacyCoreDatabases?: boolean;
 }
 
 export class AppServer {
@@ -76,26 +92,56 @@ export class AppServer {
   constructor(options: ServerConfig) {
     this.port = options.port ?? 3000;
 
+    // Determine if we should use legacy databases
+    const useLegacy = options.useLegacyCoreDatabases ?? false;
+
     // Initialize core services
     const logger = createLogger(options.logger);
     const cache = createCache(options.cache);
     const events = createEvents(options.events);
     const cron = createCron(options.cron);
-    const jobs = createJobs({ ...options.jobs, events }); // Jobs can emit events
     const sse = createSSE(options.sse);
     const rateLimiter = createRateLimiter(options.rateLimiter);
     const errors = createErrors(options.errors);
+
+    // Create adapters - use Kysely by default, or legacy SQLite if requested
+    const jobAdapter = options.jobs?.adapter ?? (useLegacy ? undefined : new KyselyJobAdapter(options.db));
+    const workflowAdapter = options.workflows?.adapter ?? (useLegacy ? undefined : new KyselyWorkflowAdapter(options.db));
+    const auditAdapter = options.audit?.adapter ?? new KyselyAuditAdapter(options.db);
+
+    // Jobs can emit events and use Kysely adapter
+    const jobs = createJobs({
+      ...options.jobs,
+      events,
+      adapter: jobAdapter,
+      // Disable built-in persistence when using Kysely adapter
+      persist: useLegacy ? options.jobs?.persist : false,
+    });
+
+    // Workflows with Kysely adapter for persistence
     const workflows = createWorkflows({
       ...options.workflows,
       events,
       jobs,
       sse,
+      adapter: workflowAdapter,
     });
+
+    // Processes - still uses its own adapter pattern but can use Kysely
+    // Note: ProcessesImpl creates its own SqliteProcessAdapter internally
+    // For full Kysely support, we need to modify processes.ts
     const processes = createProcesses({
       ...options.processes,
       events,
     });
 
+    // New services
+    const audit = createAudit({
+      ...options.audit,
+      adapter: auditAdapter,
+    });
+    const websocket = createWebSocket(options.websocket);
+
     this.coreServices = {
       db: options.db,
       config: options.config ?? {},
@@ -109,6 +155,8 @@ export class AppServer {
       errors,
       workflows,
       processes,
+      audit,
+      websocket,
     };
 
     this.manager = new PluginManager(this.coreServices);
@@ -844,12 +892,18 @@ ${factoryFunction}
     // Stop SSE (closes all client connections)
     this.coreServices.sse.shutdown();
 
+    // Stop WebSocket connections
+    this.coreServices.websocket.shutdown();
+
     // Stop background services
     await this.coreServices.processes.shutdown();
     await this.coreServices.workflows.stop();
     await this.coreServices.jobs.stop();
     await this.coreServices.cron.stop();
 
+    // Stop audit service (cleanup timers)
+    this.coreServices.audit.stop();
+
     logger.info("Server shutdown complete");
   }
 }