@donkeylabs/server 2.0.37 → 2.1.0

package/src/core/events.ts

@@ -11,20 +11,33 @@ export interface Subscription {
   unsubscribe(): void;
 }
 
+export interface EventMetadata {
+  traceId?: string;
+  source?: string;
+  [key: string]: any;
+}
+
 export interface EventRecord {
   event: string;
   data: any;
   timestamp: Date;
+  metadata?: EventMetadata;
 }
 
 export interface EventAdapter {
-  publish(event: string, data: any): Promise<void>;
+  publish(event: string, data: any, metadata?: EventMetadata): Promise<void>;
   getHistory(event: string, limit?: number): Promise<EventRecord[]>;
+  /** Subscribe to events from other instances (for distributed adapters) */
+  subscribe?(callback: (event: string, data: any, metadata?: EventMetadata) => void): Promise<void>;
+  /** Stop the adapter and clean up resources */
+  stop?(): Promise<void>;
 }
 
 export interface EventsConfig {
   adapter?: EventAdapter;
   maxHistorySize?: number;
+  /** SSE service for auto-propagating distributed events to SSE clients */
+  sse?: import("./sse").SSE;
 }
 
 /**
@@ -41,11 +54,11 @@ export interface Events {
   /**
    * Emit a typed event (when EventRegistry is augmented)
    */
-  emit<K extends keyof EventRegistry>(event: K, data: EventRegistry[K]): Promise<void>;
+  emit<K extends keyof EventRegistry>(event: K, data: EventRegistry[K], metadata?: EventMetadata): Promise<void>;
   /**
    * Emit an untyped event (fallback for dynamic event names)
    */
-  emit<T = any>(event: string, data: T): Promise<void>;
+  emit<T = any>(event: string, data: T, metadata?: EventMetadata): Promise<void>;
 
   /**
    * Subscribe to a typed event (when EventRegistry is augmented)
@@ -67,6 +80,8 @@ export interface Events {
 
   off(event: string, handler?: EventHandler): void;
   getHistory(event: string, limit?: number): Promise<EventRecord[]>;
+  /** Stop the event bus and clean up resources */
+  stop(): Promise<void>;
 }
 
 // In-memory event adapter with history
@@ -78,11 +93,12 @@ export class MemoryEventAdapter implements EventAdapter {
     this.maxHistorySize = maxHistorySize;
   }
 
-  async publish(event: string, data: any): Promise<void> {
+  async publish(event: string, data: any, metadata?: EventMetadata): Promise<void> {
     const record: EventRecord = {
       event,
       data,
       timestamp: new Date(),
+      metadata,
     };
 
     this.history.push(record);
@@ -103,16 +119,41 @@ export class MemoryEventAdapter implements EventAdapter {
 class EventsImpl implements Events {
   private handlers = new Map<string, Set<EventHandler>>();
   private adapter: EventAdapter;
+  private sse?: import("./sse").SSE;
+  private stopped = false;
 
   constructor(config: EventsConfig = {}) {
     this.adapter = config.adapter ?? new MemoryEventAdapter(config.maxHistorySize);
+    this.sse = config.sse;
+
+    // If adapter supports distributed subscriptions, set up the callback
+    if (this.adapter.subscribe) {
+      this.adapter.subscribe((event, data, metadata) => {
+        // Dispatch to local handlers without re-publishing to adapter
+        this.dispatchToLocalHandlers(event, data, metadata);
+        // Propagate to SSE clients so browser subscribers see distributed events
+        this.sse?.broadcastAll(event, data);
+      });
+    }
   }
 
-  async emit<T = any>(event: string, data: T): Promise<void> {
+  async emit<T = any>(event: string, data: T, metadata?: EventMetadata): Promise<void> {
+    if (this.stopped) return;
+
     // Store in adapter (for history/persistence)
-    await this.adapter.publish(event, data);
+    await this.adapter.publish(event, data, metadata);
+
+    // Dispatch to local handlers
+    await this.dispatchToLocalHandlers(event, data, metadata);
+  }
 
-    // Notify handlers
+  /**
+   * Dispatch an event to locally registered handlers.
+   * Separated from emit() so distributed adapters can deliver remote events
+   * without re-publishing to the adapter.
+   */
+  private async dispatchToLocalHandlers(event: string, data: any, _metadata?: EventMetadata): Promise<void> {
+    // Notify exact-match handlers
     const eventHandlers = this.handlers.get(event);
     if (eventHandlers) {
       const promises: Promise<void>[] = [];
@@ -185,6 +226,12 @@ class EventsImpl implements Events {
     return this.adapter.getHistory(event, limit);
   }
 
+  async stop(): Promise<void> {
+    this.stopped = true;
+    await this.adapter.stop?.();
+    this.handlers.clear();
+  }
+
   private matchPattern(event: string, pattern: string): boolean {
     // Convert glob pattern to regex (e.g., "user.*" -> /^user\..*$/)
     const regex = new RegExp(

package/src/core/health.ts

@@ -0,0 +1,165 @@
+// Core Health Check Service
+// Provides liveness and readiness probes for production deployments
+
+import type { Kysely } from "kysely";
+
+export type HealthStatus = "healthy" | "degraded" | "unhealthy";
+
+export interface HealthCheckResult {
+  status: HealthStatus;
+  message?: string;
+  latencyMs?: number;
+}
+
+export interface HealthCheck {
+  name: string;
+  check: () => Promise<HealthCheckResult> | HealthCheckResult;
+  /** Whether failure of this check marks the service as unhealthy (default: true) */
+  critical?: boolean;
+}
+
+export interface HealthResponse {
+  status: HealthStatus;
+  uptime: number;
+  timestamp: string;
+  checks: Record<string, HealthCheckResult>;
+}
+
+export interface HealthConfig {
+  checks?: HealthCheck[];
+  /** Path for liveness probe (default: "/_health") */
+  livenessPath?: string;
+  /** Path for readiness probe (default: "/_ready") */
+  readinessPath?: string;
+  /** Timeout per check in ms (default: 5000) */
+  checkTimeout?: number;
+  /** Whether to register a built-in database check (default: true) */
+  dbCheck?: boolean;
+}
+
+export interface Health {
+  /** Register a health check */
+  register(check: HealthCheck): void;
+  /** Run all readiness checks */
+  check(): Promise<HealthResponse>;
+  /** Fast liveness probe (no external checks) */
+  liveness(isShuttingDown: boolean): HealthResponse;
+}
+
+class HealthImpl implements Health {
+  private checks: HealthCheck[] = [];
+  private startTime = Date.now();
+  private checkTimeout: number;
+
+  constructor(config: HealthConfig = {}) {
+    this.checkTimeout = config.checkTimeout ?? 5000;
+
+    if (config.checks) {
+      for (const check of config.checks) {
+        this.checks.push(check);
+      }
+    }
+  }
+
+  register(check: HealthCheck): void {
+    this.checks.push(check);
+  }
+
+  async check(): Promise<HealthResponse> {
+    const results: Record<string, HealthCheckResult> = {};
+    let overallStatus: HealthStatus = "healthy";
+
+    await Promise.all(
+      this.checks.map(async (check) => {
+        const start = Date.now();
+        try {
+          const result = await Promise.race([
+            Promise.resolve(check.check()),
+            new Promise<HealthCheckResult>((_, reject) =>
+              setTimeout(() => reject(new Error("Health check timed out")), this.checkTimeout)
+            ),
+          ]);
+          results[check.name] = {
+            ...result,
+            latencyMs: result.latencyMs ?? Date.now() - start,
+          };
+
+          const isCritical = check.critical !== false;
+          if (result.status === "unhealthy" && isCritical) {
+            overallStatus = "unhealthy";
+          } else if (result.status === "degraded" && overallStatus !== "unhealthy") {
+            overallStatus = "degraded";
+          } else if (result.status === "unhealthy" && !isCritical && overallStatus !== "unhealthy") {
+            overallStatus = "degraded";
+          }
+        } catch (err) {
+          const message = err instanceof Error ? err.message : String(err);
+          results[check.name] = {
+            status: "unhealthy",
+            message,
+            latencyMs: Date.now() - start,
+          };
+          const isCritical = check.critical !== false;
+          if (isCritical) {
+            overallStatus = "unhealthy";
+          } else if (overallStatus !== "unhealthy") {
+            overallStatus = "degraded";
+          }
+        }
+      })
+    );
+
+    return {
+      status: overallStatus,
+      uptime: Math.floor((Date.now() - this.startTime) / 1000),
+      timestamp: new Date().toISOString(),
+      checks: results,
+    };
+  }
+
+  liveness(isShuttingDown: boolean): HealthResponse {
+    return {
+      status: isShuttingDown ? "unhealthy" : "healthy",
+      uptime: Math.floor((Date.now() - this.startTime) / 1000),
+      timestamp: new Date().toISOString(),
+      checks: {},
+    };
+  }
+}
+
+/**
+ * Create a built-in database health check.
+ */
+export function createDbHealthCheck(db: Kysely<any>): HealthCheck {
+  return {
+    name: "database",
+    critical: true,
+    check: async () => {
+      const start = Date.now();
+      try {
+        await db.selectFrom(db.dynamic.ref("sqlite_master") as any)
+          .select(db.dynamic.ref("1") as any)
+          .execute()
+          .catch(async () => {
+            // Fallback for non-SQLite databases
+            const { sql } = await import("kysely");
+            await sql`SELECT 1`.execute(db);
+          });
+        return {
+          status: "healthy" as const,
+          latencyMs: Date.now() - start,
+        };
+      } catch (err) {
+        return {
+          status: "unhealthy" as const,
+          message: err instanceof Error ? err.message : String(err),
+          latencyMs: Date.now() - start,
+        };
+      }
+    },
+  };
+}
+
+export function createHealth(config?: HealthConfig): Health {
+  return new HealthImpl(config);
+}

package/src/core/index.ts

@@ -21,6 +21,7 @@ export {
 export {
   type Events,
   type EventHandler,
+  type EventMetadata,
   type Subscription,
   type EventRecord,
   type EventAdapter,
@@ -302,3 +303,14 @@ export {
   PersistentTransport,
   type PersistentTransportConfig,
 } from "./logs-transport";
+
+export {
+  type Health,
+  type HealthCheck,
+  type HealthCheckResult,
+  type HealthConfig,
+  type HealthResponse,
+  type HealthStatus,
+  createHealth,
+  createDbHealthCheck,
+} from "./health";

package/src/core/jobs.ts

@@ -56,6 +56,8 @@ export interface Job {
   lastHeartbeat?: Date;
   /** Current process state */
   processState?: ExternalJobProcessState;
+  /** Trace ID for distributed tracing */
+  traceId?: string;
 }
 
 export interface JobHandler<T = any, R = any> {
@@ -66,6 +68,8 @@ export interface JobHandlerContext {
   logger?: Logger;
   emit?: (event: string, data?: Record<string, any>) => Promise<void>;
   log?: (level: LogLevel, message: string, data?: Record<string, any>) => void;
+  /** Trace ID for distributed tracing */
+  traceId?: string;
 }
 
 /** Options for listing all jobs */
@@ -128,9 +132,9 @@ export interface Jobs {
   /** Register an external job (Python, Go, Shell, etc.) */
   registerExternal(name: string, config: ExternalJobConfig): void;
   /** Enqueue a job (works for both in-process and external jobs) */
-  enqueue<T = any>(name: string, data: T, options?: { maxAttempts?: number }): Promise<string>;
+  enqueue<T = any>(name: string, data: T, options?: { maxAttempts?: number; traceId?: string }): Promise<string>;
   /** Schedule a job to run at a specific time */
-  schedule<T = any>(name: string, data: T, runAt: Date, options?: { maxAttempts?: number }): Promise<string>;
+  schedule<T = any>(name: string, data: T, runAt: Date, options?: { maxAttempts?: number; traceId?: string }): Promise<string>;
   /** Get a job by ID */
   get(jobId: string): Promise<Job | null>;
   /** Cancel a pending job */
@@ -339,7 +343,7 @@ class JobsImpl implements Jobs {
     return this.externalConfigs.has(name);
   }
 
-  async enqueue<T = any>(name: string, data: T, options: { maxAttempts?: number } = {}): Promise<string> {
+  async enqueue<T = any>(name: string, data: T, options: { maxAttempts?: number; traceId?: string } = {}): Promise<string> {
     const isExternal = this.isExternalJob(name);
 
     if (!isExternal && !this.handlers.has(name)) {
@@ -355,6 +359,7 @@ class JobsImpl implements Jobs {
       maxAttempts: options.maxAttempts ?? this.defaultMaxAttempts,
       external: isExternal || undefined,
       processState: isExternal ? "spawning" : undefined,
+      traceId: options.traceId,
     });
 
     return job.id;
@@ -364,7 +369,7 @@ class JobsImpl implements Jobs {
     name: string,
     data: T,
     runAt: Date,
-    options: { maxAttempts?: number } = {}
+    options: { maxAttempts?: number; traceId?: string } = {}
   ): Promise<string> {
     const isExternal = this.isExternalJob(name);
 
@@ -382,6 +387,7 @@ class JobsImpl implements Jobs {
       maxAttempts: options.maxAttempts ?? this.defaultMaxAttempts,
       external: isExternal || undefined,
       processState: isExternal ? "spawning" : undefined,
+      traceId: options.traceId,
     });
 
     return job.id;
@@ -1147,6 +1153,7 @@ class JobsImpl implements Jobs {
         logger: scopedLogger,
         emit,
         log,
+        traceId: job.traceId,
       });
 
       await this.adapter.update(job.id, {

package/src/core/subprocess-bootstrap.ts

@@ -20,6 +20,7 @@ import {
   KyselyWorkflowAdapter,
   MemoryAuditAdapter,
   MemoryLogsAdapter,
+  createHealth,
 } from "./index";
 import { PluginManager, type CoreServices, type ConfiguredPlugin } from "../core";
 
@@ -96,6 +97,7 @@ export async function bootstrapSubprocess(
   const audit = createAudit({ adapter: auditAdapter });
   const websocket = createWebSocket();
   const storage = createStorage();
+  const health = createHealth({ dbCheck: false });
 
   const core: CoreServices = {
     db,
@@ -114,6 +116,7 @@ export async function bootstrapSubprocess(
     websocket,
     storage,
     logs,
+    health,
   };
 
   workflows.setCore(core);

package/src/core/workflow-executor.ts

@@ -226,6 +226,7 @@ function createIpcEventBridge(socket: Socket, instanceId: string): StateMachineE
         type: "progress",
         instanceId: id,
         timestamp: Date.now(),
+        stepName: currentStep,
         progress,
         completedSteps: completed,
         totalSteps: total,

package/src/core/workflow-state-machine.ts

@@ -751,12 +751,13 @@ export class WorkflowStateMachine {
 
     this.events.onStepCompleted(instanceId, stepName, output, nextStep);
 
-    // Calculate progress
+    // Calculate progress — re-fetch after persist to get accurate count
+    const updated = await this.adapter.getInstance(instanceId);
     const totalSteps = definition.steps.size;
-    const completedSteps = Object.values(instance.stepResults).filter(
-      (r) => r.status === "completed",
-    ).length + 1; // +1 for current step
-    const progress = Math.round((completedSteps / totalSteps) * 100);
+    const completedSteps = updated
+      ? Object.values(updated.stepResults).filter((r) => r.status === "completed").length
+      : 1;
+    const progress = Math.min(100, Math.round((completedSteps / totalSteps) * 100));
 
     this.events.onProgress(instanceId, progress, stepName, completedSteps, totalSteps);
   }

package/src/core/workflows.ts

@@ -1762,20 +1762,21 @@ class WorkflowsImpl implements Workflows {
         await this.emitEvent("workflow.progress", {
           instanceId,
           progress: event.progress,
+          currentStep: event.stepName,
           completedSteps: event.completedSteps,
           totalSteps: event.totalSteps,
         });
         if (this.sse) {
           this.sse.broadcast(`workflow:${instanceId}`, "progress", {
             progress: event.progress,
+            currentStep: event.stepName,
             completedSteps: event.completedSteps,
             totalSteps: event.totalSteps,
           });
           this.sse.broadcast("workflows:all", "workflow.progress", {
             instanceId,
             progress: event.progress,
-            completedSteps: event.completedSteps,
-            totalSteps: event.totalSteps,
+            currentStep: event.stepName,
           });
         }
         break;

package/src/core.ts

@@ -17,6 +17,7 @@ import type { Audit } from "./core/audit";
 import type { WebSocketService } from "./core/websocket";
 import type { Storage } from "./core/storage";
 import type { Logs } from "./core/logs";
+import type { Health } from "./core/health";
 
 // ============================================
 // Auto-detect caller module for plugin define()
@@ -137,6 +138,7 @@ export interface CoreServices {
   websocket: WebSocketService;
   storage: Storage;
   logs: Logs;
+  health: Health;
 }
 
 /**
@@ -182,6 +184,8 @@ export interface GlobalContext {
   ip: string;
   /** Unique request ID */
   requestId: string;
+  /** Trace ID for distributed tracing (from X-Request-Id/X-Trace-Id header, or defaults to requestId) */
+  traceId: string;
   /** Authenticated user (set by auth middleware) */
   user?: any;
   /**
@@ -431,7 +435,11 @@ export type InferService<T> = UnwrapPluginFactory<T> extends { _service: infer S
   : never;
 export type InferSchema<T> = UnwrapPluginFactory<T> extends { _schema: infer S } ? S : never;
 export type InferHandlers<T> = UnwrapPluginFactory<T> extends { handlers?: infer H } ? H : {};
-export type InferMiddleware<T> = UnwrapPluginFactory<T> extends { middleware?: (ctx: any) => infer M } ? M : {};
+export type InferMiddleware<T> = UnwrapPluginFactory<T> extends {
+  middleware?: (ctx: any, service: any) => infer M;
+}
+  ? M
+  : {};
 export type InferDependencies<T> = UnwrapPluginFactory<T> extends { _dependencies: infer D } ? D : readonly [];
 export type InferConfig<T> = T extends (config: infer C) => any ? C : void;
 export type InferEvents<T> = UnwrapPluginFactory<T> extends { events?: infer E } ? E : {};

package/src/generator/index.ts

@@ -19,6 +19,10 @@ export interface RouteInfo {
   outputSource?: string;
   /** SSE event schemas (for sse handler) */
   eventsSource?: Record<string, string>;
+  /** API version (semver) if from a versioned router */
+  version?: string;
+  /** Whether this route version is deprecated */
+  deprecated?: boolean;
 }
 
 export interface EventInfo {

package/src/harness.ts

@@ -17,6 +17,7 @@ import {
   createWebSocket,
   createStorage,
   createLogs,
+  createHealth,
   KyselyJobAdapter,
   KyselyWorkflowAdapter,
   MemoryAuditAdapter,
@@ -71,6 +72,7 @@ export async function createTestHarness(targetPlugin: Plugin, dependencies: Plug
   const websocket = createWebSocket();
   const storage = createStorage(); // Uses memory adapter by default
   const logs = createLogs({ adapter: new MemoryLogsAdapter(), events });
+  const health = createHealth({ dbCheck: false }); // No DB check in unit tests
 
   const core: CoreServices = {
     db,
@@ -89,6 +91,7 @@ export async function createTestHarness(targetPlugin: Plugin, dependencies: Plug
     websocket,
     storage,
     logs,
+    health,
   };
 
   const manager = new PluginManager(core);
@@ -158,28 +161,37 @@ export class TestApiClient extends ApiClientBase {
    * const user = await client.call("users.create", { name: "Test", email: "test@example.com" });
    * ```
    */
-  async call<TOutput = any>(route: string, input: any = {}): Promise<TOutput> {
+  async call<TOutput = any>(
+    route: string,
+    input: any = {},
+    options?: { version?: string }
+  ): Promise<TOutput> {
     const routeDef = this.routeMap.get(route);
     if (!routeDef) {
       throw new Error(`Route not found: ${route}. Available routes: ${[...this.routeMap.keys()].join(", ")}`);
     }
 
+    const versionHeaders: Record<string, string> = {};
+    if (options?.version) {
+      versionHeaders["X-API-Version"] = options.version;
+    }
+
     // Handle different handler types
     if (routeDef.handler === "typed" || routeDef.handler === "formData") {
-      return this.request(route, input);
+      return this.request(route, input, { headers: versionHeaders });
     } else if (routeDef.handler === "stream" || routeDef.handler === "html") {
       const response = await this.rawRequest(route, {
         method: "POST",
-        headers: { "Content-Type": "application/json" },
+        headers: { "Content-Type": "application/json", ...versionHeaders },
         body: JSON.stringify(input),
       });
       return response as any;
     } else if (routeDef.handler === "raw") {
-      const response = await this.rawRequest(route);
+      const response = await this.rawRequest(route, { headers: versionHeaders });
       return response as any;
     }
 
-    return this.request(route, input);
+    return this.request(route, input, { headers: versionHeaders });
   }
 
   /**

package/src/index.ts

@@ -4,6 +4,7 @@
 export {
   AppServer,
   type ServerConfig,
+  type ShutdownConfig,
   // Lifecycle hooks
   type HookContext,
   type OnReadyHandler,
@@ -91,14 +92,34 @@ export function defineConfig(config: DonkeylabsConfig): DonkeylabsConfig {
 // Re-export HttpError for custom error creation
 export { HttpError } from "./core/errors";
 
+// Versioning
+export {
+  parseSemVer,
+  type SemVer,
+  type VersioningConfig,
+  type DeprecationInfo,
+  type RouterOptions,
+} from "./versioning";
+
 // Core services types
 export {
   type Logger,
   type LogLevel,
   type ErrorFactory,
   type ErrorFactories,
+  type EventMetadata,
 } from "./core/index";
 
+// Health checks
+export {
+  type Health,
+  type HealthCheck,
+  type HealthCheckResult,
+  type HealthConfig,
+  type HealthResponse,
+  type HealthStatus,
+} from "./core/health";
+
 // Logs (persistent logging)
 export {
   type Logs,

package/src/router.ts

@@ -2,6 +2,7 @@
 import { z } from "zod";
 import type { GlobalContext, PluginHandlerRegistry } from "./core";
 import type { MiddlewareDefinition } from "./middleware";
+import type { RouterOptions, DeprecationInfo } from "./versioning";
 
 export type ServerContext = GlobalContext;
 
@@ -255,6 +256,8 @@ export interface IRouter {
   getRoutes(): RouteDefinition<any, any, any>[];
   getMetadata(): RouteMetadata[];
   getPrefix(): string;
+  getVersion(): string | undefined;
+  getDeprecation(): DeprecationInfo | undefined;
 }
 
 export class Router implements IRouter {
@@ -262,9 +265,13 @@ export class Router implements IRouter {
   private childRouters: IRouter[] = [];
   private prefix: string;
   private _middlewareStack: MiddlewareDefinition[] = [];
+  private _version?: string;
+  private _deprecated?: DeprecationInfo;
 
-  constructor(prefix: string = "") {
+  constructor(prefix: string = "", options?: RouterOptions) {
     this.prefix = prefix;
+    this._version = options?.version;
+    this._deprecated = options?.deprecated;
   }
 
   route(name: string): IRouteBuilder<this> {
@@ -277,6 +284,11 @@ export class Router implements IRouter {
       const fullPrefix = this.prefix ? `${this.prefix}.${prefixOrRouter}` : prefixOrRouter;
       const childRouter = new Router(fullPrefix);
       childRouter._middlewareStack = [...this._middlewareStack];
+      // Inherit parent version unless child overrides
+      if (this._version && !childRouter._version) {
+        childRouter._version = this._version;
+        childRouter._deprecated = this._deprecated;
+      }
       // Track child router so its routes are included in getRoutes()
       this.childRouters.push(childRouter);
       return childRouter;
@@ -355,6 +367,14 @@ export class Router implements IRouter {
   getPrefix(): string {
     return this.prefix;
   }
+
+  getVersion(): string | undefined {
+    return this._version;
+  }
+
+  getDeprecation(): DeprecationInfo | undefined {
+    return this._deprecated;
+  }
 }
 
 /** Creates a Proxy that intercepts middleware method calls and adds them to the router's middleware stack */
@@ -372,7 +392,7 @@ function createMiddlewareBuilderProxy<TRouter extends Router>(router: TRouter):
   });
 }
 
-export const createRouter = (prefix?: string): IRouter => new Router(prefix);
+export const createRouter = (prefix?: string, options?: RouterOptions): IRouter => new Router(prefix, options);
 
 /**
  * Define a route with type inference for input/output schemas.