@donkeylabs/server 2.0.21 → 2.0.22

package/src/core/logs.ts

@@ -0,0 +1,398 @@
+/**
+ * Core Logs Service
+ *
+ * Persistent, filterable, event-driven logging.
+ * Writes log entries to a separate logs database with buffered writes.
+ * Emits events so users can build their own SSE routes or subscribers.
+ */
+
+import type { Events } from "./events";
+import type { LogLevel } from "./logger";
+
+// ============================================
+// Types
+// ============================================
+
+export type LogSource = "system" | "cron" | "job" | "workflow" | "plugin" | "route";
+
+export interface PersistentLogEntry {
+  id: string;
+  timestamp: Date;
+  level: LogLevel;
+  message: string;
+  source: LogSource;
+  sourceId?: string;
+  tags?: string[];
+  data?: Record<string, any>;
+  context?: Record<string, any>;
+}
+
+export interface LogsQueryFilters {
+  /** Filter by source type */
+  source?: LogSource;
+  /** Filter by source identifier */
+  sourceId?: string;
+  /** Filter by minimum log level */
+  level?: LogLevel;
+  /** Filter by tags (entries must contain all specified tags) */
+  tags?: string[];
+  /** Search message text (LIKE on message) */
+  search?: string;
+  /** Filter by date range (start) */
+  startDate?: Date;
+  /** Filter by date range (end) */
+  endDate?: Date;
+  /** Maximum number of results (default: 100) */
+  limit?: number;
+  /** Offset for pagination */
+  offset?: number;
+}
+
+export interface LogsRetentionConfig {
+  /** Default retention in days (default: 14) */
+  defaultDays?: number;
+  /** Per-source retention overrides in days */
+  bySource?: Partial<Record<LogSource, number>>;
+  /** Cleanup interval in ms (default: 86400000 = 24h) */
+  cleanupInterval?: number;
+}
+
+export interface LogsConfig {
+  /** Custom adapter (defaults to KyselyLogsAdapter) */
+  adapter?: LogsAdapter;
+  /** Events service for emitting log events */
+  events?: Events;
+  /** Retention configuration */
+  retention?: LogsRetentionConfig;
+  /** Minimum level for persistence (default: "info") */
+  minLevel?: LogLevel;
+  /** Buffer flush interval in ms (default: 50) */
+  flushInterval?: number;
+  /** Maximum buffer size before forced flush (default: 100) */
+  maxBufferSize?: number;
+  /** Database path (default: ".donkeylabs/logs.db") */
+  dbPath?: string;
+}
+
+// ============================================
+// Adapter Interface
+// ============================================
+
+export interface LogsAdapter {
+  /** Write a batch of log entries */
+  writeBatch(entries: PersistentLogEntry[]): Promise<void>;
+  /** Write a single log entry */
+  write(entry: PersistentLogEntry): Promise<void>;
+  /** Query log entries with filters */
+  query(filters: LogsQueryFilters): Promise<PersistentLogEntry[]>;
+  /** Get log entries by source */
+  getBySource(source: LogSource, sourceId?: string, limit?: number): Promise<PersistentLogEntry[]>;
+  /** Count log entries matching filters */
+  count(filters: LogsQueryFilters): Promise<number>;
+  /** Delete entries older than a given date, optionally for a specific source */
+  deleteOlderThan(date: Date, source?: LogSource): Promise<number>;
+  /** Stop the adapter (cleanup resources) */
+  stop(): void;
+}
+
+// ============================================
+// Service Interface
+// ============================================
+
+export interface Logs {
+  /** Write a log entry (synchronous, enqueues to buffer) */
+  write(entry: Omit<PersistentLogEntry, "id" | "timestamp">): void;
+  /** Query log entries with filters */
+  query(filters: LogsQueryFilters): Promise<PersistentLogEntry[]>;
+  /** Get log entries by source */
+  getBySource(source: LogSource, sourceId?: string, limit?: number): Promise<PersistentLogEntry[]>;
+  /** Count log entries matching filters */
+  count(filters: LogsQueryFilters): Promise<number>;
+  /** Flush the write buffer */
+  flush(): Promise<void>;
+  /** Stop the logs service (flush + cleanup) */
+  stop(): void;
+}
+
+// ============================================
+// Log Level Ordering
+// ============================================
+
+const LOG_LEVEL_VALUES: Record<LogLevel, number> = {
+  debug: 0,
+  info: 1,
+  warn: 2,
+  error: 3,
+};
+
+// ============================================
+// In-Memory Adapter (for testing)
+// ============================================
+
+export class MemoryLogsAdapter implements LogsAdapter {
+  private entries: PersistentLogEntry[] = [];
+
+  async writeBatch(entries: PersistentLogEntry[]): Promise<void> {
+    this.entries.push(...entries);
+  }
+
+  async write(entry: PersistentLogEntry): Promise<void> {
+    this.entries.push(entry);
+  }
+
+  async query(filters: LogsQueryFilters): Promise<PersistentLogEntry[]> {
+    let results = [...this.entries];
+
+    if (filters.source) {
+      results = results.filter((e) => e.source === filters.source);
+    }
+    if (filters.sourceId) {
+      results = results.filter((e) => e.sourceId === filters.sourceId);
+    }
+    if (filters.level) {
+      const minLevel = LOG_LEVEL_VALUES[filters.level];
+      results = results.filter((e) => LOG_LEVEL_VALUES[e.level] >= minLevel);
+    }
+    if (filters.tags && filters.tags.length > 0) {
+      results = results.filter(
+        (e) => e.tags && filters.tags!.every((t) => e.tags!.includes(t))
+      );
+    }
+    if (filters.search) {
+      const searchLower = filters.search.toLowerCase();
+      results = results.filter((e) =>
+        e.message.toLowerCase().includes(searchLower)
+      );
+    }
+    if (filters.startDate) {
+      results = results.filter((e) => e.timestamp >= filters.startDate!);
+    }
+    if (filters.endDate) {
+      results = results.filter((e) => e.timestamp <= filters.endDate!);
+    }
+
+    // Sort by timestamp descending (newest first)
+    results.sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime());
+
+    const offset = filters.offset ?? 0;
+    const limit = filters.limit ?? 100;
+    return results.slice(offset, offset + limit);
+  }
+
+  async getBySource(
+    source: LogSource,
+    sourceId?: string,
+    limit: number = 100
+  ): Promise<PersistentLogEntry[]> {
+    return this.query({ source, sourceId, limit });
+  }
+
+  async count(filters: LogsQueryFilters): Promise<number> {
+    const results = await this.query({ ...filters, limit: undefined, offset: undefined });
+    return results.length;
+  }
+
+  async deleteOlderThan(date: Date, source?: LogSource): Promise<number> {
+    const before = this.entries.length;
+    this.entries = this.entries.filter((e) => {
+      if (source && e.source !== source) return true;
+      return e.timestamp >= date;
+    });
+    return before - this.entries.length;
+  }
+
+  stop(): void {
+    // No cleanup needed
+  }
+}
+
+// ============================================
+// Service Implementation
+// ============================================
+
+const MAX_BUFFER_OVERFLOW = 10_000;
+
+class LogsImpl implements Logs {
+  private adapter: LogsAdapter;
+  private events?: Events;
+  private buffer: PersistentLogEntry[] = [];
+  private flushTimer: ReturnType<typeof setInterval> | null = null;
+  private cleanupTimer: ReturnType<typeof setInterval> | null = null;
+  private maxBufferSize: number;
+  private minLevel: number;
+  private retention: LogsRetentionConfig;
+  private stopped = false;
+  private flushing = false;
+
+  constructor(config: LogsConfig = {}) {
+    this.adapter = config.adapter ?? new MemoryLogsAdapter();
+    this.events = config.events;
+    this.maxBufferSize = config.maxBufferSize ?? 100;
+    this.minLevel = LOG_LEVEL_VALUES[config.minLevel ?? "info"];
+    this.retention = config.retention ?? {};
+
+    // Start flush timer
+    const flushInterval = config.flushInterval ?? 50;
+    this.flushTimer = setInterval(() => {
+      this.flush().catch((err) => {
+        // Use console.log to avoid circular logging
+        console.error("[Logs] Flush error:", err);
+      });
+    }, flushInterval);
+
+    // Start retention cleanup timer
+    const cleanupInterval = this.retention.cleanupInterval ?? 86400000; // 24h
+    this.cleanupTimer = setInterval(() => {
+      this.runCleanup().catch((err) => {
+        console.error("[Logs] Cleanup error:", err);
+      });
+    }, cleanupInterval);
+  }
+
+  write(entry: Omit<PersistentLogEntry, "id" | "timestamp">): void {
+    if (this.stopped) return;
+
+    // Check minimum level
+    if (LOG_LEVEL_VALUES[entry.level] < this.minLevel) return;
+
+    const fullEntry: PersistentLogEntry = {
+      ...entry,
+      id: `log_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`,
+      timestamp: new Date(),
+    };
+
+    this.buffer.push(fullEntry);
+
+    // Check buffer overflow
+    if (this.buffer.length > MAX_BUFFER_OVERFLOW) {
+      console.warn(
+        `[Logs] Buffer overflow (${this.buffer.length} entries), dropping oldest entries`
+      );
+      this.buffer = this.buffer.slice(-this.maxBufferSize);
+    }
+
+    // Flush if buffer is full
+    if (this.buffer.length >= this.maxBufferSize) {
+      this.flush().catch((err) => {
+        console.error("[Logs] Flush error:", err);
+      });
+    }
+  }
+
+  async query(filters: LogsQueryFilters): Promise<PersistentLogEntry[]> {
+    return this.adapter.query(filters);
+  }
+
+  async getBySource(
+    source: LogSource,
+    sourceId?: string,
+    limit?: number
+  ): Promise<PersistentLogEntry[]> {
+    return this.adapter.getBySource(source, sourceId, limit);
+  }
+
+  async count(filters: LogsQueryFilters): Promise<number> {
+    return this.adapter.count(filters);
+  }
+
+  async flush(): Promise<void> {
+    if (this.buffer.length === 0 || this.flushing) return;
+
+    this.flushing = true;
+    const entries = this.buffer.splice(0);
+
+    try {
+      await this.adapter.writeBatch(entries);
+
+      // Emit events for each entry after successful write
+      if (this.events) {
+        for (const entry of entries) {
+          try {
+            // All logs
+            await this.events.emit("log.created", entry);
+            // By source type: "log.cron", "log.workflow"
+            await this.events.emit(`log.${entry.source}`, entry);
+            // Specific: "log.cron.cleanup-task", "log.workflow.wf_123"
+            if (entry.sourceId) {
+              await this.events.emit(
+                `log.${entry.source}.${entry.sourceId}`,
+                entry
+              );
+            }
+          } catch (err) {
+            // Don't let event emission errors break the flush
+            console.error("[Logs] Event emission error:", err);
+          }
+        }
+      }
+    } catch (err) {
+      // Put entries back if write failed (they'll be retried next flush)
+      this.buffer.unshift(...entries);
+      // Silently catch - will retry next flush
+      // Use console to avoid circular logging
+      console.error("[Logs] Write batch failed, will retry:", err);
+    } finally {
+      this.flushing = false;
+    }
+  }
+
+  stop(): void {
+    this.stopped = true;
+
+    if (this.flushTimer) {
+      clearInterval(this.flushTimer);
+      this.flushTimer = null;
+    }
+
+    if (this.cleanupTimer) {
+      clearInterval(this.cleanupTimer);
+      this.cleanupTimer = null;
+    }
+
+    this.adapter.stop();
+  }
+
+  private async runCleanup(): Promise<void> {
+    const defaultDays = this.retention.defaultDays ?? 14;
+    const bySource = this.retention.bySource ?? {};
+
+    // Get all source types to clean
+    const sources: LogSource[] = [
+      "system",
+      "cron",
+      "job",
+      "workflow",
+      "plugin",
+      "route",
+    ];
+
+    for (const source of sources) {
+      const days = bySource[source] ?? defaultDays;
+      if (days <= 0) continue;
+
+      const cutoff = new Date();
+      cutoff.setDate(cutoff.getDate() - days);
+
+      try {
+        const deleted = await this.adapter.deleteOlderThan(cutoff, source);
+        if (deleted > 0) {
+          console.log(
+            `[Logs] Cleaned up ${deleted} old ${source} log entries (>${days} days)`
+          );
+        }
+      } catch (err: any) {
+        // Silently ignore table-not-found errors
+        if (err?.message?.includes("no such table")) return;
+        console.error(`[Logs] Cleanup error for ${source}:`, err);
+      }
+    }
+  }
+}
+
+// ============================================
+// Factory Function
+// ============================================
+
+export function createLogs(config?: LogsConfig): Logs {
+  return new LogsImpl(config);
+}

package/src/core/workflow-executor.ts

@@ -44,10 +44,10 @@ async function main(): Promise<void> {
   const proxyConnection = new WorkflowProxyConnection(socket);
 
   // Create database connection + adapter (subprocess owns its own persistence)
+  const sqlite = new Database(dbPath);
+  sqlite.run("PRAGMA busy_timeout = 5000");
   const db = new Kysely<any>({
-    dialect: new BunSqliteDialect({
-      database: new Database(dbPath),
-    }),
+    dialect: new BunSqliteDialect({ database: sqlite }),
   });
   const adapter = new KyselyWorkflowAdapter(db, { cleanupDays: 0 });
 
@@ -76,10 +76,19 @@ async function main(): Promise<void> {
     const plugins = createPluginsProxy(proxyConnection);
     const coreServices = createCoreServicesProxy(proxyConnection);
 
+    // Wrap coreServices proxy so that `db` resolves locally instead of via IPC.
+    // Spreading a Proxy with no ownKeys trap loses all proxied properties.
+    const coreWithDb = new Proxy(coreServices, {
+      get(target, prop, receiver) {
+        if (prop === "db") return db;
+        return Reflect.get(target, prop, receiver);
+      },
+    });
+
     // Create state machine with IPC event bridge
     const sm = new WorkflowStateMachine({
       adapter,
-      core: { ...coreServices, db } as any,
+      core: coreWithDb as any,
       plugins,
       events: createIpcEventBridge(socket, instanceId),
       pollInterval: 1000,

package/src/core/workflows.test.ts

@@ -412,6 +412,62 @@ describe("WorkflowDefinition", () => {
     expect(isolatedWf.isolated).toBe(true);
     expect(inlineWf.isolated).toBe(false);
   });
+
+  it("should auto-detect sourceModule as a valid file:// URL after build()", () => {
+    const wf = workflow("auto-detect")
+      .task("s", { handler: async () => 1 })
+      .build();
+
+    expect(wf.sourceModule).toBeDefined();
+    expect(wf.sourceModule).toMatch(/^file:\/\//);
+    // Should point to this test file
+    expect(wf.sourceModule).toContain("workflows.test.ts");
+  });
+});
+
+describe("register() with auto-detected sourceModule", () => {
+  let workflows: ReturnType<typeof createWorkflows>;
+  let adapter: MemoryWorkflowAdapter;
+
+  beforeEach(() => {
+    adapter = new MemoryWorkflowAdapter();
+    workflows = createWorkflows({ adapter });
+  });
+
+  afterEach(async () => {
+    await workflows.stop();
+  });
+
+  it("should not warn when registering isolated workflow with auto-detected sourceModule", () => {
+    const wf = workflow("auto-isolated")
+      .task("s", { handler: async () => 1 })
+      .build();
+
+    // sourceModule should be set by build()
+    expect(wf.sourceModule).toBeDefined();
+
+    const warnings: string[] = [];
+    const origWarn = console.warn;
+    console.warn = (...args: any[]) => warnings.push(args.join(" "));
+    try {
+      workflows.register(wf);
+    } finally {
+      console.warn = origWarn;
+    }
+
+    expect(warnings.filter((w) => w.includes("no modulePath"))).toHaveLength(0);
+  });
+
+  it("should prefer explicit modulePath over auto-detected sourceModule", () => {
+    const wf = workflow("explicit-path")
+      .task("s", { handler: async () => 1 })
+      .build();
+
+    // Register with explicit modulePath
+    expect(() => {
+      workflows.register(wf, { modulePath: "file:///explicit/path.ts" });
+    }).not.toThrow();
+  });
 });
 
 describe("Choice steps (inline)", () => {

package/src/core/workflows.ts

@@ -12,9 +12,10 @@ import type { Events } from "./events";
 import type { Jobs } from "./jobs";
 import type { SSE } from "./sse";
 import type { z } from "zod";
+import { sql } from "kysely";
 import type { CoreServices } from "../core";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
 import {
   createWorkflowSocketServer,
   type WorkflowSocketServer,
@@ -24,6 +25,31 @@ import {
 import { isProcessAlive } from "./external-jobs";
 import { WorkflowStateMachine, type StateMachineEvents } from "./workflow-state-machine";
 
+// ============================================
+// Auto-detect caller module for isolated workflows
+// ============================================
+
+const WORKFLOWS_FILE = resolve(fileURLToPath(import.meta.url));
+
+/**
+ * Walk the call stack to find the file that invoked build().
+ * Returns a file:// URL string or undefined if detection fails.
+ */
+function captureCallerUrl(): string | undefined {
+  const stack = new Error().stack ?? "";
+  for (const line of stack.split("\n").slice(1)) {
+    const match = line.match(/at\s+(?:.*?\s+\(?)?([^\s():]+):\d+:\d+/);
+    if (match) {
+      let filePath = match[1];
+      if (filePath.startsWith("file://")) filePath = fileURLToPath(filePath);
+      if (filePath.startsWith("native")) continue;
+      filePath = resolve(filePath);
+      if (filePath !== WORKFLOWS_FILE) return pathToFileURL(filePath).href;
+    }
+  }
+  return undefined;
+}
+
 // Type helper for Zod schema inference
 type ZodSchema = z.ZodTypeAny;
 type InferZodOutput<T extends ZodSchema> = z.infer<T>;
@@ -144,6 +170,8 @@ export interface WorkflowDefinition {
    * Set to false for lightweight workflows that benefit from inline execution.
    */
   isolated?: boolean;
+  /** Auto-detected module URL where this workflow was built. Used as fallback for isolated execution. */
+  sourceModule?: string;
 }
 
 // ============================================
@@ -576,6 +604,7 @@ export class WorkflowBuilder {
       timeout: this._timeout,
       defaultRetry: this._defaultRetry,
       isolated: this._isolated,
+      sourceModule: captureCallerUrl(),
     };
   }
 }
@@ -617,10 +646,15 @@ export interface WorkflowsConfig {
 export interface WorkflowRegisterOptions {
   /**
    * Module path for isolated workflows.
-   * Required when workflow.isolated !== false and running in isolated mode.
-   * Use `import.meta.url` to get the current module's path.
+   * Auto-detected from the call site of `build()` in most cases.
+   * Only needed if the workflow definition is re-exported from a different
+   * module than the one that calls `build()`.
    *
    * @example
+   * // Usually not needed — auto-detected:
+   * workflows.register(myWorkflow);
+   *
+   * // Override when re-exporting from another module:
    * workflows.register(myWorkflow, { modulePath: import.meta.url });
    */
   modulePath?: string;
@@ -649,6 +683,8 @@ export interface Workflows {
   stop(): Promise<void>;
   /** Set core services (called after initialization to resolve circular dependency) */
   setCore(core: CoreServices): void;
+  /** Resolve dbPath from the database instance (call after setCore, before resume) */
+  resolveDbPath(): Promise<void>;
   /** Set plugin services (called after plugins are initialized) */
   setPlugins(plugins: Record<string, any>): void;
   /** Update metadata for a workflow instance (used by isolated workflows) */
@@ -728,19 +764,21 @@ class WorkflowsImpl implements Workflows {
 
   setCore(core: CoreServices): void {
     this.core = core;
-    // Extract DB path if using Kysely adapter (for isolated workflows)
-    if (!this.dbPath && (core.db as any)?.getExecutor) {
-      // Try to get the database path from the Kysely instance
-      // This is a bit hacky but necessary for isolated workflows
-      try {
-        const executor = (core.db as any).getExecutor();
-        const adapter = executor?.adapter;
-        if (adapter?.db?.filename) {
-          this.dbPath = adapter.db.filename;
-        }
-      } catch {
-        // Ignore - dbPath might be set manually
+  }
+
+  async resolveDbPath(): Promise<void> {
+    if (this.dbPath) return;
+    if (!this.core?.db) return;
+
+    // Use PRAGMA database_list to get the file path — works with any SQLite dialect
+    try {
+      const result = await sql<{ name: string; file: string }>`PRAGMA database_list`.execute(this.core.db);
+      const main = result.rows.find((r) => r.name === "main");
+      if (main?.file && main.file !== "" && main.file !== ":memory:") {
+        this.dbPath = main.file;
       }
+    } catch {
+      // Not a SQLite database or PRAGMA not supported — dbPath stays unset
     }
   }
 
@@ -761,14 +799,15 @@ class WorkflowsImpl implements Workflows {
       throw new Error(`Workflow "${definition.name}" is already registered`);
     }
 
-    // Store module path for isolated workflows
-    if (options?.modulePath) {
-      this.workflowModulePaths.set(definition.name, options.modulePath);
+    // Resolve module path: explicit option > auto-detected sourceModule
+    const modulePath = options?.modulePath ?? definition.sourceModule;
+    if (modulePath) {
+      this.workflowModulePaths.set(definition.name, modulePath);
     } else if (definition.isolated !== false) {
-      // Warn if isolated workflow has no module path
+      // Warn only if neither explicit nor auto-detected path is available
       console.warn(
-        `[Workflows] Workflow "${definition.name}" is isolated but no modulePath provided. ` +
-        `Use: workflows.register(myWorkflow, { modulePath: import.meta.url })`
+        `[Workflows] Workflow "${definition.name}" is isolated but no modulePath could be detected. ` +
+        `Pass { modulePath: import.meta.url } to register().`
       );
     }
 
@@ -823,6 +862,11 @@ class WorkflowsImpl implements Workflows {
         console.warn(
           `[Workflows] Workflow "${workflowName}" falling back to inline execution (no modulePath)`
         );
+      } else if (isIsolated && modulePath && !this.dbPath) {
+        console.warn(
+          `[Workflows] Workflow "${workflowName}" falling back to inline execution (dbPath could not be auto-detected). ` +
+            `Set workflows.dbPath in your server config to enable isolated execution.`
+        );
       }
       this.startInlineWorkflow(instance.id, definition);
     }

package/src/core.ts

@@ -16,6 +16,7 @@ import type { Processes } from "./core/processes";
 import type { Audit } from "./core/audit";
 import type { WebSocketService } from "./core/websocket";
 import type { Storage } from "./core/storage";
+import type { Logs } from "./core/logs";
 
 export interface PluginRegistry {}
 
@@ -109,6 +110,7 @@ export interface CoreServices {
   audit: Audit;
   websocket: WebSocketService;
   storage: Storage;
+  logs: Logs;
 }
 
 /**

package/src/harness.ts

@@ -16,9 +16,11 @@ import {
   createAudit,
   createWebSocket,
   createStorage,
+  createLogs,
   KyselyJobAdapter,
   KyselyWorkflowAdapter,
   MemoryAuditAdapter,
+  MemoryLogsAdapter,
 } from "./core/index";
 import { AppServer, type ServerConfig } from "./server";
 import type { IRouter, RouteDefinition } from "./router";
@@ -68,6 +70,7 @@ export async function createTestHarness(targetPlugin: Plugin, dependencies: Plug
   const audit = createAudit({ adapter: new MemoryAuditAdapter() });
   const websocket = createWebSocket();
   const storage = createStorage(); // Uses memory adapter by default
+  const logs = createLogs({ adapter: new MemoryLogsAdapter(), events });
 
   const core: CoreServices = {
     db,
@@ -85,6 +88,7 @@ export async function createTestHarness(targetPlugin: Plugin, dependencies: Plug
     audit,
     websocket,
     storage,
+    logs,
   };
 
   const manager = new PluginManager(core);