@cogitator-ai/workflows 0.8.0 → 0.10.0

package/dist/manager/workflow-manager.js

@@ -15,6 +15,15 @@ import { WorkflowScheduler } from '../scheduler.js';
 import { createJobScheduler } from './scheduler.js';
 import { InMemoryRunStore } from './run-store.js';
 import { createTracer } from '../observability/tracer.js';
+import { createMetricsCollector } from '../observability/metrics.js';
+/** A finite number stored under `key` in the run's metadata (scheduling options live there) */
+function metadataNumber(run, key) {
+    const value = run.metadata?.[key];
+    return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
+}
+function errorRecord(error) {
+    return { name: error.name, message: error.message, stack: error.stack };
+}
 /**
  * Workflow manager implementation
  */
@@ -26,11 +35,14 @@ export class DefaultWorkflowManager {
     executor;
     workflows = new Map();
     activeRuns = new Map();
+    /** Execute options of runs that are active or paused, reused when a paused run resumes */
+    runOptions = new Map();
     stateChangeCallbacks = new Set();
     runLocks = new Map();
     defaultTimeout;
     tracer;
     metrics;
+    disabledMetrics;
     constructor(config) {
         this.cogitator = config.cogitator;
         this.defaultTimeout = config.defaultTimeout;
@@ -42,14 +54,18 @@ export class DefaultWorkflowManager {
         this.scheduler = createJobScheduler({
             runStore: this.runStore,
             maxConcurrency: config.maxConcurrency,
-            onRunReady: (runId) => this.handleRunReady(runId),
+            onRunReady: (runId) => {
+                this.handleRunReady(runId).catch((error) => {
+                    console.warn(`[WorkflowManager] Scheduled run '${runId}' could not be started:`, error);
+                });
+            },
         });
         if (config.onRunStateChange) {
             this.stateChangeCallbacks.add(config.onRunStateChange);
         }
     }
     /**
-     * Start the manager (begins processing scheduled runs)
+     * Start the manager (begins processing scheduled runs and cron jobs)
      */
     start() {
         this.scheduler.start();
@@ -61,18 +77,46 @@ export class DefaultWorkflowManager {
         this.scheduler.stop();
     }
     /**
-     * Register a workflow for scheduling
+     * Register a workflow for scheduling, retries and resuming
      */
     registerWorkflow(workflow) {
         this.workflows.set(workflow.name, workflow);
     }
     /**
-     * Schedule a workflow for later execution
+     * Schedule a workflow for later execution. `cron` queues one run at the next
+     * occurrence; use `registerCronJob` for a recurring schedule.
      */
     async schedule(workflow, options) {
         this.registerWorkflow(workflow);
         return this.scheduler.scheduleRun(workflow, options);
     }
+    /**
+     * Run `workflow` on every occurrence of the cron `expression` while the manager is
+     * started. Each occurrence queues a run with `jobOptions` and `triggerId: 'cron:<jobId>'`.
+     * Returns the job id.
+     */
+    registerCronJob(workflow, expression, options) {
+        this.registerWorkflow(workflow);
+        return this.scheduler.registerCronJob(workflow, expression, options);
+    }
+    /**
+     * Remove a cron job; runs it already queued are kept
+     */
+    unregisterCronJob(jobId) {
+        return this.scheduler.unregisterCronJob(jobId);
+    }
+    /**
+     * Pause or continue a cron job
+     */
+    setCronJobEnabled(jobId, enabled) {
+        return this.scheduler.setCronJobEnabled(jobId, enabled);
+    }
+    /**
+     * List the registered cron jobs
+     */
+    getCronJobs() {
+        return this.scheduler.getCronJobs();
+    }
     /**
      * Execute a workflow immediately
      */
@@ -99,104 +143,12 @@ export class DefaultWorkflowManager {
         };
         await this.runStore.save(run);
         this.notifyStateChange(run);
-        const abortController = new AbortController();
-        this.activeRuns.set(runId, { abort: () => abortController.abort() });
         this.scheduler.runStarted(runId);
-        let timedOut = false;
-        const timeoutHandle = this.defaultTimeout !== undefined && this.defaultTimeout > 0
-            ? setTimeout(() => {
-                timedOut = true;
-                abortController.abort();
-            }, this.defaultTimeout)
-            : undefined;
-        const runTracer = options?.tracing ? createTracer(options.tracing) : this.tracer;
-        const runMetrics = options?.metrics?.enabled === false ? undefined : this.metrics;
-        try {
-            const executed = await this.executor.execute(workflow, input, {
-                checkpoint: !!this.checkpointStore,
-                ...options,
-                signal: abortController.signal,
-                tracer: runTracer,
-                metricsCollector: runMetrics,
-                onNodeStart: (node) => {
-                    void this.updateRunNodes(runId, node, 'start');
-                    options?.onNodeStart?.(node);
-                },
-                onNodeComplete: (node, result, duration) => {
-                    void this.updateRunNodes(runId, node, 'complete');
-                    options?.onNodeComplete?.(node, result, duration);
-                },
-                onNodeError: (node, error) => {
-                    void this.updateRunNodes(runId, node, 'error');
-                    options?.onNodeError?.(node, error);
-                },
-            });
-            const result = timedOut
-                ? {
-                    ...executed,
-                    error: new Error(`Workflow run '${runId}' timed out after ${String(this.defaultTimeout)}ms`),
-                }
-                : executed;
-            if (result.error) {
-                await this.runStore.update(runId, {
-                    status: 'failed',
-                    state: result.state,
-                    completedAt: Date.now(),
-                    checkpointId: result.checkpointId,
-                    error: {
-                        name: result.error.name,
-                        message: result.error.message,
-                        stack: result.error.stack,
-                    },
-                });
-                const updatedRun = await this.runStore.get(runId);
-                if (updatedRun)
-                    this.notifyStateChange(updatedRun);
-                return result;
-            }
-            await this.runStore.update(runId, {
-                status: 'completed',
-                state: result.state,
-                output: result.state,
-                completedAt: Date.now(),
-                checkpointId: result.checkpointId,
-            });
-            const updatedRun = await this.runStore.get(runId);
-            if (updatedRun)
-                this.notifyStateChange(updatedRun);
-            return result;
-        }
-        catch (error) {
-            const err = error instanceof Error ? error : new Error(String(error));
-            await this.runStore.update(runId, {
-                status: 'failed',
-                completedAt: Date.now(),
-                error: {
-                    name: err.name,
-                    message: err.message,
-                    stack: err.stack,
-                },
-            });
-            const updatedRun = await this.runStore.get(runId);
-            if (updatedRun)
-                this.notifyStateChange(updatedRun);
-            throw error;
-        }
-        finally {
-            if (timeoutHandle)
-                clearTimeout(timeoutHandle);
-            if (runTracer && runTracer !== this.tracer) {
-                await runTracer.flush().catch((error) => {
-                    console.warn('[WorkflowManager] Failed to flush run traces:', error);
-                });
-            }
-            this.activeRuns.delete(runId);
-            this.runLocks.delete(runId);
-            this.scheduler.runCompleted(runId);
-        }
+        return this.runWorkflow({ runId, workflow, input, options, timeout: this.defaultTimeout });
     }
     /**
-     * Cancel a run
+     * Cancel a run. A running run is aborted: the nodes in flight get the abort signal and
+     * no further node starts.
      */
     async cancel(runId, reason) {
         const run = await this.runStore.get(runId);
@@ -210,18 +162,13 @@ export class DefaultWorkflowManager {
             return;
         }
         if (run.status === 'running' || run.status === 'paused') {
-            const active = this.activeRuns.get(runId);
-            if (active) {
-                active.abort();
-            }
-            await this.runStore.update(runId, {
+            await this.stopRun(runId, 'cancelled', {
                 status: 'cancelled',
                 completedAt: Date.now(),
                 error: reason ? { name: 'CancelError', message: reason } : undefined,
             });
-            const updatedRun = await this.runStore.get(runId);
-            if (updatedRun)
-                this.notifyStateChange(updatedRun);
+            if (!this.activeRuns.has(runId))
+                this.runOptions.delete(runId);
         }
     }
     /**
@@ -243,7 +190,9 @@ export class DefaultWorkflowManager {
         return this.runStore.getStats(workflowName);
     }
     /**
-     * Pause a running workflow (aborts execution; resume requires a checkpoint)
+     * Pause a running workflow. Execution is aborted (nodes in flight get the abort signal)
+     * and `resume()` continues from the run's last checkpoint, so the manager needs a
+     * `checkpointStore`.
      */
     async pause(runId) {
         const run = await this.runStore.get(runId);
@@ -252,35 +201,62 @@ export class DefaultWorkflowManager {
         if (run.status !== 'running') {
             throw new Error(`Cannot pause run in status: ${run.status}`);
         }
-        const active = this.activeRuns.get(runId);
-        if (active) {
-            active.abort();
+        if (!this.checkpointStore) {
+            throw new Error(`Cannot pause run '${runId}': the manager has no checkpointStore to resume it from (use cancel() to stop it)`);
         }
-        await this.runStore.update(runId, {
-            status: 'paused',
-            pausedAt: Date.now(),
-        });
-        const updatedRun = await this.runStore.get(runId);
-        if (updatedRun)
-            this.notifyStateChange(updatedRun);
+        await this.stopRun(runId, 'paused', { status: 'paused', pausedAt: Date.now() });
     }
     /**
-     * Resume a paused workflow
+     * Resume a paused workflow from its last checkpoint (from the start when it was paused
+     * before the first checkpoint). Nodes that completed before the checkpoint are not run
+     * again. Resolves once the run is running again; follow it with `onRunStateChange` or
+     * `getStatus`. `options` default to the ones the run was started with in this process.
      */
-    async resume(runId) {
+    async resume(runId, options) {
         const run = await this.runStore.get(runId);
         if (!run)
             throw new Error(`Run not found: ${runId}`);
         if (run.status !== 'paused') {
             throw new Error(`Cannot resume run in status: ${run.status}`);
         }
+        if (this.activeRuns.has(runId)) {
+            throw new Error(`Cannot resume run '${runId}' yet: the nodes it was running when paused have not finished`);
+        }
+        const workflow = this.workflows.get(run.workflowName);
+        if (!workflow) {
+            throw new Error(`Workflow not found: ${run.workflowName}`);
+        }
+        const checkpoint = run.checkpointId
+            ? await this.checkpointStore?.load(run.checkpointId)
+            : undefined;
+        if (run.checkpointId && !checkpoint) {
+            throw new Error(`Checkpoint '${run.checkpointId}' of run '${runId}' not found`);
+        }
         await this.runStore.update(runId, {
             status: 'running',
             pausedAt: undefined,
+            currentNodes: [],
         });
         const updatedRun = await this.runStore.get(runId);
         if (updatedRun)
             this.notifyStateChange(updatedRun);
+        this.scheduler.runStarted(runId);
+        void this.runWorkflow({
+            runId,
+            workflow,
+            options: options ?? this.runOptions.get(runId),
+            timeout: this.runTimeout(run),
+            ...(checkpoint
+                ? {
+                    input: checkpoint.state,
+                    resumeFrom: {
+                        workflowId: checkpoint.workflowId,
+                        skipNodes: new Set(checkpoint.completedNodes),
+                        nodeResults: checkpoint.nodeResults,
+                    },
+                }
+                : { input: run.input }),
+        }).catch(() => { });
     }
     /**
      * Retry a failed run
@@ -315,6 +291,7 @@ export class DefaultWorkflowManager {
                 ...run.metadata,
                 retriedFrom: runId,
                 retriedAt: now,
+                retryAttempt: (metadataNumber(run, 'retryAttempt') ?? 0) + 1,
             },
         };
         await this.runStore.save(newRun);
@@ -365,51 +342,21 @@ export class DefaultWorkflowManager {
                 replayedAt: now,
             },
         };
+        this.registerWorkflow(workflow);
         await this.runStore.save(newRun);
         this.notifyStateChange(newRun);
-        const skipNodes = new Set(newRun.completedNodes);
-        const abortController = new AbortController();
-        this.activeRuns.set(newRunId, { abort: () => abortController.abort() });
         this.scheduler.runStarted(newRunId);
-        try {
-            const result = await this.executor.execute(workflow, run.state, {
-                checkpoint: !!this.checkpointStore,
-                skipNodes,
+        return this.runWorkflow({
+            runId: newRunId,
+            workflow,
+            input: run.state,
+            timeout: this.defaultTimeout,
+            resumeFrom: {
+                workflowId: newRunId,
+                skipNodes: new Set(kept),
                 nodeResults: keptResults,
-                signal: abortController.signal,
-            });
-            await this.runStore.update(newRunId, {
-                status: result.error ? 'failed' : 'completed',
-                state: result.state,
-                output: result.error ? undefined : result.state,
-                completedAt: Date.now(),
-                checkpointId: result.checkpointId,
-                error: result.error
-                    ? { name: result.error.name, message: result.error.message, stack: result.error.stack }
-                    : undefined,
-            });
-            const updatedRun = await this.runStore.get(newRunId);
-            if (updatedRun)
-                this.notifyStateChange(updatedRun);
-            return result;
-        }
-        catch (error) {
-            const err = error instanceof Error ? error : new Error(String(error));
-            await this.runStore.update(newRunId, {
-                status: 'failed',
-                completedAt: Date.now(),
-                error: { name: err.name, message: err.message, stack: err.stack },
-            });
-            const updatedRun = await this.runStore.get(newRunId);
-            if (updatedRun)
-                this.notifyStateChange(updatedRun);
-            throw error;
-        }
-        finally {
-            this.activeRuns.delete(newRunId);
-            this.runLocks.delete(newRunId);
-            this.scheduler.runCompleted(newRunId);
-        }
+            },
+        });
     }
     /**
      * Get count of active runs
@@ -440,15 +387,22 @@ export class DefaultWorkflowManager {
     dispose() {
         this.stop();
         this.activeRuns.clear();
+        this.runOptions.clear();
         this.runLocks.clear();
         this.workflows.clear();
         this.stateChangeCallbacks.clear();
         this.scheduler.dispose();
     }
+    /** Scheduling `timeout` stored on the run, else the manager's `defaultTimeout` */
+    runTimeout(run) {
+        return metadataNumber(run, 'timeout') ?? this.defaultTimeout;
+    }
     async handleRunReady(runId) {
         const run = await this.runStore.get(runId);
-        if (!run)
+        if (!run || (run.status !== 'pending' && run.status !== 'scheduled')) {
+            this.scheduler.runCompleted(runId);
             return;
+        }
         const workflow = this.workflows.get(run.workflowName);
         if (!workflow) {
             await this.runStore.update(runId, {
@@ -460,49 +414,170 @@ export class DefaultWorkflowManager {
                 },
             });
             this.scheduler.runCompleted(runId);
+            const failedRun = await this.runStore.get(runId);
+            if (failedRun)
+                this.notifyStateChange(failedRun);
             return;
         }
         await this.runStore.update(runId, {
             status: 'running',
             startedAt: Date.now(),
         });
-        const abortController = new AbortController();
-        this.activeRuns.set(runId, { abort: () => abortController.abort() });
+        const startedRun = await this.runStore.get(runId);
+        if (startedRun)
+            this.notifyStateChange(startedRun);
         try {
-            const result = await this.executor.execute(workflow, run.input, {
-                signal: abortController.signal,
+            await this.runWorkflow({
+                runId,
+                workflow,
+                input: run.input,
+                timeout: this.runTimeout(run),
             });
-            await this.runStore.update(runId, {
-                status: result.error ? 'failed' : 'completed',
-                state: result.state,
-                output: result.error ? undefined : result.state,
-                completedAt: Date.now(),
-                checkpointId: result.checkpointId,
-                error: result.error
-                    ? { name: result.error.name, message: result.error.message, stack: result.error.stack }
-                    : undefined,
+        }
+        catch { }
+        const finished = await this.runStore.get(runId);
+        const maxRetries = finished ? metadataNumber(finished, 'maxRetries') : undefined;
+        if (finished?.status === 'failed' &&
+            maxRetries !== undefined &&
+            (metadataNumber(finished, 'retryAttempt') ?? 0) < maxRetries) {
+            await this.retry(runId);
+        }
+    }
+    /**
+     * Abort an active run for `reason` and record `update`. The run's own final write waits
+     * for this one, so the stop status is never overwritten by a stale 'running' outcome.
+     */
+    async stopRun(runId, reason, update) {
+        const write = this.runStore.update(runId, update);
+        const active = this.activeRuns.get(runId);
+        if (active) {
+            active.stopReason = reason;
+            active.stopWrite = write.catch(() => { });
+            active.controller.abort();
+        }
+        await write;
+        const updatedRun = await this.runStore.get(runId);
+        if (updatedRun)
+            this.notifyStateChange(updatedRun);
+    }
+    /**
+     * Execute a run that is already stored as running and counted by the scheduler, then
+     * record how it ended: completed, failed, or paused/cancelled when it was stopped.
+     */
+    async runWorkflow(launch) {
+        const { runId, workflow, input, options, timeout, resumeFrom } = launch;
+        const active = { controller: new AbortController() };
+        this.activeRuns.set(runId, active);
+        if (options)
+            this.runOptions.set(runId, options);
+        let timedOut = false;
+        const timeoutHandle = timeout !== undefined && timeout > 0
+            ? setTimeout(() => {
+                timedOut = true;
+                active.controller.abort();
+            }, timeout)
+            : undefined;
+        const runTracer = options?.tracing ? createTracer(options.tracing) : this.tracer;
+        const runMetrics = options?.metrics?.enabled === false
+            ? (this.disabledMetrics ??= createMetricsCollector({ enabled: false }))
+            : this.metrics;
+        try {
+            const executed = await this.executor.execute(workflow, input, {
+                checkpoint: !!this.checkpointStore,
+                workflowId: runId,
+                ...options,
+                ...resumeFrom,
+                signal: active.controller.signal,
+                tracer: runTracer,
+                metricsCollector: runMetrics,
+                ...this.trackNodes(runId, options),
             });
+            await this.settleNodeUpdates(runId);
+            await active.stopWrite;
+            const stopped = executed.error ? active.stopReason : undefined;
+            const error = stopped
+                ? new Error(`Workflow run '${runId}' was ${stopped}`)
+                : timedOut
+                    ? new Error(`Workflow run '${runId}' timed out after ${String(timeout)}ms`)
+                    : executed.error;
+            const result = error === executed.error ? executed : { ...executed, error };
+            const checkpoint = result.checkpointId ? { checkpointId: result.checkpointId } : {};
+            if (stopped) {
+                await this.finishRun(runId, { status: stopped, state: result.state, ...checkpoint });
+            }
+            else if (result.error) {
+                await this.finishRun(runId, {
+                    status: 'failed',
+                    state: result.state,
+                    completedAt: Date.now(),
+                    ...checkpoint,
+                    error: errorRecord(result.error),
+                });
+            }
+            else {
+                await this.finishRun(runId, {
+                    status: 'completed',
+                    state: result.state,
+                    output: result.state,
+                    completedAt: Date.now(),
+                    pausedAt: undefined,
+                    error: undefined,
+                    ...checkpoint,
+                });
+            }
+            return result;
         }
         catch (error) {
             const err = error instanceof Error ? error : new Error(String(error));
-            await this.runStore.update(runId, {
-                status: 'failed',
-                completedAt: Date.now(),
-                error: {
-                    name: err.name,
-                    message: err.message,
-                    stack: err.stack,
-                },
-            });
+            await this.settleNodeUpdates(runId);
+            await active.stopWrite;
+            await this.finishRun(runId, active.stopReason
+                ? { status: active.stopReason }
+                : { status: 'failed', completedAt: Date.now(), error: errorRecord(err) });
+            throw error;
         }
         finally {
+            if (timeoutHandle)
+                clearTimeout(timeoutHandle);
+            if (runTracer && runTracer !== this.tracer) {
+                await runTracer.flush().catch((error) => {
+                    console.warn('[WorkflowManager] Failed to flush run traces:', error);
+                });
+            }
             this.activeRuns.delete(runId);
+            this.runLocks.delete(runId);
+            if (active.stopReason !== 'paused')
+                this.runOptions.delete(runId);
             this.scheduler.runCompleted(runId);
-            const updatedRun = await this.runStore.get(runId);
-            if (updatedRun)
-                this.notifyStateChange(updatedRun);
         }
     }
+    async finishRun(runId, update) {
+        await this.runStore.update(runId, update);
+        const updatedRun = await this.runStore.get(runId);
+        if (updatedRun)
+            this.notifyStateChange(updatedRun);
+    }
+    /** Node callbacks that record the run's current, completed and failed nodes, then call `options`' own. */
+    trackNodes(runId, options) {
+        return {
+            onNodeStart: (node) => {
+                void this.updateRunNodes(runId, node, 'start');
+                options?.onNodeStart?.(node);
+            },
+            onNodeComplete: (node, result, duration) => {
+                void this.updateRunNodes(runId, node, 'complete');
+                options?.onNodeComplete?.(node, result, duration);
+            },
+            onNodeError: (node, error) => {
+                void this.updateRunNodes(runId, node, 'error');
+                options?.onNodeError?.(node, error);
+            },
+        };
+    }
+    /** Waits for the node updates still being written, so the final run record has them all. */
+    async settleNodeUpdates(runId) {
+        await this.runLocks.get(runId);
+    }
     async updateRunNodes(runId, nodeId, action) {
         const prev = this.runLocks.get(runId) ?? Promise.resolve();
         const next = prev.then(async () => {
@@ -510,17 +585,22 @@ export class DefaultWorkflowManager {
             if (!run)
                 return;
             const updates = {};
+            const others = run.currentNodes.filter((n) => n !== nodeId);
             switch (action) {
                 case 'start':
-                    updates.currentNodes = [...run.currentNodes, nodeId];
+                    updates.currentNodes = [...others, nodeId];
                     break;
                 case 'complete':
-                    updates.currentNodes = run.currentNodes.filter((n) => n !== nodeId);
-                    updates.completedNodes = [...run.completedNodes, nodeId];
+                    updates.currentNodes = others;
+                    updates.completedNodes = run.completedNodes.includes(nodeId)
+                        ? run.completedNodes
+                        : [...run.completedNodes, nodeId];
                     break;
                 case 'error':
-                    updates.currentNodes = run.currentNodes.filter((n) => n !== nodeId);
-                    updates.failedNodes = [...run.failedNodes, nodeId];
+                    updates.currentNodes = others;
+                    updates.failedNodes = run.failedNodes.includes(nodeId)
+                        ? run.failedNodes
+                        : [...run.failedNodes, nodeId];
                     break;
             }
             await this.runStore.update(runId, updates);