@cogitator-ai/worker 0.9.24 → 0.10.0

package/dist/queue.js

@@ -10,6 +10,55 @@
 import { Queue } from 'bullmq';
 import { nanoid } from 'nanoid';
 import { DEFAULT_QUEUE_NAME, createBullConnection, queuePrefix, } from './connection.js';
+/**
+ * Names BullMQ keeps next to the jobs of a queue, under the same prefix: a job under one of them
+ * would overwrite a key of the queue itself
+ */
+const QUEUE_KEYS = new Set([
+    'active',
+    'wait',
+    'waiting-children',
+    'paused',
+    'id',
+    'delayed',
+    'prioritized',
+    'stalled-check',
+    'completed',
+    'failed',
+    'stalled',
+    'repeat',
+    'limiter',
+    'meta',
+    'events',
+    'pc',
+    'marker',
+    'de',
+]);
+/**
+ * The id a producer gave a job, once checked: BullMQ builds Redis keys from it, so it must be a
+ * non-empty string without ':', not an integer and not a name of the queue's own keys
+ */
+function checkJobId(jobId) {
+    if (jobId === '' || jobId.includes(':')) {
+        throw new Error(`A job id must be a non-empty string without ':', got '${jobId}'`);
+    }
+    if (`${Number.parseInt(jobId, 10)}` === jobId) {
+        throw new Error(`A job id must not be an integer, got '${jobId}'`);
+    }
+    if (QUEUE_KEYS.has(jobId)) {
+        throw new Error(`A job id must not be '${jobId}', a key the queue keeps for itself`);
+    }
+    return jobId;
+}
+const isResume = (payload) => payload.type === 'agent' && payload.resume !== undefined;
+const describe = (payload) => isResume(payload)
+    ? 'a resumed agent job'
+    : `${payload.type === 'agent' ? 'an' : 'a'} ${payload.type} job`;
+/**
+ * A queue of agent, workflow and swarm jobs. Each `add*Job` takes a `jobId`: the job is added
+ * under it, and adding a job whose id the queue holds already gives the job it holds back instead
+ * of adding a second one, so producers can retry an add without running the work twice.
+ */
 export class JobQueue {
     queue;
     connection;
@@ -33,7 +82,7 @@ export class JobQueue {
      * Add an agent execution job to the queue
      */
     async addAgentJob(agentConfig, input, options) {
-        const jobId = nanoid();
+        const jobId = options?.jobId === undefined ? nanoid() : checkJobId(options.jobId);
         const threadId = options?.threadId ?? nanoid();
         const payload = {
             type: 'agent',
@@ -44,11 +93,7 @@ export class JobQueue {
             ...(options?.userId !== undefined && { userId: options.userId }),
             metadata: options?.metadata,
         };
-        return this.queue.add('agent', payload, {
-            jobId,
-            priority: options?.priority,
-            delay: options?.delay,
-        });
+        return this.add(payload, options);
     }
     /**
      * Continue an agent job whose run paused for tool approvals (its result has
@@ -77,7 +122,7 @@ export class JobQueue {
         if (!threadId) {
             throw new Error('Cannot resume an agent job without the checkpoint or thread of its run');
         }
-        const jobId = nanoid();
+        const jobId = options?.jobId === undefined ? nanoid() : checkJobId(options.jobId);
         const payload = {
             type: 'agent',
             jobId,
@@ -92,17 +137,13 @@ export class JobQueue {
             },
             metadata: options?.metadata,
         };
-        return this.queue.add('agent', payload, {
-            jobId,
-            priority: options?.priority,
-            delay: options?.delay,
-        });
+        return this.add(payload, options);
     }
     /**
      * Add a workflow execution job to the queue
      */
     async addWorkflowJob(workflowConfig, input, options) {
-        const jobId = nanoid();
+        const jobId = options?.jobId === undefined ? nanoid() : checkJobId(options.jobId);
         const runId = options?.runId ?? nanoid();
         const payload = {
             type: 'workflow',
@@ -112,17 +153,13 @@ export class JobQueue {
             runId,
             metadata: options?.metadata,
         };
-        return this.queue.add('workflow', payload, {
-            jobId,
-            priority: options?.priority,
-            delay: options?.delay,
-        });
+        return this.add(payload, options);
     }
     /**
      * Add a swarm execution job to the queue
      */
     async addSwarmJob(swarmConfig, input, options) {
-        const jobId = nanoid();
+        const jobId = options?.jobId === undefined ? nanoid() : checkJobId(options.jobId);
         const payload = {
             type: 'swarm',
             jobId,
@@ -130,11 +167,7 @@ export class JobQueue {
             input,
             metadata: options?.metadata,
         };
-        return this.queue.add('swarm', payload, {
-            jobId,
-            priority: options?.priority,
-            delay: options?.delay,
-        });
+        return this.add(payload, options);
     }
     /**
      * Queue one agent turn of a swarm as a BullMQ job. A `WorkerPool` runs it and publishes the
@@ -143,7 +176,7 @@ export class JobQueue {
      * to a Redis list that `DistributedSwarmWorker` consumes, so run that worker for them.
      */
     async addSwarmAgentJob(swarmId, agentName, agentConfig, input, options) {
-        const jobId = nanoid();
+        const jobId = options?.jobId === undefined ? nanoid() : checkJobId(options.jobId);
         const keyPrefix = options?.keyPrefix ?? 'swarm';
         const payload = {
             type: 'swarm-agent',
@@ -162,11 +195,32 @@ export class JobQueue {
                 results: `${keyPrefix}:${swarmId}:results`,
             },
         };
-        return this.queue.add('swarm-agent', payload, {
-            jobId,
+        return this.add(payload, options);
+    }
+    /**
+     * Add the job, and when the producer gave its id, hand back the job the queue holds under it:
+     * adding a job whose id the queue holds already adds nothing and gives that job back, so a
+     * producer that retries an add, such as after a timeout, does not run the work twice, even when
+     * the two adds race. A job the queue removed (after it completed or failed, past
+     * `removeOnComplete` or `removeOnFail`) is gone, and its id adds a new job.
+     *
+     * @throws Error when the queue holds the id as another kind of job
+     */
+    async add(payload, options) {
+        const added = (await this.queue.add(payload.type, payload, {
+            jobId: payload.jobId,
             priority: options?.priority,
             delay: options?.delay,
-        });
+        }));
+        if (options?.jobId === undefined)
+            return added;
+        const held = (await this.queue.getJob(payload.jobId));
+        if (!held)
+            return added;
+        if (held.data.type !== payload.type || isResume(held.data) !== isResume(payload)) {
+            throw new Error(`The queue holds job '${payload.jobId}' as ${describe(held.data)}, not as ${describe(payload)}`);
+        }
+        return held;
     }
     /**
      * Get job by ID

package/dist/queue.js.map

@@ -1 +1 @@
-{"version":3,"file":"queue.js","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,KAAK,EAAY,MAAM,QAAQ,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAgBhC,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,WAAW,GAEZ,MAAM,iBAAiB,CAAC;AAEzB,MAAM,OAAO,QAAQ;IACX,KAAK,CAAoB;IACzB,UAAU,CAAiB;IAEnC,YAAY,MAAmB;QAC7B,IAAI,CAAC,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,IAAI,kBAAkB,EAAE;YACxD,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,UAAU;YACtC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC;YACjC,iBAAiB,EAAE;gBACjB,QAAQ,EAAE,MAAM,CAAC,iBAAiB,EAAE,QAAQ,IAAI,CAAC;gBACjD,OAAO,EAAE,MAAM,CAAC,iBAAiB,EAAE,OAAO,IAAI;oBAC5C,IAAI,EAAE,aAAa;oBACnB,KAAK,EAAE,IAAI;iBACZ;gBACD,gBAAgB,EAAE,MAAM,CAAC,iBAAiB,EAAE,gBAAgB,IAAI,GAAG;gBACnE,YAAY,EAAE,MAAM,CAAC,iBAAiB,EAAE,YAAY,IAAI,GAAG;aAC5D;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CACf,WAA4B,EAC5B,KAAa,EACb,OAMC;QAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QACvB,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,MAAM,EAAE,CAAC;QAE/C,MAAM,OAAO,GAAoB;YAC/B,IAAI,EAAE,OAAO;YACb,KAAK;YACL,WAAW;YACX,KAAK;YACL,QAAQ;YACR,GAAG,CAAC,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAChE,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE;YACtC,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,KAAK,EAAE,OAAO,EAAE,KAAK;SACtB,CAAkC,CAAC;IACtC,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,cAAc,CAClB,WAA4B,EAC5B,MAAkE,EAClE,OAUC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2CAA2C,MAAM,CAAC,MAAM,cAAc,CAAC,CAAC;QAC1F,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,EAAE,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;QAChE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC5F,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QAEvB,MAAM,OAAO,GAAoB;YAC/B,IAAI,EAAE,OAAO;YACb,KAAK;YACL,WAAW;YACX,KAAK,EAAE,EAAE;YACT,QAAQ;YACR,GAAG,CAAC,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAChE,MAAM,EAAE;gBACN,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;gBAC3D,GAAG,CAAC,OAAO,EAAE,SAAS,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC3D,GAAG,CAAC,OAAO,EAAE,eAAe,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,eAAe,EAAE,CAAC;aAC9E;YACD,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE;YACtC,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,KAAK,EAAE,OAAO,EAAE,KAAK;SACtB,CAAkC,CAAC;IACtC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAClB,cAAkC,EAClC,KAA8B,EAC9B,OAKC;QAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,MAAM,EAAE,CAAC;QAEzC,MAAM,OAAO,GAAuB;YAClC,IAAI,EAAE,UAAU;YAChB,KAAK;YACL,cAAc;YACd,KAAK;YACL,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,OAAO,EAAE;YACzC,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,KAAK,EAAE,OAAO,EAAE,KAAK;SACtB,CAAqC,CAAC;IACzC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CACf,WAA4B,EAC5B,KAAa,EACb,OAIC;QAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QAEvB,MAAM,OAAO,GAAoB;YAC/B,IAAI,EAAE,OAAO;YACb,KAAK;YACL,WAAW;YACX,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE;YACtC,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,KAAK,EAAE,OAAO,EAAE,KAAK;SACtB,CAAkC,CAAC;IACtC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,gBAAgB,CACpB,OAAe,EACf,SAAiB,EACjB,WAA4B,EAC5B,KAAa,EACb,OAUC;QAED,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;QACvB,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC;QAEhD,MAAM,OAAO,GAAyB;YACpC,IAAI,EAAE,aAAa;YACnB,KAAK;YACL,OAAO;YACP,SAAS;YACT,WAAW;YACX,KAAK;YACL,OAAO,EAAE,OAAO,EAAE,OAAO;YACzB,UAAU,EACR,OAAO,EAAE,OAAO,KAAK,SAAS;gBAC5B,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE;gBACrD,CAAC,CAAC,OAAO,EAAE,UAAU;YACzB,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI;gBAC/B,UAAU,EAAE,GAAG,SAAS,IAAI,OAAO,aAAa;gBAChD,QAAQ,EAAE,GAAG,SAAS,IAAI,OAAO,WAAW;gBAC5C,OAAO,EAAE,GAAG,SAAS,IAAI,OAAO,UAAU;aAC3C;SACF,CAAC;QAEF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,EAAE,OAAO,EAAE;YAC5C,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,KAAK,EAAE,OAAO,EAAE,KAAK;SACtB,CAAuC,CAAC;IAC3C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa;QACxB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,KAAa;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAC;QAC3B,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,CAAC;YAC7F,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE;SAC7B,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;QAEpC,OAAO;YACL,OAAO;YACP,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC;YAC1B,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,CAAC;YAChC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC;YAC1B,OAAO;YACP,KAAK,EAAE,OAAO,GAAG,OAAO;YACxB,WAAW;SACZ,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CACT,KAAa,EACb,KAAa,EACb,IAA4D;QAE5D,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;IAClC,CAAC;CACF"}
+{"version":3,"file":"queue.js","sourceRoot":"","sources":["../src/queue.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,KAAK,EAAY,MAAM,QAAQ,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAgBhC,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,WAAW,GAEZ,MAAM,iBAAiB,CAAC;AAEzB;;;GAGG;AACH,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,QAAQ;IACR,MAAM;IACN,kBAAkB;IAClB,QAAQ;IACR,IAAI;IACJ,SAAS;IACT,aAAa;IACb,eAAe;IACf,WAAW;IACX,QAAQ;IACR,SAAS;IACT,QAAQ;IACR,SAAS;IACT,MAAM;IACN,QAAQ;IACR,IAAI;IACJ,QAAQ;IACR,IAAI;CACL,CAAC,CAAC;AAEH;;;GAGG;AACH,SAAS,UAAU,CAAC,KAAa;IAC/B,IAAI,KAAK,KAAK,EAAE,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,yDAAyD,KAAK,GAAG,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,KAAK,KAAK,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,yCAAyC,KAAK,GAAG,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,yBAAyB,KAAK,qCAAqC,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC;AACnG,MAAM,QAAQ,GAAG,CAAC,OAAmB,EAAE,EAAE,CACvC,QAAQ,CAAC,OAAO,CAAC;IACf,CAAC,CAAC,qBAAqB;IACvB,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,OAAO,CAAC,IAAI,MAAM,CAAC;AAErE;;;;GAIG;AACH,MAAM,OAAO,QAAQ;IACX,KAAK,CAAoB;IACzB,UAAU,CAAiB;IAEnC,YAAY,MAAmB;QAC7B,IAAI,CAAC,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrD,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,IAAI,kBAAkB,EAAE;YACxD,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,UAAU;YACtC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC;YACjC,iBAAiB,EAAE;gBACjB,QAAQ,EAAE,MAAM,CAAC,iBAAiB,EAAE,QAAQ,IAAI,CAAC;gBACjD,OAAO,EAAE,MAAM,CAAC,iBAAiB,EAAE,OAAO,IAAI;oBAC5C,IAAI,EAAE,aAAa;oBACnB,KAAK,EAAE,IAAI;iBACZ;gBACD,gBAAgB,EAAE,MAAM,CAAC,iBAAiB,EAAE,gBAAgB,IAAI,GAAG;gBACnE,YAAY,EAAE,MAAM,CAAC,iBAAiB,EAAE,YAAY,IAAI,GAAG;aAC5D;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CACf,WAA4B,EAC5B,KAAa,EACb,OAQC;QAED,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClF,MAAM,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,MAAM,EAAE,CAAC;QAE/C,MAAM,OAAO,GAAoB;YAC/B,IAAI,EAAE,OAAO;YACb,KAAK;YACL,WAAW;YACX,KAAK;YACL,QAAQ;YACR,GAAG,CAAC,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAChE,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;;;;;;;;;;;;OAkBG;IACH,KAAK,CAAC,cAAc,CAClB,WAA4B,EAC5B,MAAkE,EAClE,OAYC;QAED,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;YAC9D,MAAM,IAAI,KAAK,CAAC,2CAA2C,MAAM,CAAC,MAAM,cAAc,CAAC,CAAC;QAC1F,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,CAAC,UAAU,EAAE,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;QAChE,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,wEAAwE,CAAC,CAAC;QAC5F,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAElF,MAAM,OAAO,GAAoB;YAC/B,IAAI,EAAE,OAAO;YACb,KAAK;YACL,WAAW;YACX,KAAK,EAAE,EAAE;YACT,QAAQ;YACR,GAAG,CAAC,OAAO,EAAE,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAChE,MAAM,EAAE;gBACN,GAAG,CAAC,MAAM,CAAC,UAAU,IAAI,EAAE,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;gBAC3D,GAAG,CAAC,OAAO,EAAE,SAAS,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC3D,GAAG,CAAC,OAAO,EAAE,eAAe,IAAI,EAAE,eAAe,EAAE,OAAO,CAAC,eAAe,EAAE,CAAC;aAC9E;YACD,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAClB,cAAkC,EAClC,KAA8B,EAC9B,OAOC;QAED,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClF,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,MAAM,EAAE,CAAC;QAEzC,MAAM,OAAO,GAAuB;YAClC,IAAI,EAAE,UAAU;YAChB,KAAK;YACL,cAAc;YACd,KAAK;YACL,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CACf,WAA4B,EAC5B,KAAa,EACb,OAMC;QAED,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAElF,MAAM,OAAO,GAAoB;YAC/B,IAAI,EAAE,OAAO;YACb,KAAK;YACL,WAAW;YACX,KAAK;YACL,QAAQ,EAAE,OAAO,EAAE,QAAQ;SAC5B,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,gBAAgB,CACpB,OAAe,EACf,SAAiB,EACjB,WAA4B,EAC5B,KAAa,EACb,OAYC;QAED,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAClF,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,OAAO,CAAC;QAEhD,MAAM,OAAO,GAAyB;YACpC,IAAI,EAAE,aAAa;YACnB,KAAK;YACL,OAAO;YACP,SAAS;YACT,WAAW;YACX,KAAK;YACL,OAAO,EAAE,OAAO,EAAE,OAAO;YACzB,UAAU,EACR,OAAO,EAAE,OAAO,KAAK,SAAS;gBAC5B,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,UAAU,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE;gBACrD,CAAC,CAAC,OAAO,EAAE,UAAU;YACzB,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI;gBAC/B,UAAU,EAAE,GAAG,SAAS,IAAI,OAAO,aAAa;gBAChD,QAAQ,EAAE,GAAG,SAAS,IAAI,OAAO,WAAW;gBAC5C,OAAO,EAAE,GAAG,SAAS,IAAI,OAAO,UAAU;aAC3C;SACF,CAAC;QAEF,OAAO,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,GAAG,CACf,OAAU,EACV,OAA0E;QAE1E,MAAM,KAAK,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE;YACzD,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,KAAK,EAAE,OAAO,EAAE,KAAK;SACtB,CAAC,CAAW,CAAC;QACd,IAAI,OAAO,EAAE,KAAK,KAAK,SAAS;YAAE,OAAO,KAAK,CAAC;QAC/C,MAAM,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAuB,CAAC;QAC5E,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACxB,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YACjF,MAAM,IAAI,KAAK,CACb,wBAAwB,OAAO,CAAC,KAAK,QAAQ,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,YAAY,QAAQ,CAAC,OAAO,CAAC,EAAE,CAChG,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CAAC,KAAa;QACxB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,WAAW,CAAC,KAAa;QAC7B,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAC;QAC3B,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;IACxB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,UAAU;QACd,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC9C,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,SAAS,CAAC;YAC7F,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE;SAC7B,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;QAClE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;QAEpC,OAAO;YACL,OAAO;YACP,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC;YAC1B,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,CAAC;YAChC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC;YAC1B,OAAO;YACP,KAAK,EAAE,OAAO,GAAG,OAAO;YACxB,WAAW;SACZ,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK,CACT,KAAa,EACb,KAAa,EACb,IAA4D;QAE5D,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IAC9C,CAAC;IAED;;OAEG;IACH,QAAQ;QACN,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,CAAC;IAClC,CAAC;CACF"}

package/dist/types.d.ts

@@ -2,7 +2,7 @@
  * Worker types for distributed job processing
  */
 import type { Cogitator } from '@cogitator-ai/core';
-import type { AgentWireConfig, AgentWireResponseFormat, AgentWireRunResult, AgentWireUsage, RunCheckpoint, Tool, ToolApprovalDecision } from '@cogitator-ai/types';
+import type { AgentWireConfig, AgentWireResponseFormat, AgentWireRunResult, AgentWireUsage, CheckpointStore, RunCheckpoint, Tool, ToolApprovalDecision } from '@cogitator-ai/types';
 import type { SwarmAgentJobPayload as SwarmAgentJobContract, SwarmAgentJobResult as SwarmAgentJobResultContract } from '@cogitator-ai/swarms';
 /**
  * An agent's response format in a form that travels through the queue: a Zod schema becomes
@@ -161,16 +161,39 @@ export interface AgentJobResult extends AgentWireRunResult {
         total: number;
     };
 }
+/**
+ * What a job used and cost, summed over its agent runs. `inputTokens` and `outputTokens` are
+ * absent where the runs reported only their totals.
+ */
+export interface JobUsage {
+    inputTokens?: number;
+    outputTokens?: number;
+    totalTokens: number;
+    /** USD */
+    cost: number;
+    /** Agent runs counted */
+    runs: number;
+}
 export interface WorkflowJobResult {
     type: 'workflow';
     output: Record<string, unknown>;
     nodeResults: Record<string, unknown>;
+    /** What the agent nodes used and cost */
+    usage: JobUsage;
     duration: number;
 }
 export interface SwarmJobResult {
     type: 'swarm';
     output: string;
     rounds: number;
+    /** What the swarm's agent runs used and cost, in all and by agent */
+    usage: JobUsage & {
+        byAgent: Record<string, {
+            tokens: number;
+            cost: number;
+            runs: number;
+        }>;
+    };
     agentOutputs: {
         agent: string;
         output: string;
@@ -229,6 +252,13 @@ export interface JobExecutionOptions {
      * `WorkerPool.cancelJob()` and a shutdown that runs out of time stop the work
      */
     signal?: AbortSignal;
+    /**
+     * Whether the queue runs the job again after it failed with `error`: a workflow job keeps its
+     * checkpoint for that attempt, and lets it go when none follows. `WorkerPool` answers no for
+     * the last attempt, a cancelled job and a paused agent run, and yes for a job a shutdown
+     * stopped, which another worker takes up. Default: no attempt follows
+     */
+    willRetry?: (error: unknown) => boolean;
 }
 /**
  * Runtime dependencies used to execute jobs on a worker
@@ -241,6 +271,12 @@ export interface WorkerRuntime {
      * name; a job that needs a tool missing from this list fails.
      */
     tools?: Tool[];
+    /**
+     * Where workflow jobs keep their checkpoint, saved after every node, so a retried job goes on
+     * from it. `WorkerPool` keeps them in its Redis by default, a processor called on its own runs
+     * without them.
+     */
+    checkpoints?: CheckpointStore;
 }
 export interface WorkerConfig extends QueueConfig, WorkerRuntime {
     /** Number of worker instances */
@@ -251,6 +287,12 @@ export interface WorkerConfig extends QueueConfig, WorkerRuntime {
     lockDuration?: number;
     /** Stalled job check interval */
     stalledInterval?: number;
+    /**
+     * Seconds the default Redis store keeps a workflow checkpoint after its last save, for a run
+     * that is never resumed or let go, such as one whose worker crashed on its last attempt.
+     * Ignored with `checkpoints`. Default: 7 days
+     */
+    checkpointTtl?: number;
 }
 /**
  * State of a queued job, as `JobQueue.getJobState()` reports it:

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,KAAK,EACV,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,cAAc,EACd,aAAa,EACb,IAAI,EACJ,oBAAoB,EACrB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EACV,oBAAoB,IAAI,qBAAqB,EAC7C,mBAAmB,IAAI,2BAA2B,EACnD,MAAM,sBAAsB,CAAC;AAE9B;;;;GAIG;AACH,MAAM,MAAM,wBAAwB,GAAG,uBAAuB,CAAC;AAE/D;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG,eAAe,CAAC;AAE9C;;;;;;;;;GASG;AACH,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,sBAAsB,EAAE,CAAC;IAChC,KAAK,EAAE,sBAAsB,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,OAAO,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,CAAC;IACvD,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,eAAe,CAAC;IAC7B,uGAAuG;IACvG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,kBAAkB,GAC5B,WAAW,GAAG,WAAW,GAAG,MAAM,GAAG,YAAY,GAAG,gBAAgB,GAAG,UAAU,CAAC;AAEpF,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,kBAAkB,CAAC;IAC9B,mEAAmE;IACnE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,YAAY,GAAG,UAAU,GAAG,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC;AAE9F,MAAM,WAAW,mBAAmB;IAClC,yBAAyB;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;CAC1C;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,aAAa,GAAG,YAAY,GAAG,cAAc,GAAG,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAElG;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,aAAa,CAAC;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,2FAA2F;IAC3F,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;IACjD,4FAA4F;IAC5F,eAAe,CAAC,EAAE,oBAAoB,CAAC;CACxC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,eAAe,CAAC;IAC7B,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+EAA+E;IAC/E,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,kBAAkB,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,eAAe,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,qBAAqB,CAAC;AAEzD,MAAM,MAAM,UAAU,GACpB,eAAe,GAAG,kBAAkB,GAAG,eAAe,GAAG,oBAAoB,CAAC;AAEhF,wEAAwE;AACxE,MAAM,MAAM,aAAa,GAAG,cAAc,CAAC;AAE3C;;;;;;;GAOG;AACH,MAAM,WAAW,cAAe,SAAQ,kBAAkB;IACxD,IAAI,EAAE,OAAO,CAAC;IACd,2DAA2D;IAC3D,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,mBAAoB,SAAQ,2BAA2B;IACtE,IAAI,EAAE,aAAa,CAAC;CACrB;AAED,MAAM,MAAM,SAAS,GAAG,cAAc,GAAG,iBAAiB,GAAG,cAAc,GAAG,mBAAmB,CAAC;AAElG,MAAM,WAAW,WAAW;IAC1B,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,EAAE;QACL,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,qDAAqD;QACrD,GAAG,CAAC,EAAE,OAAO,CAAC;QACd,uBAAuB;QACvB,OAAO,CAAC,EAAE;YACR,KAAK,EAAE;gBAAE,IAAI,EAAE,MAAM,CAAC;gBAAC,IAAI,EAAE,MAAM,CAAA;aAAE,EAAE,CAAC;SACzC,CAAC;KACH,CAAC;IACF,0BAA0B;IAC1B,iBAAiB,CAAC,EAAE;QAClB,kCAAkC;QAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,uBAAuB;QACvB,OAAO,CAAC,EAAE;YACR,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC;YAC9B,KAAK,EAAE,MAAM,CAAC;SACf,CAAC;QACF,kCAAkC;QAClC,gBAAgB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;QACpC,+BAA+B;QAC/B,YAAY,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;KACjC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;OAIG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,4FAA4F;IAC5F,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB;;;OAGG;IACH,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;CAChB;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW,EAAE,aAAa;IAC9D,iCAAiC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iCAAiC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,QAAQ,GAChB,SAAS,GACT,aAAa,GACb,SAAS,GACT,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,kBAAkB,GAClB,SAAS,CAAC;AAEd,MAAM,WAAW,YAAY;IAC3B,4DAA4D;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,uBAAuB;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,4EAA4E;IAC5E,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,WAAW,EAAE,MAAM,CAAC;CACrB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,KAAK,EACV,eAAe,EACf,uBAAuB,EACvB,kBAAkB,EAClB,cAAc,EACd,eAAe,EACf,aAAa,EACb,IAAI,EACJ,oBAAoB,EACrB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EACV,oBAAoB,IAAI,qBAAqB,EAC7C,mBAAmB,IAAI,2BAA2B,EACnD,MAAM,sBAAsB,CAAC;AAE9B;;;;GAIG;AACH,MAAM,MAAM,wBAAwB,GAAG,uBAAuB,CAAC;AAE/D;;;;;;GAMG;AACH,MAAM,MAAM,eAAe,GAAG,eAAe,CAAC;AAE9C;;;;;;;;;GASG;AACH,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,sBAAsB,EAAE,CAAC;IAChC,KAAK,EAAE,sBAAsB,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,sBAAsB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,OAAO,GAAG,WAAW,GAAG,WAAW,GAAG,UAAU,CAAC;IACvD,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,MAAM,CAAC;IACX,4DAA4D;IAC5D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,eAAe;IAC9B,WAAW,EAAE,eAAe,CAAC;IAC7B,uGAAuG;IACvG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,kBAAkB,GAC5B,WAAW,GAAG,WAAW,GAAG,MAAM,GAAG,YAAY,GAAG,gBAAgB,GAAG,UAAU,CAAC;AAEpF,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,kBAAkB,CAAC;IAC9B,mEAAmE;IACnE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4CAA4C;IAC5C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,MAAM,iBAAiB,GAAG,QAAQ,GAAG,YAAY,GAAG,UAAU,GAAG,QAAQ,GAAG,IAAI,GAAG,IAAI,CAAC;AAE9F,MAAM,WAAW,mBAAmB;IAClC,yBAAyB;IACzB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,GAAG,IAAI,CAAC;CAC1C;AAED;;;;;;;;;GASG;AACH,MAAM,MAAM,aAAa,GAAG,YAAY,GAAG,cAAc,GAAG,eAAe,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAElG;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,aAAa,CAAC;IACxB,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,UAAU,CAAC,EAAE,aAAa,CAAC;IAC3B,2FAA2F;IAC3F,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,oBAAoB,CAAC,CAAC;IACjD,4FAA4F;IAC5F,eAAe,CAAC,EAAE,oBAAoB,CAAC;CACxC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,eAAe,CAAC;IAC7B,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+EAA+E;IAC/E,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,kBAAkB,CAAC;IACnC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,eAAe,CAAC;IAC7B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,qBAAqB,CAAC;AAEzD,MAAM,MAAM,UAAU,GACpB,eAAe,GAAG,kBAAkB,GAAG,eAAe,GAAG,oBAAoB,CAAC;AAEhF,wEAAwE;AACxE,MAAM,MAAM,aAAa,GAAG,cAAc,CAAC;AAE3C;;;;;;;GAOG;AACH,MAAM,WAAW,cAAe,SAAQ,kBAAkB;IACxD,IAAI,EAAE,OAAO,CAAC;IACd,2DAA2D;IAC3D,UAAU,CAAC,EAAE;QACX,MAAM,EAAE,MAAM,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;KACf,CAAC;CACH;AAED;;;GAGG;AACH,MAAM,WAAW,QAAQ;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU;IACV,IAAI,EAAE,MAAM,CAAC;IACb,yBAAyB;IACzB,IAAI,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,UAAU,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACrC,yCAAyC;IACzC,KAAK,EAAE,QAAQ,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,qEAAqE;IACrE,KAAK,EAAE,QAAQ,GAAG;QAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE;YAAE,MAAM,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;KAAE,CAAC;IAC9F,YAAY,EAAE;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,mBAAoB,SAAQ,2BAA2B;IACtE,IAAI,EAAE,aAAa,CAAC;CACrB;AAED,MAAM,MAAM,SAAS,GAAG,cAAc,GAAG,iBAAiB,GAAG,cAAc,GAAG,mBAAmB,CAAC;AAElG,MAAM,WAAW,WAAW;IAC1B,6CAA6C;IAC7C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd;;;OAGG;IACH,KAAK,EAAE;QACL,GAAG,CAAC,EAAE,MAAM,CAAC;QACb,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,qDAAqD;QACrD,GAAG,CAAC,EAAE,OAAO,CAAC;QACd,uBAAuB;QACvB,OAAO,CAAC,EAAE;YACR,KAAK,EAAE;gBAAE,IAAI,EAAE,MAAM,CAAC;gBAAC,IAAI,EAAE,MAAM,CAAA;aAAE,EAAE,CAAC;SACzC,CAAC;KACH,CAAC;IACF,0BAA0B;IAC1B,iBAAiB,CAAC,EAAE;QAClB,kCAAkC;QAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,uBAAuB;QACvB,OAAO,CAAC,EAAE;YACR,IAAI,EAAE,aAAa,GAAG,OAAO,CAAC;YAC9B,KAAK,EAAE,MAAM,CAAC;SACf,CAAC;QACF,kCAAkC;QAClC,gBAAgB,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;QACpC,+BAA+B;QAC/B,YAAY,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;KACjC,CAAC;CACH;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC;;;;OAIG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;;OAKG;IACH,SAAS,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;CACzC;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,4FAA4F;IAC5F,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB;;;OAGG;IACH,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;IACf;;;;OAIG;IACH,WAAW,CAAC,EAAE,eAAe,CAAC;CAC/B;AAED,MAAM,WAAW,YAAa,SAAQ,WAAW,EAAE,aAAa;IAC9D,iCAAiC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iCAAiC;IACjC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,0BAA0B;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,iCAAiC;IACjC,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,QAAQ,GAChB,SAAS,GACT,aAAa,GACb,SAAS,GACT,QAAQ,GACR,WAAW,GACX,QAAQ,GACR,kBAAkB,GAClB,SAAS,CAAC;AAEd,MAAM,WAAW,YAAY;IAC3B,4DAA4D;IAC5D,OAAO,EAAE,MAAM,CAAC;IAChB,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,kCAAkC;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,uBAAuB;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,+BAA+B;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,4EAA4E;IAC5E,KAAK,EAAE,MAAM,CAAC;IACd,+BAA+B;IAC/B,WAAW,EAAE,MAAM,CAAC;CACrB"}

package/dist/worker.d.ts

@@ -19,6 +19,7 @@ export declare class WorkerPool {
     private readonly events;
     private readonly runtime;
     private publisher?;
+    private checkpointStore?;
     private isRunning;
     /** Job duration and per-type counters, ready for Prometheus exposition */
     readonly metrics: MetricsCollector;
@@ -37,6 +38,12 @@ export declare class WorkerPool {
      */
     private processJob;
     private runJob;
+    /**
+     * Whether the queue runs a job again after it failed with `error`, as `processJob` hands the
+     * failure on: a job a shutdown stopped is taken up by another worker, one cancelled or whose
+     * agent run paused fails for good, any other gets the attempts it has left
+     */
+    private willRetry;
     /**
      * Cancel a job this pool is running: its agent runs, workflow nodes and swarm turns are
      * aborted and the job fails without retries.
@@ -45,6 +52,11 @@ export declare class WorkerPool {
      */
     cancelJob(jobId: string, reason?: string): boolean;
     private getPublisher;
+    /**
+     * Where workflow jobs keep their checkpoints: `checkpoints` of the config, else this pool's
+     * Redis, under the queue's prefix and name, so queues sharing a Redis keep theirs apart
+     */
+    private workflowCheckpoints;
     /**
      * Get current worker count
      */

package/dist/worker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAiB,MAAM,SAAS,CAAC;AAKhG,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAUhD,MAAM,WAAW,gBAAgB;IAC/B,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC;IACjE,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;IAC5D,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACpD,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CACxC;AAED,qBAAa,UAAU;IACrB,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,WAAW,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA0B;IAClD,OAAO,CAAC,SAAS,CAAC,CAAc;IAChC,OAAO,CAAC,SAAS,CAAS;IAE1B,0EAA0E;IAC1E,QAAQ,CAAC,OAAO,mBAA0B;gBAE9B,MAAM,EAAE,YAAY,EAAE,MAAM,GAAE,gBAAqB;IAS/D;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA6C5B;;;;;;;OAOG;YACW,UAAU;IAexB,OAAO,CAAC,MAAM;IAuBd;;;;;OAKG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,SAAc,GAAG,OAAO;IAQvD,OAAO,CAAC,YAAY;IAKpB;;OAEG;IACH,cAAc,IAAI,MAAM;IAIxB;;OAEG;IACH,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACG,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;IAOvF;;;;;OAKG;IACG,IAAI,CAAC,OAAO,SAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAwB1C;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;YASlB,kBAAkB;YAOlB,cAAc;CAO7B"}
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAMH,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAiB,MAAM,SAAS,CAAC;AAKhG,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAUhD,MAAM,WAAW,gBAAgB;IAC/B,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC;IACjE,cAAc,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;IAC5D,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACpD,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CACxC;AAaD,qBAAa,UAAU;IACrB,OAAO,CAAC,OAAO,CAAuC;IACtD,OAAO,CAAC,WAAW,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;IACtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAuE;IAC/F,OAAO,CAAC,SAAS,CAAC,CAAc;IAChC,OAAO,CAAC,eAAe,CAAC,CAAkB;IAC1C,OAAO,CAAC,SAAS,CAAS;IAE1B,0EAA0E;IAC1E,QAAQ,CAAC,OAAO,mBAA0B;gBAE9B,MAAM,EAAE,YAAY,EAAE,MAAM,GAAE,gBAAqB;IAS/D;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IA0D5B;;;;;;;OAOG;YACW,UAAU;IAexB,OAAO,CAAC,MAAM;IA2Bd;;;;OAIG;IACH,OAAO,CAAC,SAAS;IAMjB;;;;;OAKG;IACH,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,SAAc,GAAG,OAAO;IAQvD,OAAO,CAAC,YAAY;IAKpB;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IAoB3B;;OAEG;IACH,cAAc,IAAI,MAAM;IAIxB;;OAEG;IACH,aAAa,IAAI,OAAO;IAIxB;;OAEG;IACG,UAAU,CAAC,WAAW,EAAE,IAAI,CAAC,YAAY,EAAE,aAAa,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC;IAOvF;;;;;OAKG;IACG,IAAI,CAAC,OAAO,SAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAwB1C;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;YASlB,kBAAkB;YAOlB,cAAc;CAQ7B"}

package/dist/worker.js

@@ -6,12 +6,21 @@
  */
 import { UnrecoverableError, Worker } from 'bullmq';
 import { Cogitator, findAgentRunPausedError } from '@cogitator-ai/core';
+import { RedisCheckpointStore } from '@cogitator-ai/workflows';
 import { processAgentJob } from './processors/agent.js';
-import { processWorkflowJob } from './processors/workflow.js';
+import { forgetWorkflowRun, processWorkflowJob } from './processors/workflow.js';
 import { processSwarmJob } from './processors/swarm.js';
 import { processSwarmAgentJob } from './processors/swarm-agent.js';
 import { MetricsCollector } from './metrics.js';
 import { DEFAULT_QUEUE_NAME, createBullConnection, createRedisClient, queuePrefix, } from './connection.js';
+/** Seconds the default store keeps a workflow checkpoint nobody resumes or lets go */
+const DEFAULT_CHECKPOINT_TTL = 7 * 24 * 60 * 60;
+/** What a job result reports it used, in the shape the metrics count */
+function usageOf(result) {
+    if (!('usage' in result) || !result.usage)
+        return undefined;
+    return result.usage;
+}
 export class WorkerPool {
     workers = [];
     connections = [];
@@ -19,6 +28,7 @@ export class WorkerPool {
     events;
     runtime;
     publisher;
+    checkpointStore;
     isRunning = false;
     /** Job duration and per-type counters, ready for Prometheus exposition */
     metrics = new MetricsCollector();
@@ -52,12 +62,21 @@ export class WorkerPool {
                 if (job.processedOn !== undefined && job.finishedOn !== undefined) {
                     this.metrics.recordJob(job.data.type, job.finishedOn - job.processedOn);
                 }
+                const usage = usageOf(result);
+                if (usage)
+                    this.metrics.recordUsage(job.data.type, usage);
                 this.events.onJobCompleted?.(job.id ?? job.data.jobId, result);
             });
             worker.on('failed', (job, error) => {
                 if (job) {
-                    if (job.finishedOn !== undefined)
+                    if (job.finishedOn !== undefined) {
                         this.metrics.recordFailure(job.data.type);
+                        if (job.data.type === 'workflow' && this.isRunning) {
+                            forgetWorkflowRun(job.data, this.workflowCheckpoints()).catch((cause) => this.events.onWorkerError?.(new Error(`Could not delete the checkpoint of workflow job ${job.data.jobId}`, {
+                                cause,
+                            })));
+                        }
+                    }
                     this.events.onJobFailed?.(job.id ?? job.data.jobId, error);
                 }
             });
@@ -97,7 +116,7 @@ export class WorkerPool {
             case 'agent':
                 return processAgentJob(job.data, this.runtime, execution);
             case 'workflow':
-                return processWorkflowJob(job.data, this.runtime, execution);
+                return processWorkflowJob(job.data, { ...this.runtime, checkpoints: this.workflowCheckpoints() }, { ...execution, willRetry: (error) => this.willRetry(job, error, signal) });
             case 'swarm':
                 return processSwarmJob(job.data, this.runtime, execution);
             case 'swarm-agent':
@@ -113,6 +132,18 @@ export class WorkerPool {
             }
         }
     }
+    /**
+     * Whether the queue runs a job again after it failed with `error`, as `processJob` hands the
+     * failure on: a job a shutdown stopped is taken up by another worker, one cancelled or whose
+     * agent run paused fails for good, any other gets the attempts it has left
+     */
+    willRetry(job, error, signal) {
+        if (error instanceof UnrecoverableError || findAgentRunPausedError(error))
+            return false;
+        if (signal?.aborted)
+            return !this.isRunning;
+        return job.attemptsMade + 1 < (job.opts.attempts ?? 1);
+    }
     /**
      * Cancel a job this pool is running: its agent runs, workflow nodes and swarm turns are
      * aborted and the job fails without retries.
@@ -131,6 +162,30 @@ export class WorkerPool {
         this.publisher ??= createRedisClient(this.config.redis);
         return this.publisher;
     }
+    /**
+     * Where workflow jobs keep their checkpoints: `checkpoints` of the config, else this pool's
+     * Redis, under the queue's prefix and name, so queues sharing a Redis keep theirs apart
+     */
+    workflowCheckpoints() {
+        if (this.config.checkpoints)
+            return this.config.checkpoints;
+        const redis = this.getPublisher();
+        this.checkpointStore ??= new RedisCheckpointStore({
+            client: {
+                get: (key) => redis.get(key),
+                set: (key, value) => redis.set(key, value),
+                del: (...keys) => redis.del(...keys),
+                zadd: (key, score, member) => redis.zadd(key, score, member),
+                zrange: (key, start, stop) => redis.zrange(key, String(start), String(stop)),
+                zrem: (key, ...members) => redis.zrem(key, ...members),
+                expire: (key, seconds) => redis.expire(key, seconds),
+                zrangebyscore: (key, min, max) => redis.zrangebyscore(key, min, max),
+            },
+            keyPrefix: `${queuePrefix(this.config.redis)}:${this.config.name ?? DEFAULT_QUEUE_NAME}:workflow-checkpoints`,
+            ttl: this.config.checkpointTtl ?? DEFAULT_CHECKPOINT_TTL,
+        });
+        return this.checkpointStore;
+    }
     /**
      * Get current worker count
      */
@@ -201,6 +256,7 @@ export class WorkerPool {
     async closePublisher() {
         const publisher = this.publisher;
         this.publisher = undefined;
+        this.checkpointStore = undefined;
         if (publisher) {
             await publisher.quit();
         }

package/dist/worker.js.map

@@ -1 +1 @@
-{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,EAAY,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAExE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,iBAAiB,EACjB,WAAW,GAGZ,MAAM,iBAAiB,CAAC;AASzB,MAAM,OAAO,UAAU;IACb,OAAO,GAAoC,EAAE,CAAC;IAC9C,WAAW,GAAqB,EAAE,CAAC;IAC1B,MAAM,CAAe;IACrB,MAAM,CAAmB;IACzB,OAAO,CAA0B;IAC1C,SAAS,CAAe;IACxB,SAAS,GAAG,KAAK,CAAC;IAE1B,0EAA0E;IACjE,OAAO,GAAG,IAAI,gBAAgB,EAAE,CAAC;IAE1C,YAAY,MAAoB,EAAE,SAA2B,EAAE;QAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG;YACb,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI,SAAS,EAAE;YAC9C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAE3B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QAEjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,UAAU,GAAG,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/E,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAClC,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,kBAAkB,EACtC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,EAC3D;gBACE,UAAU,EAAE,UAAU,CAAC,UAAU;gBACjC,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACtC,WAAW;gBACX,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,KAAK;gBAC/C,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,KAAK;aACtD,CACF,CAAC;YAEF,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;gBACrC,IAAI,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;oBAClE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;gBAC1E,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBACjC,IAAI,GAAG,EAAE,CAAC;oBACR,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS;wBAAE,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;oBAC5E,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC3B,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,UAAU,CAAC,GAAoB,EAAE,MAAoB;QACjE,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpE,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACtC,MAAM,IAAI,kBAAkB,CAAC,kBAAkB,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACzE,CAAC;YACD,MAAM,MAAM,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;YAC9C,IAAI,MAAM;gBAAE,MAAM,IAAI,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,GAAoB,EAAE,MAAoB;QACvD,MAAM,SAAS,GAAG,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACtB,KAAK,OAAO;gBACV,OAAO,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAC5D,KAAK,UAAU;gBACb,OAAO,kBAAkB,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAC/D,KAAK,OAAO;gBACV,OAAO,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAC5D,KAAK,aAAa;gBAChB,OAAO,oBAAoB,CAAC,GAAG,CAAC,IAAI,EAAE;oBACpC,GAAG,IAAI,CAAC,OAAO;oBACf,GAAG,SAAS;oBACZ,SAAS,EAAE,IAAI,CAAC,YAAY,EAAE;oBAC9B,cAAc,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;iBACjE,CAAC,CAAC;YACL,OAAO,CAAC,CAAC,CAAC;gBACR,MAAM,WAAW,GAAU,GAAG,CAAC,IAAI,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,qBAAsB,WAA0B,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,KAAa,EAAE,MAAM,GAAG,WAAW;QAC3C,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC;gBAAE,KAAK,GAAG,IAAI,CAAC;QACpD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,YAAY;QAClB,IAAI,CAAC,SAAS,KAAK,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,WAA8C;QAC7D,OAAO;YACL,GAAG,WAAW;YACd,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;SACjC,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,KAAK;QACxB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAE5B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,KAAgD,CAAC;QACrD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC;YAC5D,IAAI,OAAO,CAAU,CAAC,OAAO,EAAE,EAAE;gBAC/B,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YACnD,CAAC,CAAC;SACH,CAAC,CAAC;QACH,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;QAE/B,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,MAAM,MAAM,IAAI,OAAO;gBAAE,MAAM,CAAC,aAAa,CAAC,qBAAqB,CAAC,CAAC;YAC1E,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,KAAK,MAAM,MAAM,IAAI,OAAO;YAAE,MAAM,CAAC,aAAa,CAAC,qBAAqB,CAAC,CAAC;QAC1E,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACrC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACvD,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;CACF;AAED,SAAS,YAAY,CAAC,MAAmB;IACvC,MAAM,MAAM,GAAY,MAAM,CAAC,MAAM,CAAC;IACtC,IAAI,MAAM,YAAY,KAAK;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC;IACnD,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1E,CAAC"}
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../src/worker.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,EAAY,MAAM,QAAQ,CAAC;AAC9D,OAAO,EAAE,SAAS,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AACxE,OAAO,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAG/D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AACjF,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EACL,kBAAkB,EAClB,oBAAoB,EACpB,iBAAiB,EACjB,WAAW,GAGZ,MAAM,iBAAiB,CAAC;AASzB,sFAAsF;AACtF,MAAM,sBAAsB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;AAEhD,wEAAwE;AACxE,SAAS,OAAO,CACd,MAAiB;IAEjB,IAAI,CAAC,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC5D,OAAO,MAAM,CAAC,KAAK,CAAC;AACtB,CAAC;AAED,MAAM,OAAO,UAAU;IACb,OAAO,GAAoC,EAAE,CAAC;IAC9C,WAAW,GAAqB,EAAE,CAAC;IAC1B,MAAM,CAAe;IACrB,MAAM,CAAmB;IACzB,OAAO,CAAuE;IACvF,SAAS,CAAe;IACxB,eAAe,CAAmB;IAClC,SAAS,GAAG,KAAK,CAAC;IAE1B,0EAA0E;IACjE,OAAO,GAAG,IAAI,gBAAgB,EAAE,CAAC;IAE1C,YAAY,MAAoB,EAAE,SAA2B,EAAE;QAC7D,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,OAAO,GAAG;YACb,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI,SAAS,EAAE;YAC9C,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;SAC1B,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QAE3B,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QACjD,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC;QAEjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,UAAU,GAAG,oBAAoB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/E,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAClC,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,kBAAkB,EACtC,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,MAAM,CAAC,EAC3D;gBACE,UAAU,EAAE,UAAU,CAAC,UAAU;gBACjC,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;gBACtC,WAAW;gBACX,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,YAAY,IAAI,KAAK;gBAC/C,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,eAAe,IAAI,KAAK;aACtD,CACF,CAAC;YAEF,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE;gBACrC,IAAI,GAAG,CAAC,WAAW,KAAK,SAAS,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;oBAClE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,WAAW,CAAC,CAAC;gBAC1E,CAAC;gBACD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC9B,IAAI,KAAK;oBAAE,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC1D,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACjE,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBACjC,IAAI,GAAG,EAAE,CAAC;oBACR,IAAI,GAAG,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;wBACjC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;wBAC1C,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;4BACnD,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,mBAAmB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE,CAC/E,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CACzB,IAAI,KAAK,CAAC,mDAAmD,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE;gCAC7E,KAAK;6BACN,CAAC,CACH,CACF,CAAC;wBACJ,CAAC;oBACH,CAAC;oBACD,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;gBAC7D,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;gBAC3B,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC;YACrC,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IAED;;;;;;;OAOG;IACK,KAAK,CAAC,UAAU,CAAC,GAAoB,EAAE,MAAoB;QACjE,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEpE,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACtC,MAAM,IAAI,kBAAkB,CAAC,kBAAkB,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACzE,CAAC;YACD,MAAM,MAAM,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;YAC9C,IAAI,MAAM;gBAAE,MAAM,IAAI,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACzD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEO,MAAM,CAAC,GAAoB,EAAE,MAAoB;QACvD,MAAM,SAAS,GAAG,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;QAChD,QAAQ,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACtB,KAAK,OAAO;gBACV,OAAO,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAC5D,KAAK,UAAU;gBACb,OAAO,kBAAkB,CACvB,GAAG,CAAC,IAAI,EACR,EAAE,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,mBAAmB,EAAE,EAAE,EAC5D,EAAE,GAAG,SAAS,EAAE,SAAS,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAC3E,CAAC;YACJ,KAAK,OAAO;gBACV,OAAO,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAC5D,KAAK,aAAa;gBAChB,OAAO,oBAAoB,CAAC,GAAG,CAAC,IAAI,EAAE;oBACpC,GAAG,IAAI,CAAC,OAAO;oBACf,GAAG,SAAS;oBACZ,SAAS,EAAE,IAAI,CAAC,YAAY,EAAE;oBAC9B,cAAc,EAAE,GAAG,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;iBACjE,CAAC,CAAC;YACL,OAAO,CAAC,CAAC,CAAC;gBACR,MAAM,WAAW,GAAU,GAAG,CAAC,IAAI,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,qBAAsB,WAA0B,CAAC,IAAI,EAAE,CAAC,CAAC;YAC3E,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,SAAS,CAAC,GAAoB,EAAE,KAAc,EAAE,MAAoB;QAC1E,IAAI,KAAK,YAAY,kBAAkB,IAAI,uBAAuB,CAAC,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACxF,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5C,OAAO,GAAG,CAAC,YAAY,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC;IACzD,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,KAAa,EAAE,MAAM,GAAG,WAAW;QAC3C,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC;gBAAE,KAAK,GAAG,IAAI,CAAC;QACpD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,YAAY;QAClB,IAAI,CAAC,SAAS,KAAK,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACxD,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;;OAGG;IACK,mBAAmB;QACzB,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;QAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAClC,IAAI,CAAC,eAAe,KAAK,IAAI,oBAAoB,CAAC;YAChD,MAAM,EAAE;gBACN,GAAG,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;gBAC5B,GAAG,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC;gBAC1C,GAAG,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;gBACpC,IAAI,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC;gBAC5D,MAAM,EAAE,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC5E,IAAI,EAAE,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;gBACtD,MAAM,EAAE,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC;gBACpD,aAAa,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC;aACrE;YACD,SAAS,EAAE,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,kBAAkB,uBAAuB;YAC7G,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,IAAI,sBAAsB;SACzD,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;IAC7B,CAAC;IAED;;OAEG;IACH,aAAa;QACX,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU,CAAC,WAA8C;QAC7D,OAAO;YACL,GAAG,WAAW;YACd,WAAW,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;SACjC,CAAC;IACJ,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,IAAI,CAAC,OAAO,GAAG,KAAK;QACxB,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAE5B,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAElB,IAAI,KAAgD,CAAC;QACrD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC;YAC5D,IAAI,OAAO,CAAU,CAAC,OAAO,EAAE,EAAE;gBAC/B,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;YACnD,CAAC,CAAC;SACH,CAAC,CAAC;QACH,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;QAE/B,IAAI,QAAQ,EAAE,CAAC;YACb,KAAK,MAAM,MAAM,IAAI,OAAO;gBAAE,MAAM,CAAC,aAAa,CAAC,qBAAqB,CAAC,CAAC;YAC1E,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAClC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,SAAS;QACb,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC7B,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QAClB,KAAK,MAAM,MAAM,IAAI,OAAO;YAAE,MAAM,CAAC,aAAa,CAAC,qBAAqB,CAAC,CAAC;QAC1E,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,kBAAkB;QAC9B,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACrC,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACvD,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;IAC9B,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,eAAe,GAAG,SAAS,CAAC;QACjC,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,SAAS,CAAC,IAAI,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;CACF;AAED,SAAS,YAAY,CAAC,MAAmB;IACvC,MAAM,MAAM,GAAY,MAAM,CAAC,MAAM,CAAC;IACtC,IAAI,MAAM,YAAY,KAAK;QAAE,OAAO,MAAM,CAAC,OAAO,CAAC;IACnD,OAAO,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1E,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cogitator-ai/worker",
-  "version": "0.9.24",
+  "version": "0.10.0",
   "description": "Distributed job queue for Cogitator agent execution",
   "keywords": [
     "cogitator",
@@ -29,15 +29,16 @@
     "node": ">=22.12.0"
   },
   "dependencies": {
-    "@cogitator-ai/core": "0.56.1",
-    "@cogitator-ai/swarms": "0.13.11",
-    "@cogitator-ai/types": "0.59.0",
+    "@cogitator-ai/core": "0.56.2",
+    "@cogitator-ai/swarms": "0.13.12",
+    "@cogitator-ai/types": "0.60.0",
+    "@cogitator-ai/workflows": "0.14.0",
     "bullmq": "^6.3.11",
     "nanoid": "^6.0.1",
     "zod": "^4.6.5"
   },
   "devDependencies": {
-    "@cogitator-ai/test-utils": "0.11.0",
+    "@cogitator-ai/test-utils": "0.11.1",
     "ioredis": "^6.0.0",
     "typescript": "^6.0.3",
     "vitest": "^5.0.3"