@elysiumjs/heracles 0.5.1 → 0.6.1

package/CHANGELOG.md

@@ -0,0 +1,10 @@
+# @elysiumjs/heracles
+
+## 0.6.0
+
+### Minor Changes
+
+- Improve Redis transport with better reliability and performance.
+- Add performance monitoring utilities.
+- Decouple queue commands (clean, list, work) into their own modules.
+- Enhance base and Redis workers.

package/dist/commands/clean.command.d.ts

@@ -0,0 +1,18 @@
+import { Command } from '@elysiumjs/core';
+/**
+ * Command used to clean a Redis stream.
+ * @author Axel Nana <axel.nana@workbud.com>
+ */
+export declare class HeraclesCleanCommand extends Command {
+    static readonly command: string;
+    static readonly description: string;
+    private redis;
+    private queues;
+    private dryRun;
+    private verbose;
+    private all;
+    private keyPrefix;
+    private retention;
+    private max;
+    run(): Promise<void>;
+}

package/dist/commands/index.d.ts

@@ -0,0 +1,3 @@
+export * from './clean.command';
+export * from './list.command';
+export * from './work.command';

package/dist/commands/list.command.d.ts

@@ -0,0 +1,10 @@
+import { Command } from '@elysiumjs/core';
+/**
+ * Command used to list all registered Heracles jobs.
+ * @author Axel Nana <axel.nana@workbud.com>
+ */
+export declare class HeraclesListCommand extends Command {
+    static readonly command: string;
+    static readonly description: string;
+    run(): Promise<void>;
+}

package/dist/commands/work.command.d.ts

@@ -0,0 +1,18 @@
+import { Command } from '@elysiumjs/core';
+/**
+ * Command used to start an Heracles worker.
+ * @author Axel Nana <axel.nana@workbud.com>
+ */
+export declare class HeraclesWorkCommand extends Command {
+    static readonly command: string;
+    static readonly description: string;
+    private id;
+    private queues;
+    private concurrency;
+    private redis;
+    private maxRetries;
+    private retryDelay;
+    private pauseOnError;
+    private verbose;
+    run(): Promise<void>;
+}

package/dist/index.d.ts

@@ -1,4 +1,5 @@
 import '@elysiumjs/core';
+import './commands';
 export type { JobProps, JobClass, JobMetadata } from './job';
 export type { JobDispatchOptions, QueueOptions, JobDispatchId } from './queue';
 export type { JobStatusInfo, Transport, TransportClass, TransportEvent } from './transport';

package/dist/index.js

@@ -1,2 +1,2 @@
 // @bun
-import"@elysiumjs/core";import{uid as b}from"radash";import{Event as W,InteractsWithConsole as N}from"@elysiumjs/core";import{map as p,uid as n}from"radash";import"reflect-metadata";import{InteractsWithConsole as c,Service as j}from"@elysiumjs/core";import{assign as I,uid as v}from"radash";var R;((Q)=>(Q.job=Symbol("elysium:job"),Q.queue=Symbol("elysium:queue"),Q.worker=Symbol("elysium:worker"),Q.transport=Symbol("elysium:transport"),Q.jobTransaction=Symbol("elysium:jobTransaction"),Q.workerPool=Symbol("elysium:workerPool")))(R||={});var A=(L)=>{if(!Reflect.hasMetadata(R.job,L))return null;return Reflect.getMetadata(R.job,L)};var F;((H)=>{H.ALLOW_OVERLAP="allow_overlap";H.NO_OVERLAP="no_overlap"})(F||={});class K extends c{#Y;#H;#L="pending";#U;#z=0;#V;static register(L={}){return function(Y){L=I({name:Y.name,queue:"default",overlapBehavior:"allow_overlap",overlapDelay:0,priority:0},L);let H=`elysium.heracles.job.${L.name}`;j.instance(H,Y);let U={name:H,queue:L.queue,overlapBehavior:L.overlapBehavior,overlapDelay:L.overlapDelay,priority:L.priority,maxRetries:L.maxRetries,retryDelay:L.retryDelay};Reflect.defineMetadata(R.job,U,Y)}}static generateJobId(...L){return`job_${Date.now()}_${v(8)}`}static description="An Heracles job.";id;dispatchId;createdAt;get startedAt(){return this.#Y}get completedAt(){return this.#H}get status(){return this.#L}get lastError(){return this.#U}get retries(){return this.#z}get queueName(){return this.#V}set queueName(L){this.#V=L}incrementRetries(){this.#z++}constructor(L,Y){super();this.id=L??this.constructor.generateJobId(),this.dispatchId=Y??`dispatch_${Date.now()}_${v(8)}`,this.createdAt=new Date}async run(){if(this.#L==="cancelled")return;try{this.#L="running",this.#Y=new Date;let L=this.#V?` in queue '${this.#V}'`:"";if(this.debug(`Job ${this.id}${L} started at ${this.#Y.toISOString()}`),await this.execute(),this.#L!=="cancelled")this.#L="completed",this.#H=new Date,this.debug(`Job ${this.id} completed successfully at ${this.#H.toISOString()} after ${this.retries} retries`)}catch(L){if(this.#U=L instanceof Error?L:new Error(String(L)),this.#L!=="cancelled")this.#L="failed",this.#H=new Date,this.error(`Job ${this.id} failed at ${this.#H.toISOString()} after ${this.retries} retries: ${this.#U.message}`),this.trace(this.#U)}}cancel(){if(this.#L==="completed"||this.#L==="failed")return!1;let L=this.#L==="running";if(this.#L="cancelled",L)this.#H=new Date;return this.info(`Job ${this.id} was cancelled${L?" while running":" before execution"}`),!0}}var k=(L,Y,H)=>{return class extends L{constructor(...U){super(...U);this.id=Y,this.dispatchId=H}}},T;((X)=>{X.PENDING="pending";X.RUNNING="running";X.COMPLETED="completed";X.FAILED="failed";X.CANCELLED="cancelled";X.SCHEDULED_FOR_RETRY="scheduled_for_retry"})(T||={});var x;((H)=>{H.CONSUMER="consumer";H.PRODUCER="producer"})(x||={});class E extends N{static queues=new Map;static exists(L){return this.queues.has(L)}static get(L,Y){if(!this.queues.has(L)){let H=new E({name:L,...Y});this.queues.set(L,H)}return this.queues.get(L)}static getAll(){return new Map(this.queues)}name;transport;transportClass;transportOptions;options;jobs=[];constructor(L){super();this.name=L.name,this.transportClass=L.transport,this.transportOptions=L.transportOptions,this.options={concurrency:L.concurrency??1,maxRetries:L.maxRetries??0,retryDelay:L.retryDelay??1000,pauseOnError:L.pauseOnError??!1,...L},this.transport=new this.transportClass("producer",this.transportOptions),this.transport.onMessage(this.handleTransportMessage.bind(this))}on(L,Y){W.on(`elysium:heracles:queue:${this.name}:${L}`,Y)}once(L,Y){W.once(`elysium:heracles:queue:${this.name}:${L}`,Y)}emit(L,Y,H=null){W.emit(`elysium:heracles:queue:${this.name}:${L}`,Y,H)}async handleTransportMessage(L){switch(L.type){case"job:status":case"job:result":if(L.queue===this.name){if(L.type==="job:result")this.info(`Job ${L.jobId} completed with status: ${L.status}`);let Y=await this.getJobStatus({jobId:L.jobId,dispatchId:L.dispatchId}),H=Y?.createdAt??new Date().toISOString();this.updateJobStatus(L.jobId,L.dispatchId,{jobId:L.jobId,dispatchId:L.dispatchId,queue:L.queue,status:L.status,error:L.error,retries:L.retries??Y?.retries??0,createdAt:H,startedAt:L.startedAt??Y?.startedAt,completedAt:L.completedAt??Y?.completedAt,updatedAt:L.updatedAt??new Date().toISOString()})}break}this.emit(L.type,L,this)}updateJobStatus(L,Y,H){if(H.status==="completed"||H.status==="failed"||H.status==="cancelled")this.debug(`Job ${L} (${Y}) reached terminal state: ${H.status}`)}async dispatch(L,Y,H=[],U={}){let V=new Date,z=V.toISOString(),X=U.jobId??L.generateJobId(...H),P=U.dispatchId??`dispatch_${Date.now()}_${n(8)}`,Z={jobId:X,dispatchId:P,queue:this.name,status:U.scheduledFor&&U.scheduledFor>V?"scheduled_for_retry":"pending",retries:0,createdAt:z,updatedAt:z};this.updateJobStatus(X,P,Z),this.debug(`Dispatching job ${X} (dispatch: ${P}, name: ${Y}) to queue '${this.name}'`);try{let _={type:"job:process",job:Y,jobId:X,dispatchId:P,args:H,queue:this.name,options:{scheduledFor:U.scheduledFor?.toISOString(),priority:U.priority,maxRetries:U.maxRetries,retryDelay:U.retryDelay,overlapBehavior:U.overlapBehavior,overlapDelay:U.overlapDelay}};return await this.transport.send(_),{jobId:Z.jobId,dispatchId:P}}catch(_){throw this.error(`Failed to dispatch job ${Y} to queue '${this.name}': ${_.message}`),this.updateJobStatus(X,P,{jobId:Z.jobId,dispatchId:P,queue:this.name,status:"failed",error:_.message,retries:0,createdAt:Z.createdAt,completedAt:new Date().toISOString(),updatedAt:new Date().toISOString()}),_}}async cancelJob(L){this.debug(`Canceling job ${L.jobId} (${L.dispatchId}) in queue '${this.name}'`);try{let Y={type:"job:cancel",jobId:L.jobId,dispatchId:L.dispatchId,queue:this.name};return await this.transport.send(Y),!0}catch(Y){return this.error(`Failed to cancel job ${L.jobId}: ${Y}`),!1}}async cancelAllJobs(){this.debug(`Canceling all jobs in queue '${this.name}'`);try{let L={type:"job:cancelAll",queue:this.name};await this.transport.send(L)}catch(L){this.error(`Failed to cancel all jobs: ${L}`)}return 0}async getJobStatus(L){try{let Y=await this.transport.getJobStatus(L.jobId,L.dispatchId,this.name);if(Y)return this.debug(`Retrieved job ${L.jobId} (${L.dispatchId}) status from transport: ${Y.status}`),Y}catch(Y){this.warning(`Failed to get job status from transport: ${Y}`)}return null}getAllJobs(){return Array.from(this.jobs)}async getJobsByStatus(L){let Y=[];for(let[H,U]of await p(this.jobs,async(V)=>[V,await this.getJobStatus(V)]))if(U?.status===L)Y.push(H);return Y}async start(){if(this.transport)await this.transport.start(),this.info(`Queue '${this.name}' transport started`)}async stop(){if(this.transport)await this.transport.stop(),this.info(`Queue '${this.name}' transport stopped`)}getOptions(){return{...this.options}}updateOptions(L){this.options={...this.options,...L}}}var y;((f)=>f.dispatch=async(Y,H=[],U={})=>{let V=A(Y);if(!V)throw new Error(`Job ${Y.name} is not marked with @Job.register() decorator`);if(!V.queue)throw new Error(`Job ${Y.name} is not registered with a queue`);if(!E.exists(V.queue))throw new Error(`Queue ${V.queue} does not exist. Please create the queue before dispatching the job.`);let z=E.get(V.queue),P={dispatchId:`dispatch_${Date.now()}_${b(8)}`,priority:V.priority,maxRetries:V.maxRetries,retryDelay:V.retryDelay,overlapBehavior:V.overlapBehavior,overlapDelay:V.overlapDelay,...U};return z.dispatch(Y,V.name,H,P)})(y||={});import{InteractsWithConsole as i,Redis as g}from"@elysiumjs/core";import{uid as S}from"radash";class D extends i{client;options;mode;messageHandlers=[];pollTimer;cleanupTimer;lastIds=new Map;checkLogs=new Map;consumerId;consumerGroupCreated=new Set;completedMessageIds=new Map;jobLocks=new Map;constructor(L,Y={}){super();this.mode=L,this.options={connection:Y.connection??"default",keyPrefix:Y.keyPrefix??"elysium:heracles",consumerGroup:Y.consumerGroup??"workers",consumerName:Y.consumerName??`worker-${S(8)}`,pollInterval:Y.pollInterval??1000,batchSize:Y.batchSize??10,statusTTL:Y.statusTTL??86400,cleanupCompletedJobs:Y.cleanupCompletedJobs!==!1,completedJobRetention:Y.completedJobRetention??3600,maxStreamSize:Y.maxStreamSize??1000},this.client=g.getConnection(this.options.connection),this.consumerId=this.mode==="consumer"?this.options.consumerName:`producer-${S(8)}`}redisCommand(L,Y){let H=Y.map((U)=>{if(typeof U==="number")return U.toString();else if(U===null||U===void 0)return"";else if(Array.isArray(U))return JSON.stringify(U);else if(typeof U==="object")return JSON.stringify(U);return U});try{return this.client.send(L,H)}catch(U){throw new Error(`Redis command failed: ${L} - ${U}`)}}async findJobMessageId(L,Y,H){let U=this.getStreamKey(H);try{let V=await this.redisCommand("XRANGE",[U,"-","+","COUNT","1000"]);for(let[z,X]of V){let P={};for(let Z=0;Z<X.length;Z+=2)P[X[Z]]=X[Z+1];if(P.jobId===L&&P.dispatchId===Y)return z}}catch(V){this.error(`Error finding job message: ${V}`)}return null}async start(){try{if(await this.redisCommand("PING",[]),this.info(`RedisTransport started in ${this.mode} mode with ID: ${this.consumerId}`),this.mode==="consumer"){if(this.startPolling(),this.options.cleanupCompletedJobs)this.scheduleStreamCleanup()}}catch(L){throw this.error(`Failed to start RedisTransport: ${L}`),new Error(`Redis connection failed: ${L}`)}}async stop(){if(this.pollTimer)clearInterval(this.pollTimer),this.pollTimer=void 0;if(this.cleanupTimer)clearInterval(this.cleanupTimer),this.cleanupTimer=void 0;this.info("RedisTransport stopped")}async send(L){try{let Y=["default"];if("queue"in L&&typeof L.queue==="string")Y=[L.queue];else if("queues"in L&&Array.isArray(L.queues))Y=L.queues;for(let H of Y){let U=this.getStreamKey(H),V=this.serializeMessage(L),z="0";if(L.type==="job:status"||L.type==="job:result")if(await this.findJobMessageId(L.jobId,L.dispatchId,H)){let P={status:L.status,updatedAt:Date.now().toString()};if(L.error)P.error=L.error;if(L.retries!==void 0)P.retries=L.retries.toString();if(L.startedAt)P.startedAt=L.startedAt;if(L.completedAt)P.completedAt=L.completedAt;await this.redisCommand("HSET",[this.getJobStatusKey(L.jobId,L.dispatchId,H),...Object.entries(P).flat()]),this.debug(`Updated status for job ${L.jobId} to ${L.status}`),await this.storeJobStatus({jobId:L.jobId,dispatchId:L.dispatchId,queue:H,status:L.status,error:L.error,retries:L.retries??0,createdAt:L.createdAt??new Date().toISOString(),startedAt:L.startedAt,completedAt:L.completedAt});return}else{let{jobId:P,dispatchId:Z}=L,_=this.getJobStatusKey(P,Z,H),$=await this.redisCommand("EXISTS",[_]);if(z=await this.redisCommand("XADD",[U,"*",...V]),$)await this.updateJobStatus(L.jobId,L.dispatchId,H,{status:L.status,error:L.error,retries:L.retries??0,startedAt:L.startedAt,completedAt:L.completedAt,updatedAt:Date.now().toString()});else await this.storeJobStatus({jobId:L.jobId,dispatchId:L.dispatchId,queue:H,status:L.status,error:L.error,retries:L.retries??0,createdAt:L.createdAt??new Date().toISOString(),startedAt:L.startedAt,completedAt:L.completedAt,messageId:z,updatedAt:Date.now().toString()})}else if(L.type==="job:process")z=await this.redisCommand("XADD",[U,"*",...V]),await this.storeJobStatus({jobId:L.jobId,dispatchId:L.dispatchId,queue:H,status:"pending",retries:0,createdAt:new Date().toISOString(),messageId:z,updatedAt:Date.now().toString()});else if(L.type==="job:update")await this.updateJobStatus(L.jobId,L.dispatchId,H,L.updates);else z=await this.redisCommand("XADD",[U,"*",...V]);this.debug(`Sent ${L.type} message to stream ${U} with ID ${z}`)}}catch(Y){throw this.error(`Failed to send message: ${Y}`),new Error(`Failed to send message: ${Y}`)}}onMessage(L){this.messageHandlers.push(L)}async getJobStatus(L,Y,H){try{let U=this.getJobStatusKey(L,Y,H),V=await this.redisCommand("HGETALL",[U]);if(V){let z={jobId:L,dispatchId:V.dispatchId,queue:H,status:V.status||"unknown",error:V.error,retries:parseInt(V.retries||"0",10),createdAt:V.createdAt||new Date().toISOString(),startedAt:V.startedAt,completedAt:V.completedAt,updatedAt:V.updatedAt,messageId:V.messageId};return this.debug(`Retrieved job status for ${L} from Redis: ${z.status}`),z}}catch(U){this.warning(`Failed to get job status from Redis: ${U}`)}try{let U=await this.findJobMessageId(L,Y,H);if(U){let V={jobId:L,dispatchId:Y,queue:H,status:"pending",retries:0,createdAt:new Date().toISOString(),messageId:U,updatedAt:Date.now().toString()};return await this.storeJobStatus(V),V}}catch(U){this.debug(`Error checking job existence in stream: ${U}`)}return{jobId:L,dispatchId:Y,queue:H,status:"unknown",retries:0,createdAt:new Date().toISOString()}}async updateJobStatus(L,Y,H,U){try{let V=this.getJobStatusKey(L,Y,H),z=Date.now().toString();if(!await this.redisCommand("EXISTS",[V])){if(U.status){let _={jobId:L,dispatchId:Y,queue:H,status:U.status,error:U.error,retries:U.retries||0,createdAt:U.createdAt||new Date().toISOString(),startedAt:U.startedAt,completedAt:U.completedAt,updatedAt:z,messageId:U.messageId};if(await this.storeJobStatus(_),U.status==="completed"||U.status==="failed"||U.status==="cancelled")for(let $ of this.messageHandlers)try{$({type:"job:result",jobId:L,dispatchId:Y,queue:H,status:U.status,error:U.error,completedAt:U.completedAt??new Date().toISOString()})}catch(G){this.error(`Error in message handler: ${G}`)}else for(let $ of this.messageHandlers)try{$({type:"job:status",jobId:L,dispatchId:Y,queue:H,status:U.status,error:U.error,updatedAt:U.updatedAt??new Date().toISOString()})}catch(G){this.error(`Error in message handler: ${G}`)}}return}let P=[],Z={};if(U.status)P.push("status",U.status),Z.status=U.status;if(U.error!==void 0)P.push("error",U.error||""),Z.error=U.error||"";if(U.retries!==void 0)P.push("retries",U.retries.toString()),Z.retries=U.retries;if(U.startedAt)P.push("startedAt",U.startedAt),Z.startedAt=U.startedAt;if(U.completedAt)P.push("completedAt",U.completedAt),Z.completedAt=U.completedAt;if(P.push("updatedAt",z),Z.updatedAt=z,P.length>0)if(await this.redisCommand("HSET",[V,...P]),await this.redisCommand("EXPIRE",[V,this.options.statusTTL]),this.debug(`Updated job ${L} status: ${U.status??"fields updated"}`),U.status==="completed"||U.status==="failed"||U.status==="cancelled"){let _=await this.getJobStatus(L,Y,H);if(_.messageId){this.trackCompletedMessage(_.messageId,this.getStreamKey(H));for(let $ of this.messageHandlers)try{$({type:"job:result",jobId:L,dispatchId:Y,queue:H,status:U.status,error:U.error,completedAt:U.completedAt||z,retries:parseInt(_.retries.toString(),10)})}catch(G){this.error(`Error in message handler: ${G}`)}}}else for(let _ of this.messageHandlers)try{_({...Z,type:"job:status",jobId:L,queue:H})}catch($){this.error(`Error in message handler: ${$}`)}}catch(V){throw this.error(`Failed to update job status: ${V}`),new Error(`Failed to update job status: ${V}`)}}async registerWorker(L,Y){try{for(let H of Y){let U=this.getStreamKey(H);await this.ensureStreamExists(U),await this.ensureConsumerGroupExists(U);let V=this.getWorkerKey(L);await this.redisCommand("HSET",[V,"id",L,"status","active","lastSeen",new Date().toISOString(),"queues",JSON.stringify(Y)]),await this.redisCommand("EXPIRE",[V,"60"]),this.debug(`Registered worker ${L} for queue ${H}`)}await this.send({type:"worker:register",workerId:L,queues:Y})}catch(H){throw this.error(`Failed to register worker: ${H}`),new Error(`Failed to register worker: ${H}`)}}async unregisterWorker(L){try{let Y=this.getWorkerKey(L);await this.redisCommand("DEL",[Y]),await this.send({type:"worker:unregister",workerId:L}),this.debug(`Unregistered worker ${L}`)}catch(Y){throw this.error(`Failed to unregister worker: ${Y}`),new Error(`Failed to unregister worker: ${Y}`)}}startPolling(){this.pollTimer=setInterval(()=>{this.pollMessages().catch((L)=>{this.error(`Error polling messages: ${L}`)})},this.options.pollInterval)}async pollMessages(){if(this.mode!=="consumer")return;let L=await this.getRegisteredQueues();for(let Y of L){let H=this.getStreamKey(Y);try{await this.ensureConsumerGroupExists(H);let U;if(this.checkLogs.get(Y)??!0)U=await this.redisCommand("XREADGROUP",["GROUP",this.options.consumerGroup,this.consumerId,"COUNT",this.options.batchSize,"STREAMS",H,0])??{};else U=await this.redisCommand("XREADGROUP",["GROUP",this.options.consumerGroup,this.consumerId,"COUNT",this.options.batchSize,"STREAMS",H,">"])??{};await this.processMessages(U[H],Y)}catch(U){if(console.error(U),!String(U).includes("timeout"))this.error(`Error reading from stream ${H}: ${U}`)}}}async processMessages(L,Y){if(!L||L.length===0){this.checkLogs.set(Y,!(this.checkLogs.get(Y)??!0));return}let H=this.getStreamKey(Y);for(let[U,V]of L)try{let z=this.deserializeMessage(V,Y);if(this.lastIds.set(H,U),z.type==="job:process"){let X=this.getJobStatusKey(z.jobId,z.dispatchId,Y);if(await this.redisCommand("EXISTS",[X])){if((await this.getJobStatus(z.jobId,z.dispatchId,Y)).status!=="pending"){this.debug(`Skipping duplicate job message for ${z.jobId}`),await this.redisCommand("XACK",[H,this.options.consumerGroup,U]);continue}}}for(let X of this.messageHandlers)try{await X(z)}catch(P){this.error(`Error in message handler: ${P}`)}if((z.type==="job:result"||z.type==="job:status")&&(z.status==="completed"||z.status==="failed"||z.status==="cancelled"))this.trackCompletedMessage(U,H);await this.redisCommand("XACK",[H,this.options.consumerGroup,U])}catch(z){this.error(`Error processing message ${U}: ${z}`)}}async getRegisteredQueues(){try{let L=`${this.options.keyPrefix}:stream:*`,Y=await this.redisCommand("KEYS",[L]);if(!Array.isArray(Y))return["default"];return Y.map((H)=>H.replace(`${this.options.keyPrefix}:stream:`,"")).filter(Boolean)}catch(L){return this.error(`Failed to get registered queues: ${L}`),["default"]}}async ensureStreamExists(L){try{if(!await this.redisCommand("XINFO",["STREAM",L]).catch(()=>null)){let H=await this.redisCommand("XADD",[L,"*","init","true"]);await this.redisCommand("XDEL",[L,H]),this.debug(`Created stream ${L}`)}}catch(Y){this.error(`Failed to ensure stream ${L} exists: ${Y}`)}}async ensureConsumerGroupExists(L){if(this.consumerGroupCreated.has(L))return;try{let Y=await this.redisCommand("XINFO",["GROUPS",L]).catch(()=>[]);if(!(Array.isArray(Y)&&Y.some((U)=>U[1]===this.options.consumerGroup)))await this.redisCommand("XGROUP",["CREATE",L,this.options.consumerGroup,"0","MKSTREAM"]),this.debug(`Created consumer group ${this.options.consumerGroup} for stream ${L}`);this.consumerGroupCreated.add(L)}catch(Y){if(Y.toString().includes("BUSYGROUP"))this.debug(`Consumer group ${this.options.consumerGroup} already exists for stream ${L}`),this.consumerGroupCreated.add(L);else throw this.error(`Failed to ensure consumer group exists: ${Y}`),Y}}async storeJobStatus(L){try{let Y=this.getJobStatusKey(L.jobId,L.dispatchId,L.queue),H=["jobId",L.jobId,"queue",L.queue,"status",L.status,"retries",L.retries.toString(),"createdAt",L.createdAt];if(L.startedAt)H.push("startedAt",L.startedAt);if(L.completedAt)H.push("completedAt",L.completedAt);if(L.error)H.push("error",L.error);if(L.messageId)H.push("messageId",L.messageId);let U=L.updatedAt||Date.now().toString();if(H.push("updatedAt",U),await this.redisCommand("HSET",[Y,...H]),await this.redisCommand("EXPIRE",[Y,this.options.statusTTL]),L.status==="pending"||L.status==="scheduled_for_retry")for(let V of this.messageHandlers)try{V({type:"job:status",jobId:L.jobId,dispatchId:L.dispatchId,queue:L.queue,status:L.status,createdAt:L.createdAt,updatedAt:U})}catch(z){this.error(`Error in message handler: ${z}`)}if(L.status==="completed"||L.status==="failed"||L.status==="cancelled"){let V=L.messageId??null;if(!V)V=await this.findJobMessageId(L.jobId,L.dispatchId,L.queue);if(V){let z=this.getStreamKey(L.queue);this.trackCompletedMessage(V,z);for(let X of this.messageHandlers)try{X({type:"job:result",jobId:L.jobId,dispatchId:L.dispatchId,queue:L.queue,status:L.status,error:L.error,completedAt:L.completedAt||new Date().toISOString(),retries:L.retries})}catch(P){this.error(`Error in message handler: ${P}`)}}}}catch(Y){throw this.error(`Failed to store job status: ${Y}`),new Error(`Failed to store job status: ${Y}`)}}serializeMessage(L){let Y=["type",L.type,"timestamp",Date.now().toString()];if("queue"in L&&typeof L.queue==="string")Y.push("queue",L.queue);switch(L.type){case"job:process":if(Y.push("job",L.job,"args",JSON.stringify(L.args||[]),"jobId",L.jobId,"dispatchId",L.dispatchId),L.options)Y.push("options",JSON.stringify(L.options));break;case"job:cancel":case"job:status":case"job:result":if(Y.push("jobId",L.jobId),Y.push("dispatchId",L.dispatchId),L.type!=="job:cancel"&&L.status)Y.push("status",L.status);if(L.type!=="job:cancel"&&L.error)Y.push("error",L.error);if(L.retries!==void 0)Y.push("retries",L.retries);if(L.startedAt)Y.push("startedAt",L.startedAt);if(L.completedAt)Y.push("completedAt",L.completedAt);break;case"job:cancelAll":break;case"worker:register":Y.push("workerId",L.workerId,"queues",JSON.stringify(L.queues??[]));break;case"worker:unregister":Y.push("workerId",L.workerId);break;case"worker:status":Y.push("workerId",L.workerId,"status",L.status,"queues",JSON.stringify(L.queues),"processing",L.processing.toString(),"waiting",L.waiting.toString());break;case"worker:ready":if(Y.push("id",L.id,"queues",JSON.stringify(L.queues)),L.status)Y.push("status",L.status);if(L.timestamp)Y.push("timestamp",L.timestamp.toString());break}return Y}deserializeMessage(L,Y){let H={};for(let z=0;z<L.length;z+=2)H[L[z]]=L[z+1];let U=H.type,V={type:U,queue:H.queue||Y};switch(U){case"job:process":return{...V,type:"job:process",job:H.job,args:JSON.parse(H.args||"[]"),options:H.options?JSON.parse(H.options):void 0,jobId:H.jobId,dispatchId:H.dispatchId};case"job:cancel":return{...V,type:"job:cancel",jobId:H.jobId,dispatchId:H.dispatchId};case"job:cancelAll":return{...V,type:"job:cancelAll"};case"job:status":return{...V,type:"job:status",jobId:H.jobId,dispatchId:H.dispatchId,status:H.status,error:H.error,retries:H.retries?parseInt(H.retries,10):0,startedAt:H.startedAt,completedAt:H.completedAt};case"job:result":return{...V,type:"job:result",jobId:H.jobId,dispatchId:H.dispatchId,status:H.status,error:H.error,completedAt:H.completedAt||new Date().toISOString()};case"worker:register":return{...V,type:"worker:register",workerId:H.workerId,queues:H.queues?JSON.parse(H.queues):[]};case"worker:unregister":return{...V,type:"worker:unregister",workerId:H.workerId};case"worker:status":return{...V,type:"worker:status",workerId:H.workerId,queues:H.queues?JSON.parse(H.queues):[],status:H.status,processing:parseInt(H.processing||"0",10),waiting:parseInt(H.waiting||"0",10)};case"worker:ready":return{...V,type:"worker:ready",id:H.id,queues:H.queues?JSON.parse(H.queues):[],status:H.status,timestamp:H.timestamp?parseInt(H.timestamp,10):void 0};default:return V}}getStreamKey(L){return`${this.options.keyPrefix}:stream:${L||"default"}`}getJobStatusKey(L,Y,H){return`${this.options.keyPrefix}:status:${H}:${L}:${Y}`}getWorkerKey(L){return`${this.options.keyPrefix}:worker:${L}`}trackCompletedMessage(L,Y){if(this.options.cleanupCompletedJobs)this.completedMessageIds.set(L,{streamKey:Y,timestamp:Date.now()}),this.debug(`Marked message ${L} for future cleanup`)}scheduleStreamCleanup(){if(this.cleanupTimer)clearInterval(this.cleanupTimer);let L=300000;this.cleanupTimer=setInterval(()=>{this.cleanupStreams().catch((Y)=>{this.error(`Error during stream cleanup: ${Y}`)})},L),this.debug(`Scheduled stream cleanup every ${L/1000} seconds`)}async cleanupStreams(){if(!this.options.cleanupCompletedJobs)return;this.debug("Starting stream cleanup for completed jobs");let L=Date.now(),Y=this.options.completedJobRetention*1000,H=new Map,U=[],V=await this.getRegisteredQueues();for(let z of V){let X=this.getStreamKey(z);try{let P=`${this.options.keyPrefix}:status:${z}:*`,Z=await this.redisCommand("KEYS",[P]);for(let _ of Z){let $=await this.redisCommand("HGETALL",[_]);if(!$||$.length===0)continue;let G={};for(let B=0;B<$.length;B+=2)G[$[B]]=$[B+1];if((G.status==="completed"||G.status==="failed"||G.status==="cancelled")&&G.updatedAt&&L-parseInt(G.updatedAt)>Y){let B=_.split(":"),J=B.pop(),l=B.pop();if(U.push(_),G.messageId){if(!H.has(X))H.set(X,[]);H.get(X).push(G.messageId)}else{let O=await this.findJobMessageId(l,J,z);if(O){if(!H.has(X))H.set(X,[]);H.get(X).push(O)}}}}}catch(P){this.error(`Error finding completed jobs for queue ${z}: ${P}`)}}for(let[z,X]of H.entries())if(X.length>0)try{for(let P=0;P<X.length;P+=100){let Z=X.slice(P,P+100);await this.redisCommand("XDEL",[z,...Z])}this.debug(`Cleaned up ${X.length} completed messages from stream ${z}`);for(let P of X)this.completedMessageIds.delete(P)}catch(P){this.error(`Error deleting completed messages from stream ${z}: ${P}`)}if(U.length>0)try{for(let z=0;z<U.length;z+=100){let X=U.slice(z,z+100);await this.redisCommand("DEL",X)}this.debug(`Cleaned up ${U.length} job status records`)}catch(z){this.error(`Error deleting expired status keys: ${z}`)}if(this.options.maxStreamSize>0)await this.trimStreamsToMaxSize()}async trimStreamsToMaxSize(){try{let L=await this.getRegisteredQueues();for(let Y of L){let H=this.getStreamKey(Y),U=await this.redisCommand("XLEN",[H]);if(U>this.options.maxStreamSize)await this.redisCommand("XTRIM",[H,"MAXLEN","~",this.options.maxStreamSize.toString()]),this.debug(`Trimmed stream ${H} from ${U} to ~${this.options.maxStreamSize} entries`)}}catch(L){this.error(`Error trimming streams to max size: ${L}`)}}async isJobLocked(L,Y){let H=this.jobLocks.get(L);if(H&&H.lockedUntil>Date.now())return!0;let U=`${this.options.keyPrefix}:lock:${Y}:${L}`;return!!await this.redisCommand("EXISTS",[U])}async acquireJobLock(L,Y,H=60000){let U=`${this.options.keyPrefix}:lock:${Y}:${L}`;if(await this.redisCommand("SET",[U,Date.now().toString(),"PX",H.toString(),"NX"])==="OK")return this.jobLocks.set(L,{lockedUntil:Date.now()+H,queue:Y}),!0;return!1}async releaseJobLock(L,Y){let H=`${this.options.keyPrefix}:lock:${Y}:${L}`;await this.redisCommand("DEL",[H]),this.jobLocks.delete(L)}}var w;((P)=>{P.IDLE="idle";P.ACTIVE="active";P.PAUSED="paused";P.DRAINING="draining";P.STOPPING="stopping";P.STOPPED="stopped";P.DISCONNECTED="disconnected"})(w||={});import{InteractsWithConsole as q}from"@elysiumjs/core";import{sort as C,uid as r}from"radash";class M extends q{#Y;transport=null;queues=new Map;defaultQueueName="default";defaultTransport;workerStatus="idle";startedAt=new Date;scheduledJobInterval=1000;scheduledJobTimer;constructor(L){super();this.#Y=L??r(24),this.createQueue({name:this.defaultQueueName})}get id(){return this.#Y}get status(){return this.workerStatus}async createQueue(L){if(this.queues.has(L.name))return this.debug(`Queue with name '${L.name}' already exists`),this;let Y={options:{...L,concurrency:L.concurrency||1,maxRetries:L.maxRetries||0,retryDelay:L.retryDelay||1000,pauseOnError:L.pauseOnError||!1},jobs:[],activeJobs:new Map,processing:0,paused:!1,draining:!1};return this.queues.set(L.name,Y),this.debug(`Created queue '${L.name}'`),this}async addJob(L,Y=this.defaultQueueName,H={}){let U=this.queues.get(Y);if(!U)throw this.error(`Queue '${Y}' does not exist`),new Error(`Queue '${Y}' does not exist`);if(U.draining)throw this.warning(`Queue '${Y}' is draining, job ${L.id} will not be added`),new Error(`Queue '${Y}' is draining`);L.queueName=Y;let V={job:L,retries:L.retries,queueName:Y,enqueuedAt:new Date,scheduledFor:H.scheduledFor,priority:H.priority??10,maxRetries:H.maxRetries,retryDelay:H.retryDelay,overlapBehavior:H.overlapBehavior,overlapDelay:H.overlapDelay};if(U.jobs.push(V),U.jobs=C(C(C(U.jobs,(z)=>z.enqueuedAt.getTime()),(z)=>z.scheduledFor?z.scheduledFor.getTime():0),(z)=>z.priority??10),this.debug(`Added job ${L.id} to queue '${Y}'`),this.workerStatus==="idle")this.workerStatus="active";if(!U.paused&&U.processing<U.options.concurrency)try{this.processQueueJobs(Y)}catch(z){this.error(`Error processing next job in queue '${Y}': ${z}`)}return this}async pause(L){if(L){let Y=this.queues.get(L);if(Y)Y.paused=!0,this.info(`Paused queue '${L}'`);else this.warning(`Cannot pause non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.pause(Y);if(Array.from(this.queues.values()).every((Y)=>Y.paused)&&Array.from(this.queues.values()).every((Y)=>Y.processing===0))this.workerStatus="paused",this.info(`Worker ${this.id} is now paused`)}return this}async resume(L){if(L){let Y=this.queues.get(L);if(Y){if(Y.paused=!1,this.info(`Resumed queue '${L}'`),this.workerStatus==="paused")this.workerStatus="active";await this.processQueueJobs(L)}else this.warning(`Cannot resume non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.resume(Y);if(Array.from(this.queues.values()).some((Y)=>Y.jobs.length>0||Y.processing>0))this.workerStatus="active",this.info(`Worker ${this.id} is now active`)}return this}async drain(L){if(L){let Y=this.queues.get(L);if(Y)Y.draining=!0,this.info(`Draining queue '${L}'`);else this.warning(`Cannot drain non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.drain(Y);this.workerStatus="draining",this.info(`Worker ${this.id} is now draining`)}return this}async clear(L){if(L){let Y=this.queues.get(L);if(Y){let H=Y.jobs.length;for(let U of Y.jobs)U.job.cancel();Y.jobs=[],this.info(`Cleared ${H} jobs from queue '${L}'`)}else this.warning(`Cannot clear non-existent queue '${L}'`)}else{for(let[Y,H]of this.queues.entries()){let U=H.jobs.length;for(let V of H.jobs)V.job.cancel();H.jobs=[],this.info(`Cleared ${U} jobs from queue '${Y}'`)}if(Array.from(this.queues.values()).every((Y)=>Y.jobs.length===0&&Y.processing===0))this.workerStatus="idle"}return this}async size(L=this.defaultQueueName){let Y=this.queues.get(L);if(!Y)throw new Error(`Queue '${L}' does not exist`);return Y.jobs.length+Y.processing}async totalSize(){let L=0;for(let Y of this.queues.values())L+=Y.jobs.length+Y.processing;return L}async getStats(){let L={};for(let[Y,H]of this.queues.entries())L[Y]={waiting:H.jobs.length,processing:H.processing,paused:H.paused,draining:H.draining};return L}getInfo(){let L=0;for(let V of this.queues.values())L+=V.processing;let Y=Array.from(this.queues.keys()),H=this.defaultTransport?this.defaultTransport.constructor.name:"None",U=async(V,z)=>{try{return await this.addJob(V,z),!0}catch(X){return!1}};return{id:this.id,status:this.workerStatus,queues:Y,activeJobs:L,transport:H,startedAt:this.startedAt,addJob:U,cancelJob:this.cancelJob.bind(this),cancelAllJobs:this.cancelAllJobs.bind(this)}}async cancelJob(L,Y){if(Y)return this.cancelJobInQueue(L,Y);else{for(let H of this.queues.keys())if(await this.cancelJobInQueue(L,H))return!0;return!1}}async cancelAllJobs(L){let Y=0;if(L){let H=this.queues.get(L);if(H){Y=H.jobs.length;for(let U of H.jobs)U.job.cancel();for(let[U,V]of H.activeJobs.entries())if(V.job.cancel())Y++;this.info(`Canceled ${Y} jobs in queue '${L}'`)}}else for(let[H,U]of this.queues.entries()){let V=U.jobs.length;for(let z of U.jobs)z.job.cancel();for(let[z,X]of U.activeJobs.entries())if(X.job.cancel())V++;Y+=V,this.info(`Canceled ${V} jobs in queue '${H}'`)}if(Y>0&&Array.from(this.queues.values()).every((H)=>H.jobs.length===0&&H.activeJobs.size===0))this.workerStatus="idle";return Y}async getJob(L,Y){if(Y){let H=this.queues.get(Y);if(H){let U=H.jobs.find((z)=>z.job.id===L);if(U)return U.job;let V=H.activeJobs.get(L);if(V)return V.job}}else for(let H of this.queues.values()){let U=H.jobs.find((z)=>z.job.id===L);if(U)return U.job;let V=H.activeJobs.get(L);if(V)return V.job}return null}async start(){if(this.workerStatus="active",this.startedAt=new Date,this.defaultTransport)await this.defaultTransport.start();for(let L of this.queues.keys())await this.processQueueJobs(L);this.startScheduledJobTimer(),this.info(`Worker ${this.id} started`)}async stop(L=!1){if(this.workerStatus="stopping",this.stopScheduledJobTimer(),L){await this.cancelAllJobs();for(let Y of this.queues.keys())await this.clear(Y)}else{for(let H of this.queues.keys())await this.drain(H);let Y=()=>{return Array.from(this.queues.values()).every((H)=>H.processing===0)};if(!Y()){this.info(`Worker ${this.id} waiting for ${Array.from(this.queues.values()).reduce((H,U)=>H+U.processing,0)} jobs to complete`);while(!Y())await new Promise((H)=>setTimeout(H,100))}}if(this.defaultTransport)await this.defaultTransport.stop();this.workerStatus="stopped",this.info(`Worker ${this.id} stopped`)}async setConcurrency(L,Y){let H=this.queues.get(L);if(!H)throw this.error(`Cannot set concurrency: Queue '${L}' does not exist`),new Error(`Queue '${L}' does not exist`);if(Y<1)this.warning(`Invalid concurrency value ${Y}, using 1 instead`),Y=1;return H.options.concurrency=Y,this.info(`Set concurrency for queue '${L}' to ${Y}`),this.processQueueJobs(L)}startScheduledJobTimer(){if(this.scheduledJobTimer)clearInterval(this.scheduledJobTimer);this.scheduledJobTimer=setInterval(()=>{this.checkScheduledJobs()},this.scheduledJobInterval)}stopScheduledJobTimer(){if(this.scheduledJobTimer)clearInterval(this.scheduledJobTimer),this.scheduledJobTimer=void 0}checkScheduledJobs(){let L=new Date;for(let[Y,H]of this.queues.entries()){if(H.paused||H.draining)continue;let U=H.jobs.filter((V)=>V.scheduledFor&&V.scheduledFor<=L);for(let V of U)V.scheduledFor=void 0;if(U.length>0&&H.processing<H.options.concurrency)this.processQueueJobs(Y)}}async processQueueJobs(L){let Y=this.queues.get(L);if(!Y||Y.paused)return;let H=[];while(H.length<Y.options.concurrency)H.push(this.processNextJob(L));await Promise.all(H);let U=Array.from(this.queues.values());if(U.some((V)=>V.processing>0))this.workerStatus="active";else if(U.every((V)=>V.jobs.length===0&&V.processing===0))this.workerStatus="idle";if(U.some((V)=>V.jobs.length>0))await Bun.sleep(10),setImmediate(()=>this.processQueueJobs(L))}async sendJobStatusUpdate(L,Y,H,U,V){try{await this.transport.updateJobStatus(L,Y,H,{dispatchId:Y,status:U,error:V,retries:0,startedAt:U==="running"?new Date().toISOString():void 0,completedAt:U==="completed"||U==="failed"||U==="cancelled"?new Date().toISOString():void 0,updatedAt:Date.now().toString()})}catch(z){this.error(`Error sending job status update: ${z}`)}}async cancelJobInQueue(L,Y){let H=this.queues.get(Y);if(!H)return!1;let U=H.jobs.findIndex((V)=>V.job.id===L);if(U>=0){if(H.jobs[U].job.cancel())return H.jobs.splice(U,1),this.info(`Canceled job ${L} in queue '${Y}'`),!0}if(H.activeJobs.has(L)){if(H.activeJobs.get(L).job.cancel())return this.info(`Canceled running job ${L} in queue '${Y}'`),!0}return!1}}import{Service as d}from"@elysiumjs/core";class h extends M{isRunning=!1;registeredQueues=new Set;constructor(L="default",Y={}){super(Y.id);this.transport=new D("consumer",{connection:L,consumerName:`worker-${this.id}`,...Y}),this.transport.onMessage(this.handleTransportMessage.bind(this)),this.debug(`RedisWorker ${this.id} created with connection: ${L}`)}async start(){if(await super.start(),this.isRunning)return;this.isRunning=!0;for(let L of this.queues.keys())this.registeredQueues.add(L);await this.transport.start(),await this.transport.registerWorker(this.id,Array.from(this.registeredQueues)),this.info(`Redis worker ${this.id} started for queues: ${Array.from(this.registeredQueues).join(", ")}`)}async stop(L=!1){this.isRunning=!1,await super.stop(L),await this.transport.unregisterWorker(this.id),await this.transport.stop(),this.info(`Redis worker ${this.id} stopped`)}async processNextJob(L){let Y=this.queues.get(L);if(!Y||Y.paused||Y.processing>=Y.options.concurrency)return;let H=-1,U=new Date;for(let P=0;P<Y.jobs.length;P++){let Z=Y.jobs[P];if(Z.scheduledFor&&Z.scheduledFor>U)continue;if(Z.job.status==="cancelled"){H=P;break}let _=Z.job.constructor;if(A(_)?.overlapBehavior==="no_overlap"){if(await this.transport.isJobLocked(Z.job.id,L))continue}H=P;break}if(H===-1)return;let V=Y.jobs.splice(H,1)[0];if(!V)return;if(V.job.status==="cancelled")return this.info(`Skipping cancelled job ${V.job.id}`),await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"cancelled"),this.processQueueJobs(L);let z=V.job.constructor,X=A(z);if(X?.overlapBehavior==="no_overlap"){if(!await this.transport.acquireJobLock(V.job.id,L)){Y.jobs.push(V);return}}Y.activeJobs.set(V.job.id,V),Y.processing++,await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"running",void 0);try{if(this.debug(`Processing job ${V.job.id} (dispatch: ${V.job.dispatchId}) from queue '${L}'`),await V.job.run(),V.job.status==="failed"){let P=V.maxRetries!==void 0?V.maxRetries:Y.options.maxRetries;if(V.retries<P){V.retries++,V.job.incrementRetries(),this.warning(`Job ${V.job.id} failed, retrying (${V.retries}/${P})`);let Z=V.retryDelay!==void 0?V.retryDelay:Y.options.retryDelay;await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"scheduled_for_retry",V.job.lastError?.message),setTimeout(()=>{if(V.job.status!=="cancelled")Y.jobs.push(V),this.processQueueJobs(L);else this.info(`Skipping cancelled job ${V.job.id} during retry delay`),this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"cancelled")},Z)}else if(this.error(`Job ${V.job.id} failed after ${V.retries} retries (max: ${P}): ${V.job.lastError?.message}`),await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"failed",V.job.lastError?.message),Y.options.pauseOnError)this.pause(L),this.warning(`Queue '${L}' paused due to job failure`)}else if(V.job.status==="completed")this.success(`Job ${V.job.id} completed successfully`),await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"completed");else if(V.job.status==="cancelled")this.info(`Job ${V.job.id} cancelled while running`),await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"cancelled")}catch(P){if(this.error(`Unexpected error processing job ${V.job.id}: ${P}`),await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"failed",P instanceof Error?P.message:String(P)),Y.options.pauseOnError)this.pause(L),this.warning(`Queue '${L}' paused due to unexpected error`)}finally{if(X?.overlapBehavior==="no_overlap")if(X.overlapDelay&&X.overlapDelay>0)setTimeout(()=>{this.transport.releaseJobLock(V.job.id,L)},X.overlapDelay);else await this.transport.releaseJobLock(V.job.id,L);Y.activeJobs.delete(V.job.id),Y.processing--,this.processQueueJobs(L)}}async createQueue(L){let Y=await super.createQueue(L);if(this.isRunning&&!this.registeredQueues.has(L.name))this.registeredQueues.add(L.name),await this.transport.registerWorker(this.id,Array.from(this.registeredQueues));return Y}async handleTransportMessage(L){try{switch(L.type){case"job:process":{let{job:Y,args:H=[],queue:U="default",options:V={},jobId:z,dispatchId:X}=L;await this.handleProcessJob(z,X,Y,H,U,V);break}case"job:cancel":{let{jobId:Y,dispatchId:H,queue:U="default"}=L;if(await this.cancelJob(Y,U))await this.sendJobStatusUpdate(Y,H,U,"cancelled");break}case"job:cancelAll":{let{queue:Y="default"}=L;await this.cancelAllJobs(Y);break}}}catch(Y){this.error(`Error handling transport message: ${Y}`)}}async handleProcessJob(L,Y,H,U,V,z){try{let X=d.get(H);if(!X)throw new Error(`Job class not found: ${H}`);let P=new(k(X,L,Y))(...U||[]);P.queueName=V;let Z={};if(z.scheduledFor)Z.scheduledFor=typeof z.scheduledFor==="string"?new Date(z.scheduledFor):z.scheduledFor;if(z.priority!==void 0)Z.priority=Number(z.priority);if(z.maxRetries!==void 0)Z.maxRetries=Number(z.maxRetries);if(z.retryDelay!==void 0)Z.retryDelay=Number(z.retryDelay);if(Z.scheduledFor&&Z.scheduledFor>new Date)await this.sendJobStatusUpdate(P.id,P.dispatchId,V,"scheduled_for_retry"),this.debug(`Scheduled job ${P.id} for future execution at ${Z.scheduledFor.toISOString()}`);else await this.sendJobStatusUpdate(P.id,P.dispatchId,V,"pending"),this.debug(`Added job ${P.id} to queue ${V} with initial status: pending`);await this.addJob(P,V,Z)}catch(X){throw this.error(`Error processing job ${H}: ${X.message}`),this.sendJobStatusUpdate(L,Y,V,"failed",X.message),X}}}export{w as WorkerStatus,x as TransportMode,h as RedisWorker,D as RedisTransport,E as Queue,T as JobStatus,F as JobOverlapBehavior,K as Job,y as Heracles,M as BaseWorker};
+import"@elysiumjs/core";import"reflect-metadata";import{InteractsWithConsole as S,Service as h}from"@elysiumjs/core";import{assign as l,uid as W}from"radash";var E;(($)=>($.job=Symbol("elysium:job"),$.queue=Symbol("elysium:queue"),$.worker=Symbol("elysium:worker"),$.transport=Symbol("elysium:transport"),$.jobTransaction=Symbol("elysium:jobTransaction"),$.workerPool=Symbol("elysium:workerPool")))(E||={});var Q=(L)=>{if(!Reflect.hasMetadata(E.job,L))return null;return Reflect.getMetadata(E.job,L)};var G;((F)=>{F.ALLOW_OVERLAP="allow_overlap";F.NO_OVERLAP="no_overlap"})(G||={});class O extends S{#Y;#F;#L="pending";#H;#z=0;#U;static register(L={}){return function(Y){L=l({name:Y.name,queue:"default",overlapBehavior:"allow_overlap",overlapDelay:0,priority:0},L);let F=`elysium.heracles.job.${L.name}`;h.instance(F,Y);let H={name:F,queue:L.queue,overlapBehavior:L.overlapBehavior,overlapDelay:L.overlapDelay,priority:L.priority,maxRetries:L.maxRetries,retryDelay:L.retryDelay};Reflect.defineMetadata(E.job,H,Y)}}static generateJobId(...L){return`job_${Date.now()}_${W(8)}`}static description="An Heracles job.";id;dispatchId;createdAt;get startedAt(){return this.#Y}get completedAt(){return this.#F}get status(){return this.#L}get lastError(){return this.#H}get retries(){return this.#z}get queueName(){return this.#U}set queueName(L){this.#U=L}incrementRetries(){this.#z++}constructor(L,Y){super();this.id=L??this.constructor.generateJobId(),this.dispatchId=Y??`dispatch_${Date.now()}_${W(8)}`,this.createdAt=new Date}async run(){if(this.#L==="cancelled")return;try{this.#L="running",this.#Y=new Date;let L=this.#U?` in queue '${this.#U}'`:"";if(this.debug(`Job ${this.id}${L} started at ${this.#Y.toISOString()}`),await this.execute(),this.#L==="running")this.#L="completed",this.#F=new Date,this.debug(`Job ${this.id} completed successfully at ${this.#F.toISOString()} after ${this.retries} retries`)}catch(L){this.fail(L instanceof Error?L:Error(String(L)))}}fail(L){if(this.#H=L,this.#L!=="cancelled")this.#L="failed",this.#F=new Date,this.error(`Job ${this.id} failed at ${this.#F.toISOString()} after ${this.retries} retries: ${this.#H.message}`),this.trace(this.#H)}cancel(){if(this.#L==="completed"||this.#L==="failed")return!1;let L=this.#L==="running";if(this.#L="cancelled",L)this.#F=new Date;return this.info(`Job ${this.id} was cancelled${L?" while running":" before execution"}`),!0}}var C=(L,Y,F)=>{return class extends L{constructor(...H){super(...H);this.id=Y,this.dispatchId=F}}},A;((V)=>{V.PENDING="pending";V.RUNNING="running";V.COMPLETED="completed";V.FAILED="failed";V.CANCELLED="cancelled";V.SCHEDULED_FOR_RETRY="scheduled_for_retry"})(A||={});import{Redis as c,Service as j}from"@elysiumjs/core";var K;((F)=>{F.CONSUMER="consumer";F.PRODUCER="producer"})(K||={});import{InteractsWithConsole as J,Redis as T}from"@elysiumjs/core";import{uid as w}from"radash";class B extends J{commandClient=null;subscriberClient=null;publisherClient=null;options;mode;messageHandlers=[];consumerId;isRunning=!1;cleanupTimer;jobLocks=new Map;constructor(L,Y={}){super();this.mode=L,this.options={connection:Y.connection||"default",keyPrefix:Y.keyPrefix||"elysium:heracles",consumerGroup:Y.consumerGroup||"workers",consumerName:Y.consumerName||`worker-${w(8)}`,batchSize:Y.batchSize||10,statusTTL:Y.statusTTL||86400,cleanupCompletedJobs:Y.cleanupCompletedJobs!==!1,completedJobRetention:Y.completedJobRetention||3600,maxStreamSize:Y.maxStreamSize||1000},this.consumerId=this.mode==="consumer"?this.options.consumerName:`producer-${w(8)}`}async initializeClients(){this.commandClient=await T.getConnection(this.options.connection).duplicate(),this.subscriberClient=await T.getConnection(this.options.connection).duplicate(),this.publisherClient=await T.getConnection(this.options.connection).duplicate(),this.setupConnectionHandlers()}setupConnectionHandlers(){this.commandClient.onconnect=()=>{this.debug("Command client connected")},this.commandClient.onclose=(L)=>{this.error(`Command client disconnected: ${L}`)},this.subscriberClient.onconnect=()=>{this.debug("Subscriber client connected")},this.subscriberClient.onclose=(L)=>{this.error(`Subscriber client disconnected: ${L}`)},this.publisherClient.onconnect=()=>{this.debug("Publisher client connected")},this.publisherClient.onclose=(L)=>{this.error(`Publisher client disconnected: ${L}`)}}async start(){if(this.isRunning)return;try{if(await this.initializeClients(),await Promise.all([this.commandClient.connect(),this.subscriberClient.connect(),this.publisherClient.connect()]),await this.commandClient.ping(),await this.publisherClient.ping(),this.info(`RedisTransport started in ${this.mode} mode with ID: ${this.consumerId}`),this.mode==="consumer"){if(await this.subscribeToChannels(),this.options.cleanupCompletedJobs)this.scheduleCleanup()}this.isRunning=!0}catch(L){throw this.error(`Failed to start RedisTransport: ${L}`),Error(`Redis connection failed: ${L}`)}}async stop(){if(this.isRunning=!1,this.cleanupTimer)clearInterval(this.cleanupTimer),this.cleanupTimer=void 0;if(this.mode==="consumer")await this.subscriberClient.unsubscribe();this.commandClient.close(),this.subscriberClient.close(),this.publisherClient.close(),this.info("RedisTransport stopped")}async subscribeToChannels(){let L=await this.getRegisteredQueues();for(let Y of L)await this.subscriberClient.subscribe(`${this.options.keyPrefix}:queue:${Y}:new`,async(F,H)=>{this.debug(`New job notification for queue ${Y}: ${F}`),await this.processJobFromStream(Y,F)}),await this.subscriberClient.subscribe(`${this.options.keyPrefix}:queue:${Y}:control`,async(F,H)=>{try{let U=JSON.parse(F);await this.handleControlMessage(U)}catch(U){this.error(`Error parsing control message: ${U}`)}}),await this.subscriberClient.subscribe(`${this.options.keyPrefix}:lock:${Y}:released`,(F,H)=>{this.debug(`Lock released for job ${F} in queue ${Y}`),this.notifyLockRelease(F,Y)});await this.subscriberClient.subscribe(`${this.options.keyPrefix}:worker:coordination`,async(Y,F)=>{try{let H=JSON.parse(Y);await this.handleWorkerCoordination(H)}catch(H){this.error(`Error handling worker coordination: ${H}`)}})}async processJobFromStream(L,Y){let F=this.getStreamKey(L);try{let H=await this.commandClient.send("XRANGE",[F,Y,Y]);if(H&&H.length>0){let[U,z]=H[0],V=this.deserializeMessage(z,L);for(let X of this.messageHandlers)try{await X(V)}catch(Z){this.error(`Error in message handler: ${Z}`)}await this.commandClient.send("XACK",[F,this.options.consumerGroup,Y])}}catch(H){this.error(`Error processing job from stream: ${H}`)}}async handleControlMessage(L){switch(L.type){case"pause":this.info(`Queue ${L.queue} paused`);break;case"resume":this.info(`Queue ${L.queue} resumed`);break;case"drain":this.info(`Queue ${L.queue} draining`);break;default:this.debug(`Unknown control message type: ${L.type}`)}}async handleWorkerCoordination(L){switch(L.type){case"worker:joined":this.debug(`Worker ${L.workerId} joined`);break;case"worker:left":this.debug(`Worker ${L.workerId} left`);break;case"rebalance":this.debug("Rebalancing work across workers");break}}async send(L){try{let Y=this.extractQueueName(L),F=this.getStreamKey(Y);if(L.type==="job:process"){let H=this.serializeMessage(L),U=await this.commandClient.send("XADD",[F,"*",...H]);await this.indexJob({jobId:L.jobId,dispatchId:L.dispatchId,queue:Y,status:"pending",priority:L.options?.priority||10,timestamp:Date.now()}),await this.storeJobStatus({jobId:L.jobId,dispatchId:L.dispatchId,queue:Y,status:"pending",retries:0,createdAt:new Date().toISOString(),updatedAt:new Date().toISOString(),messageId:U}),await this.publisherClient.publish(`${this.options.keyPrefix}:queue:${Y}:new`,U),this.debug(`Job ${L.jobId} dispatched to queue ${Y} with ID ${U}`)}else if(L.type==="job:status"||L.type==="job:result")await this.updateJobStatus(L.jobId,L.dispatchId,Y,{status:L.status,error:L.error,retries:L.retries,completedAt:L.completedAt,updatedAt:new Date().toISOString()}),await this.updateJobIndex(L.jobId,L.dispatchId,Y,L.status),await this.publisherClient.publish(`${this.options.keyPrefix}:job:${L.jobId}:status`,JSON.stringify({status:L.status,error:L.error,timestamp:Date.now()}));else if(L.type==="job:cancel")await this.publisherClient.publish(`${this.options.keyPrefix}:queue:${Y}:control`,JSON.stringify({type:"cancel",jobId:L.jobId,dispatchId:L.dispatchId}));else{let H=this.serializeMessage(L);await this.commandClient.send("XADD",[F,"*",...H])}}catch(Y){throw this.error(`Failed to send message: ${Y}`),Error(`Failed to send message: ${Y}`)}}onMessage(L){this.messageHandlers.push(L)}async getJobStatus(L,Y,F){try{let H=this.getJobStatusKey(L,Y,F),U=await this.commandClient.hmget(H,["jobId","dispatchId","queue","status","error","retries","createdAt","startedAt","completedAt","updatedAt","messageId"]);if(U&&U[0])return{jobId:U[0],dispatchId:U[1],queue:U[2]||F,status:U[3]||"unknown",error:U[4],retries:parseInt(U[5]||"0",10),createdAt:U[6]||new Date().toISOString(),startedAt:U[7],completedAt:U[8],updatedAt:U[9],messageId:U[10]};let z=await this.getJobFromIndex(L,Y,F);if(z)return{jobId:L,dispatchId:Y,queue:F,status:z.status,retries:0,createdAt:new Date(z.timestamp).toISOString()};return{jobId:L,dispatchId:Y,queue:F,status:"unknown",retries:0,createdAt:new Date().toISOString()}}catch(H){return this.warning(`Failed to get job status: ${H}`),{jobId:L,dispatchId:Y,queue:F,status:"unknown",retries:0,createdAt:new Date().toISOString()}}}async updateJobStatus(L,Y,F,H){try{let U=this.getJobStatusKey(L,Y,F),z=[];if(H.status)z.push("status",H.status);if(H.error!==void 0)z.push("error",H.error||"");if(H.retries!==void 0)z.push("retries",H.retries.toString());if(H.startedAt)z.push("startedAt",H.startedAt);if(H.completedAt)z.push("completedAt",H.completedAt);if(H.updatedAt)z.push("updatedAt",H.updatedAt);if(z.length>0){if(await this.commandClient.hmset(U,z),await this.commandClient.expire(U,this.options.statusTTL),H.status)await this.updateJobIndex(L,Y,F,H.status);await this.publisherClient.publish(`${this.options.keyPrefix}:job:${L}:status`,JSON.stringify({...H,timestamp:Date.now()})),this.debug(`Updated job ${L} status: ${H.status||"fields updated"}`)}}catch(U){throw this.error(`Failed to update job status: ${U}`),U}}async registerWorker(L,Y){try{for(let H of Y){let U=this.getStreamKey(H);await this.ensureConsumerGroupExists(U)}let F=this.getWorkerKey(L);await this.commandClient.hmset(F,["id",L,"status","active","lastSeen",new Date().toISOString(),"queues",JSON.stringify(Y)]),await this.commandClient.expire(F,60),await this.publisherClient.publish(`${this.options.keyPrefix}:worker:coordination`,JSON.stringify({type:"worker:joined",workerId:L,queues:Y,timestamp:Date.now()})),this.debug(`Registered worker ${L} for queues: ${Y.join(", ")}`)}catch(F){throw this.error(`Failed to register worker: ${F}`),F}}async unregisterWorker(L){try{let Y=this.getWorkerKey(L);await this.commandClient.del(Y),await this.publisherClient.publish(`${this.options.keyPrefix}:worker:coordination`,JSON.stringify({type:"worker:left",workerId:L,timestamp:Date.now()})),this.debug(`Unregistered worker ${L}`)}catch(Y){throw this.error(`Failed to unregister worker: ${Y}`),Y}}async indexJob(L){let Y=L.timestamp,F=`${L.jobId}:${L.dispatchId}`;await Promise.all([this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:queue:${L.queue}`,Y.toString(),F]),this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:status:${L.status}`,Y.toString(),F]),this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:priority`,L.priority.toString(),F])])}async updateJobIndex(L,Y,F,H){let U=`${L}:${Y}`,z=Date.now(),X=["pending","running","completed","failed","cancelled","scheduled_for_retry"].filter((_)=>_!==H).map((_)=>this.commandClient.send("ZREM",[`${this.options.keyPrefix}:idx:status:${_}`,U])),Z=this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:status:${H}`,z.toString(),U]);await Promise.all([...X,Z])}async getJobFromIndex(L,Y,F){let H=`${L}:${Y}`,U=await this.commandClient.zscore(`${this.options.keyPrefix}:idx:queue:${F}`,H);if(U){let z=["pending","running","completed","failed","cancelled","scheduled_for_retry"];for(let V of z)if(await this.commandClient.zscore(`${this.options.keyPrefix}:idx:status:${V}`,H))return{jobId:L,dispatchId:Y,queue:F,status:V,priority:10,timestamp:U}}return null}async storeJobStatus(L){let Y=this.getJobStatusKey(L.jobId,L.dispatchId,L.queue),F=["jobId",L.jobId,"dispatchId",L.dispatchId,"queue",L.queue,"status",L.status,"retries",L.retries.toString(),"createdAt",L.createdAt,"updatedAt",L.updatedAt||Date.now().toString()];if(L.error)F.push("error",L.error);if(L.startedAt)F.push("startedAt",L.startedAt);if(L.completedAt)F.push("completedAt",L.completedAt);if(L.messageId)F.push("messageId",L.messageId);await this.commandClient.hmset(Y,F),await this.commandClient.expire(Y,this.options.statusTTL)}async ensureConsumerGroupExists(L){try{await this.commandClient.send("XGROUP",["CREATE",L,this.options.consumerGroup,"0","MKSTREAM"]),this.debug(`Created consumer group ${this.options.consumerGroup} for ${L}`)}catch(Y){if(!Y.message?.includes("BUSYGROUP"))this.error(`Error creating consumer group: ${Y}`)}}async getRegisteredQueues(){try{let L=`${this.options.keyPrefix}:stream:*`,Y=await this.commandClient.keys(L);if(!Array.isArray(Y)||Y.length===0)return["default"];return Y.map((F)=>F.replace(`${this.options.keyPrefix}:stream:`,"")).filter(Boolean)}catch(L){return this.error(`Failed to get registered queues: ${L}`),["default"]}}scheduleCleanup(){this.cleanupTimer=setInterval(async()=>{try{await this.performCleanup()}catch(Y){this.error(`Error during cleanup: ${Y}`)}},300000),this.debug("Scheduled cleanup every 300 seconds")}async performCleanup(){this.debug("Starting cleanup operation");let L=Date.now()-this.options.completedJobRetention*1000,Y=await this.getRegisteredQueues(),F=["completed","failed","cancelled"];for(let z of F){let V=`${this.options.keyPrefix}:idx:status:${z}`;await this.commandClient.send("ZREMRANGEBYSCORE",[V,"-inf",L.toString()])}for(let z of Y){let V=this.getStreamKey(z),X=await this.commandClient.send("XLEN",[V]);if(X>this.options.maxStreamSize)await this.commandClient.send("XTRIM",[V,"MAXLEN","~",this.options.maxStreamSize.toString()]),this.debug(`Trimmed stream ${V} from ${X} to ~${this.options.maxStreamSize}`)}let H=`${this.options.keyPrefix}:status:*`,U=await this.commandClient.keys(H);for(let z of U){let V=await this.commandClient.ttl(z);if(V===-1||V===-2)await this.commandClient.del(z)}this.debug("Cleanup operation completed")}extractQueueName(L){if("queue"in L&&typeof L.queue==="string")return L.queue;return"default"}serializeMessage(L){let Y=["type",L.type,"timestamp",Date.now().toString()];if("queue"in L&&L.queue)Y.push("queue",L.queue);switch(L.type){case"job:process":if(Y.push("job",L.job,"args",JSON.stringify(L.args||[]),"jobId",L.jobId,"dispatchId",L.dispatchId),L.options)Y.push("options",JSON.stringify(L.options));break;case"job:cancel":case"job:status":case"job:result":if(Y.push("jobId",L.jobId),Y.push("dispatchId",L.dispatchId),L.type!=="job:cancel"){if(L.status)Y.push("status",L.status);if(L.error)Y.push("error",L.error);if(L.retries!==void 0)Y.push("retries",L.retries.toString());if("completedAt"in L&&L.completedAt)Y.push("completedAt",L.completedAt)}break;case"worker:register":if(Y.push("workerId",L.workerId),L.queues)Y.push("queues",JSON.stringify(L.queues));break;case"worker:unregister":Y.push("workerId",L.workerId);break}return Y}deserializeMessage(L,Y){let F={};for(let z=0;z<L.length;z+=2)F[L[z]]=L[z+1];let H=F.type,U={type:H,queue:F.queue||Y};switch(H){case"job:process":return{...U,type:"job:process",job:F.job,args:JSON.parse(F.args||"[]"),jobId:F.jobId,dispatchId:F.dispatchId,options:F.options?JSON.parse(F.options):void 0};case"job:cancel":return{...U,type:"job:cancel",jobId:F.jobId,dispatchId:F.dispatchId};case"job:status":return{...U,type:"job:status",jobId:F.jobId,dispatchId:F.dispatchId,status:F.status,error:F.error,retries:F.retries?parseInt(F.retries,10):void 0};case"job:result":return{...U,type:"job:result",jobId:F.jobId,dispatchId:F.dispatchId,status:F.status,error:F.error,completedAt:F.completedAt||new Date().toISOString()};default:return U}}notifyLockRelease(L,Y){this.jobLocks.delete(L);for(let F of this.messageHandlers)try{F({type:"job:update",jobId:L,dispatchId:"",queue:Y,status:"lock_released",updates:{}})}catch(H){this.error(`Error notifying lock release: ${H}`)}}async isJobLocked(L,Y){let F=this.jobLocks.get(L);if(F&&F.lockedUntil>Date.now())return!0;let H=`${this.options.keyPrefix}:lock:${Y}:${L}`;return await this.commandClient.exists(H)}async acquireJobLock(L,Y,F=60000){let H=`${this.options.keyPrefix}:lock:${Y}:${L}`,U=`${this.consumerId}:${Date.now()}`;if(await this.commandClient.set(H,U,"PX",F.toString(),"NX")==="OK")return this.jobLocks.set(L,{lockedUntil:Date.now()+F,queue:Y}),this.debug(`Acquired lock for job ${L} in queue ${Y}`),!0;return!1}async releaseJobLock(L,Y){let F=`${this.options.keyPrefix}:lock:${Y}:${L}`;await this.commandClient.del(F),this.jobLocks.delete(L),await this.publisherClient.publish(`${this.options.keyPrefix}:lock:${Y}:released`,L),this.debug(`Released lock for job ${L} in queue ${Y}`)}getStreamKey(L){return`${this.options.keyPrefix}:stream:${L||"default"}`}getJobStatusKey(L,Y,F){return`${this.options.keyPrefix}:status:${F}:${L}:${Y}`}getWorkerKey(L){return`${this.options.keyPrefix}:worker:${L}`}}import{InteractsWithConsole as I}from"@elysiumjs/core";import{sort as M,uid as N}from"radash";var v;((X)=>{X.IDLE="idle";X.ACTIVE="active";X.PAUSED="paused";X.DRAINING="draining";X.STOPPING="stopping";X.STOPPED="stopped";X.DISCONNECTED="disconnected"})(v||={});class P extends I{#Y;transport=null;queues=new Map;defaultQueueName="default";defaultTransport;workerStatus="idle";startedAt=new Date;scheduledJobInterval=1000;scheduledJobTimer;constructor(L){super();this.#Y=L??N(24),this.createQueue({name:this.defaultQueueName})}get id(){return this.#Y}get status(){return this.workerStatus}async createQueue(L){if(this.queues.has(L.name))return this.debug(`Queue with name '${L.name}' already exists`),this;let Y={options:{...L,concurrency:L.concurrency||1,maxRetries:L.maxRetries||0,retryDelay:L.retryDelay||1000,pauseOnError:L.pauseOnError||!1},jobs:[],activeJobs:new Map,retryingJobs:new Map,paused:!1,draining:!1};return this.queues.set(L.name,Y),this.debug(`Created queue '${L.name}'`),this}async addJob(L,Y=this.defaultQueueName,F={}){let H=this.queues.get(Y);if(!H)throw this.error(`Queue '${Y}' does not exist`),Error(`Queue '${Y}' does not exist`);if(H.draining)throw this.warning(`Queue '${Y}' is draining, job ${L.id} will not be added`),Error(`Queue '${Y}' is draining`);L.queueName=Y;let U={job:L,retries:L.retries,queueName:Y,enqueuedAt:new Date,scheduledFor:F.scheduledFor,priority:F.priority??10,maxRetries:F.maxRetries,retryDelay:F.retryDelay,overlapBehavior:F.overlapBehavior,overlapDelay:F.overlapDelay};if(H.jobs.push(U),H.jobs=M(M(M(H.jobs,(z)=>z.enqueuedAt.getTime()),(z)=>z.scheduledFor?z.scheduledFor.getTime():0),(z)=>z.priority??10),this.debug(`Added job ${L.id} to queue '${Y}'`),this.workerStatus==="idle")this.workerStatus="active";if(!H.paused&&H.activeJobs.size<H.options.concurrency)try{this.processQueueJobs(Y)}catch(z){this.error(`Error processing next job in queue '${Y}': ${z}`)}return this}async pause(L){if(L){let Y=this.queues.get(L);if(Y)Y.paused=!0,this.info(`Paused queue '${L}'`);else this.warning(`Cannot pause non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.pause(Y);if(Array.from(this.queues.values()).every((Y)=>Y.paused)&&Array.from(this.queues.values()).every((Y)=>Y.activeJobs.size===0))this.workerStatus="paused",this.info(`Worker ${this.id} is now paused`)}return this}async resume(L){if(L){let Y=this.queues.get(L);if(Y){if(Y.paused=!1,this.info(`Resumed queue '${L}'`),this.workerStatus==="paused")this.workerStatus="active";await this.processQueueJobs(L)}else this.warning(`Cannot resume non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.resume(Y);if(Array.from(this.queues.values()).some((Y)=>Y.jobs.length>0||Y.activeJobs.size>0))this.workerStatus="active",this.info(`Worker ${this.id} is now active`)}return this}async drain(L){if(L){let Y=this.queues.get(L);if(Y)Y.draining=!0,this.info(`Draining queue '${L}'`);else this.warning(`Cannot drain non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.drain(Y);this.workerStatus="draining",this.info(`Worker ${this.id} is now draining`)}return this}async clear(L){if(L){let Y=this.queues.get(L);if(Y){let F=Y.jobs.length;for(let H of Y.jobs)H.job.cancel();Y.jobs=[],this.info(`Cleared ${F} jobs from queue '${L}'`)}else this.warning(`Cannot clear non-existent queue '${L}'`)}else{for(let[Y,F]of this.queues.entries()){let H=F.jobs.length;for(let U of F.jobs)U.job.cancel();F.jobs=[],this.info(`Cleared ${H} jobs from queue '${Y}'`)}if(Array.from(this.queues.values()).every((Y)=>Y.jobs.length===0&&Y.activeJobs.size===0))this.workerStatus="idle"}return this}async size(L=this.defaultQueueName){let Y=this.queues.get(L);if(!Y)throw Error(`Queue '${L}' does not exist`);return Y.jobs.length+Y.activeJobs.size}async totalSize(){let L=0;for(let Y of this.queues.values())L+=Y.jobs.length+Y.activeJobs.size;return L}async getStats(){let L={};for(let[Y,F]of this.queues.entries())L[Y]={waiting:F.jobs.length,retrying:F.retryingJobs.size,processing:F.activeJobs.size,paused:F.paused,draining:F.draining};return L}getInfo(){let L=0;for(let U of this.queues.values())L+=U.activeJobs.size;let Y=Array.from(this.queues.keys()),F=this.defaultTransport?this.defaultTransport.constructor.name:"None",H=async(U,z)=>{try{return await this.addJob(U,z),!0}catch(V){return!1}};return{id:this.id,status:this.workerStatus,queues:Y,activeJobs:L,transport:F,startedAt:this.startedAt,addJob:H,cancelJob:this.cancelJob.bind(this),cancelAllJobs:this.cancelAllJobs.bind(this)}}async cancelJob(L,Y){if(Y)return this.cancelJobInQueue(L,Y);else{for(let F of this.queues.keys())if(await this.cancelJobInQueue(L,F))return!0;return!1}}async cancelAllJobs(L){let Y=0;if(L){let F=this.queues.get(L);if(F){Y=F.jobs.length;for(let H of F.jobs)H.job.cancel();for(let[H,U]of F.activeJobs.entries())if(U.job.cancel())Y++;this.info(`Canceled ${Y} jobs in queue '${L}'`)}}else for(let[F,H]of this.queues.entries()){let U=H.jobs.length;for(let z of H.jobs)z.job.cancel();for(let[z,V]of H.activeJobs.entries())if(V.job.cancel())U++;Y+=U,this.info(`Canceled ${U} jobs in queue '${F}'`)}if(Y>0&&Array.from(this.queues.values()).every((F)=>F.jobs.length===0&&F.activeJobs.size===0))this.workerStatus="idle";return Y}async getJob(L,Y){if(Y){let F=this.queues.get(Y);if(F){let H=F.jobs.find((z)=>z.job.id===L);if(H)return H.job;let U=F.activeJobs.get(L);if(U)return U.job}}else for(let F of this.queues.values()){let H=F.jobs.find((z)=>z.job.id===L);if(H)return H.job;let U=F.activeJobs.get(L);if(U)return U.job}return null}async start(){if(this.workerStatus="active",this.startedAt=new Date,this.defaultTransport)await this.defaultTransport.start();for(let L of this.queues.keys())await this.processQueueJobs(L);this.startScheduledJobTimer(),this.info(`Worker ${this.id} started`)}async stop(L=!1){if(this.workerStatus="stopping",this.stopScheduledJobTimer(),L){await this.cancelAllJobs();for(let Y of this.queues.keys())await this.clear(Y)}else{for(let F of this.queues.keys())await this.drain(F);let Y=()=>{return Array.from(this.queues.values()).every((F)=>F.activeJobs.size===0)};if(!Y()){this.info(`Worker ${this.id} waiting for ${Array.from(this.queues.values()).reduce((F,H)=>F+H.activeJobs.size,0)} jobs to complete`);while(!Y())await new Promise((F)=>setTimeout(F,100))}}if(this.defaultTransport)await this.defaultTransport.stop();this.workerStatus="stopped",this.info(`Worker ${this.id} stopped`)}async setConcurrency(L,Y){let F=this.queues.get(L);if(!F)throw this.error(`Cannot set concurrency: Queue '${L}' does not exist`),Error(`Queue '${L}' does not exist`);if(Y<1)this.warning(`Invalid concurrency value ${Y}, using 1 instead`),Y=1;return F.options.concurrency=Y,this.info(`Set concurrency for queue '${L}' to ${Y}`),this.processQueueJobs(L)}startScheduledJobTimer(){if(this.scheduledJobTimer)clearInterval(this.scheduledJobTimer);this.scheduledJobTimer=setInterval(()=>{this.checkScheduledJobs()},this.scheduledJobInterval)}stopScheduledJobTimer(){if(this.scheduledJobTimer)clearInterval(this.scheduledJobTimer),this.scheduledJobTimer=void 0}checkScheduledJobs(){let L=new Date;for(let[Y,F]of this.queues.entries()){if(F.paused||F.draining)continue;let H=F.jobs.filter((U)=>U.scheduledFor&&U.scheduledFor<=L);for(let U of H)U.scheduledFor=void 0;if(H.length>0&&F.activeJobs.size<F.options.concurrency)this.processQueueJobs(Y)}}async processQueueJobs(L){let Y=this.queues.get(L);if(!Y||Y.paused)return;let F=[];while(F.length<Y.options.concurrency)F.push(this.processNextJob(L));await Promise.all(F);let H=Array.from(this.queues.values());if(H.some((U)=>U.activeJobs.size>0))this.workerStatus="active";else if(H.every((U)=>U.jobs.length===0&&U.activeJobs.size===0))this.workerStatus="idle";if(H.some((U)=>U.jobs.length>0))await Bun.sleep(10),setImmediate(()=>this.processQueueJobs(L))}async sendJobStatusUpdate(L,Y,F,H,U){try{await this.transport.updateJobStatus(L,Y,F,{dispatchId:Y,status:H,error:U,retries:0,startedAt:H==="running"?new Date().toISOString():void 0,completedAt:H==="completed"||H==="failed"||H==="cancelled"?new Date().toISOString():void 0,updatedAt:Date.now().toString()})}catch(z){this.error(`Error sending job status update: ${z}`)}}async cancelJobInQueue(L,Y){let F=this.queues.get(Y);if(!F)return!1;let H=F.jobs.findIndex((U)=>U.job.id===L);if(H>=0){if(F.jobs[H].job.cancel())return F.jobs.splice(H,1),this.info(`Canceled job ${L} in queue '${Y}'`),!0}if(F.activeJobs.has(L)){if(F.activeJobs.get(L).job.cancel())return this.info(`Canceled running job ${L} in queue '${Y}'`),!0}return!1}}class f extends P{isRunning=!1;registeredQueues=new Set;pendingBatches=new Map;batchTimer;metrics={jobsProcessed:0,jobsFailed:0,jobsRetried:0,averageProcessingTime:0};metricsClient;metricsTimer;maxBatchSize;batchInterval;constructor(L="default",Y={}){super(Y.id);if(this.maxBatchSize=Y.maxBatchSize||10,this.batchInterval=Y.batchInterval||100,this.transport=new B("consumer",{connection:L,consumerName:`worker-${this.id}`,batchSize:this.maxBatchSize,...Y}),this.transport.onMessage(this.handleTransportMessage.bind(this)),Y.enableMetrics)this.initializeMetrics(L);this.debug(`RedisWorker ${this.id} created with batch size: ${this.maxBatchSize}`)}async initializeMetrics(L){this.metricsClient=await c.getConnection(L).duplicate()}async reportMetrics(){if(!this.metricsClient||!this.metricsClient.connected)return;let L=`worker:${this.id}:metrics`;try{await this.metricsClient.hmset(L,["jobsProcessed",this.metrics.jobsProcessed.toString(),"jobsFailed",this.metrics.jobsFailed.toString(),"jobsRetried",this.metrics.jobsRetried.toString(),"averageProcessingTime",this.metrics.averageProcessingTime.toString(),"lastProcessedAt",(this.metrics.lastProcessedAt||new Date).toISOString(),"timestamp",Date.now().toString()]),await this.metricsClient.expire(L,3600)}catch(Y){this.error(`Failed to report metrics: ${Y}`)}}async start(){if(this.isRunning)return;await super.start(),this.isRunning=!0;for(let L of this.queues.keys())this.registeredQueues.add(L);if(await this.transport.start(),this.metricsClient)this.metricsTimer=setInterval(()=>{this.reportMetrics()},1e4),await this.metricsClient.connect();await this.transport.registerWorker(this.id,Array.from(this.registeredQueues)),this.startBatchProcessing(),this.info(`Redis worker ${this.id} started for queues: ${Array.from(this.registeredQueues).join(", ")}`)}async stop(L=!1){if(this.isRunning=!1,this.stopBatchProcessing(),!L)await this.processAllPendingBatches();if(await super.stop(L),await this.transport.unregisterWorker(this.id),await this.transport.stop(),this.metricsClient)this.metricsClient.close(),clearInterval(this.metricsTimer),this.metricsClient=void 0;this.info(`Redis worker ${this.id} stopped`)}startBatchProcessing(){this.batchTimer=setInterval(()=>{this.processBatches()},this.batchInterval)}stopBatchProcessing(){if(this.batchTimer)clearInterval(this.batchTimer),this.batchTimer=void 0}async processAllPendingBatches(){for(let[L,Y]of this.pendingBatches)if(Y.jobs.length>0)await this.processBatch(Y);this.pendingBatches.clear()}async processBatches(){let L=Date.now();for(let[Y,F]of this.pendingBatches){if(!this.queues.get(Y))continue;if(F.jobs.length>=this.maxBatchSize||F.jobs.length>0&&L-F.timestamp>this.batchInterval)this.pendingBatches.delete(Y),await this.processBatch(F)}}async processBatch(L){let Y=this.queues.get(L.queue);if(!Y||Y.paused)return;let F=Date.now(),H=new Map,U=[];for(let Z of L.jobs)if(Q(Z.job.constructor)?.overlapBehavior==="no_overlap"){let R=Z.job.id;if(!H.has(R))H.set(R,[]);H.get(R).push(Z)}else U.push(Z);let z=U.map((Z)=>this.processJobInternal(Z,L.queue)),V=[];for(let[Z,_]of H)V.push(this.processOverlapJobs(_,L.queue));await Promise.all([...z,...V]);let X=Date.now()-F;this.updateMetrics(L.jobs.length,X),this.debug(`Processed batch of ${L.jobs.length} jobs from queue '${L.queue}' in ${X}ms`)}async processOverlapJobs(L,Y){for(let F of L)if(await this.transport.acquireJobLock(F.job.id,Y))try{await this.processJobInternal(F,Y)}finally{let U=Q(F.job.constructor);if(U?.overlapDelay&&U.overlapDelay>0)setTimeout(()=>{this.transport.releaseJobLock(F.job.id,Y)},U.overlapDelay);else await this.transport.releaseJobLock(F.job.id,Y)}else{let U=this.queues.get(Y);if(U)U.jobs.push(F)}}async processJobInternal(L,Y){let F=this.queues.get(Y);if(!F)return;F.activeJobs.set(L.job.id,L),await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,Y,"running");try{if(await L.job.run(),L.job.status==="completed")this.metrics.jobsProcessed++,await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,Y,"completed");else if(L.job.status==="failed")await this.handleJobFailure(L,F,Y);else if(L.job.status==="cancelled")await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,Y,"cancelled")}catch(H){this.error(`Error processing job ${L.job.id}: ${H}`),L.job.fail(H instanceof Error?H:Error(String(H))),await this.handleJobFailure(L,F,Y)}finally{F.activeJobs.delete(L.job.id),this.metrics.lastProcessedAt=new Date}}async handleJobFailure(L,Y,F){let H=L.maxRetries!==void 0?L.maxRetries:Y.options.maxRetries;if(L.retries<H){L.retries++,L.job.incrementRetries(),this.metrics.jobsRetried++;let U=L.retryDelay!==void 0?L.retryDelay:Y.options.retryDelay;await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,F,"scheduled_for_retry",L.job.lastError?.message),Y.retryingJobs.set(L.job.id,L),setTimeout(()=>{if(L.job.status!=="cancelled")this.addJobToBatch(L,F);Y.retryingJobs.delete(L.job.id)},U)}else if(this.metrics.jobsFailed++,await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,F,"failed",L.job.lastError?.message),Y.options.pauseOnError)await this.pause(F)}addJobToBatch(L,Y){if(!this.pendingBatches.has(Y))this.pendingBatches.set(Y,{queue:Y,jobs:[],timestamp:Date.now()});let F=this.pendingBatches.get(Y);if(F.jobs.push(L),F.jobs.length>=this.maxBatchSize)this.pendingBatches.delete(Y),this.processBatch(F)}async processNextJob(L){let Y=this.queues.get(L);if(!Y||Y.paused||Y.activeJobs.size>=Y.options.concurrency)return;let F=new Date,H=[];for(let U=0;U<Y.jobs.length&&H.length<this.maxBatchSize;U++){let z=Y.jobs[U];if(z.scheduledFor&&z.scheduledFor>F)continue;if(z.job.status==="cancelled"){Y.jobs.splice(U,1),U--;continue}H.push(z)}for(let U of H){let z=Y.jobs.indexOf(U);if(z>=0)Y.jobs.splice(z,1)}if(H.length>0){let U={queue:L,jobs:H,timestamp:Date.now()};if(H.length===1)await this.processBatch(U);else this.pendingBatches.set(L,U)}}async handleTransportMessage(L){try{switch(L.type){case"job:process":{let{job:Y,args:F=[],queue:H="default",options:U={},jobId:z,dispatchId:V}=L;await this.handleProcessJob(z,V,Y,F,H,U);break}case"job:cancel":{let{jobId:Y,queue:F="default"}=L;if(await this.cancelJob(Y,F))await this.sendJobStatusUpdate(Y,L.dispatchId,F,"cancelled");break}case"job:cancelAll":{let{queue:Y="default"}=L;await this.cancelAllJobs(Y);break}case"job:update":{if(L.status==="lock_released"){let Y=L.queue;await this.processQueueJobs(Y)}break}}}catch(Y){this.error(`Error handling transport message: ${Y}`)}}async handleProcessJob(L,Y,F,H,U,z){try{let V=j.get(F);if(!V)throw Error(`Job class not found: ${F}`);let X=new(C(V,L,Y))(...H||[]);X.queueName=U;let Z={};if(z.scheduledFor)Z.scheduledFor=typeof z.scheduledFor==="string"?new Date(z.scheduledFor):z.scheduledFor;if(z.priority!==void 0)Z.priority=Number(z.priority);if(z.maxRetries!==void 0)Z.maxRetries=Number(z.maxRetries);if(z.retryDelay!==void 0)Z.retryDelay=Number(z.retryDelay);let _=Z.scheduledFor&&Z.scheduledFor>new Date?"scheduled_for_retry":"pending";await this.sendJobStatusUpdate(X.id,X.dispatchId,U,_),await this.addJob(X,U,Z)}catch(V){this.error(`Error processing job ${F}: ${V.message}`),await this.sendJobStatusUpdate(L,Y,U,"failed",V.message)}}updateMetrics(L,Y){let F=this.metrics.averageProcessingTime,H=this.metrics.jobsProcessed+this.metrics.jobsFailed;this.metrics.averageProcessingTime=(F*H+Y)/(H+L)}async createQueue(L){let Y=await super.createQueue(L);if(this.isRunning&&!this.registeredQueues.has(L.name))this.registeredQueues.add(L.name),await this.transport.registerWorker(this.id,Array.from(this.registeredQueues));return Y}}import{uid as i}from"radash";import{Event as D,InteractsWithConsole as p}from"@elysiumjs/core";import{map as g,uid as n}from"radash";class x extends p{static queues=new Map;static exists(L){return this.queues.has(L)}static get(L,Y){if(!this.queues.has(L)){let F=new x({name:L,...Y});this.queues.set(L,F)}return this.queues.get(L)}static getAll(){return new Map(this.queues)}name;transport;transportClass;transportOptions;options;jobs=[];constructor(L){super();this.name=L.name,this.transportClass=L.transport,this.transportOptions=L.transportOptions,this.options={concurrency:L.concurrency??1,maxRetries:L.maxRetries??0,retryDelay:L.retryDelay??1000,pauseOnError:L.pauseOnError??!1,...L},this.transport=new this.transportClass("producer",this.transportOptions),this.transport.onMessage(this.handleTransportMessage.bind(this))}on(L,Y){D.on(`elysium:heracles:queue:${this.name}:${L}`,Y)}once(L,Y){D.once(`elysium:heracles:queue:${this.name}:${L}`,Y)}emit(L,Y,F=null){D.emit(`elysium:heracles:queue:${this.name}:${L}`,Y,F)}async handleTransportMessage(L){switch(L.type){case"job:status":case"job:result":if(L.queue===this.name){if(L.type==="job:result")this.info(`Job ${L.jobId} completed with status: ${L.status}`);let Y=await this.getJobStatus({jobId:L.jobId,dispatchId:L.dispatchId}),F=Y?.createdAt??new Date().toISOString();this.updateJobStatus(L.jobId,L.dispatchId,{jobId:L.jobId,dispatchId:L.dispatchId,queue:L.queue,status:L.status,error:L.error,retries:L.retries??Y?.retries??0,createdAt:F,startedAt:L.startedAt??Y?.startedAt,completedAt:L.completedAt??Y?.completedAt,updatedAt:L.updatedAt??new Date().toISOString()})}break}this.emit(L.type,L,this)}updateJobStatus(L,Y,F){if(F.status==="completed"||F.status==="failed"||F.status==="cancelled")this.debug(`Job ${L} (${Y}) reached terminal state: ${F.status}`)}async dispatch(L,Y,F=[],H={}){let U=new Date,z=U.toISOString(),V=H.jobId??L.generateJobId(...F),X=H.dispatchId??`dispatch_${Date.now()}_${n(8)}`,Z={jobId:V,dispatchId:X,queue:this.name,status:H.scheduledFor&&H.scheduledFor>U?"scheduled_for_retry":"pending",retries:0,createdAt:z,updatedAt:z};this.updateJobStatus(V,X,Z),this.debug(`Dispatching job ${V} (dispatch: ${X}, name: ${Y}) to queue '${this.name}'`);try{let _={type:"job:process",job:Y,jobId:V,dispatchId:X,args:F,queue:this.name,options:{scheduledFor:H.scheduledFor?.toISOString(),priority:H.priority,maxRetries:H.maxRetries,retryDelay:H.retryDelay,overlapBehavior:H.overlapBehavior,overlapDelay:H.overlapDelay}};return await this.transport.send(_),{jobId:Z.jobId,dispatchId:X}}catch(_){throw this.error(`Failed to dispatch job ${Y} to queue '${this.name}': ${_.message}`),this.updateJobStatus(V,X,{jobId:Z.jobId,dispatchId:X,queue:this.name,status:"failed",error:_.message,retries:0,createdAt:Z.createdAt,completedAt:new Date().toISOString(),updatedAt:new Date().toISOString()}),_}}async cancelJob(L){this.debug(`Canceling job ${L.jobId} (${L.dispatchId}) in queue '${this.name}'`);try{let Y={type:"job:cancel",jobId:L.jobId,dispatchId:L.dispatchId,queue:this.name};return await this.transport.send(Y),!0}catch(Y){return this.error(`Failed to cancel job ${L.jobId}: ${Y}`),!1}}async cancelAllJobs(){this.debug(`Canceling all jobs in queue '${this.name}'`);try{let L={type:"job:cancelAll",queue:this.name};await this.transport.send(L)}catch(L){this.error(`Failed to cancel all jobs: ${L}`)}return 0}async getJobStatus(L){try{let Y=await this.transport.getJobStatus(L.jobId,L.dispatchId,this.name);if(Y)return this.debug(`Retrieved job ${L.jobId} (${L.dispatchId}) status from transport: ${Y.status}`),Y}catch(Y){this.warning(`Failed to get job status from transport: ${Y}`)}return null}getAllJobs(){return Array.from(this.jobs)}async getJobsByStatus(L){let Y=[];for(let[F,H]of await g(this.jobs,async(U)=>[U,await this.getJobStatus(U)]))if(H?.status===L)Y.push(F);return Y}async start(){if(this.transport)await this.transport.start(),this.info(`Queue '${this.name}' transport started`)}async stop(){if(this.transport)await this.transport.stop(),this.info(`Queue '${this.name}' transport stopped`)}getOptions(){return{...this.options}}updateOptions(L){this.options={...this.options,...L}}}var y;((k)=>k.dispatch=async(Y,F=[],H={})=>{let U=Q(Y);if(!U)throw Error(`Job ${Y.name} is not marked with @Job.register() decorator`);if(!U.queue)throw Error(`Job ${Y.name} is not registered with a queue`);if(!x.exists(U.queue))throw Error(`Queue ${U.queue} does not exist. Please create the queue before dispatching the job.`);let z=x.get(U.queue),X={dispatchId:`dispatch_${Date.now()}_${i(8)}`,priority:U.priority,maxRetries:U.maxRetries,retryDelay:U.retryDelay,overlapBehavior:U.overlapBehavior,overlapDelay:U.overlapDelay,...H};return z.dispatch(Y,U.name,F,X)})(y||={});export{v as WorkerStatus,K as TransportMode,f as RedisWorker,B as RedisTransport,x as Queue,A as JobStatus,G as JobOverlapBehavior,O as Job,y as Heracles,P as BaseWorker};

package/dist/job.d.ts

@@ -197,6 +197,11 @@ export declare abstract class Job extends InteractsWithConsole {
      * @returns A promise that resolves when the job completes (successfully or not).
      */
     run(): Promise<void>;
+    /**
+     * Marks the job as failed.
+     * @param error The error that caused the job to fail.
+     */
+    fail(error: Error): void;
     /**
      * Cancels the job.
      * @returns True if the job was cancelled, false if it was already in a terminal state.

package/dist/transports/redis.transport.d.ts

@@ -1,13 +1,13 @@
-import type { JobStatusInfo, Transport, TransportEvent, TransportMode } from '../transport';
+import type { JobStatusInfo, Transport, TransportEvent } from '../transport';
 import { InteractsWithConsole } from '@elysiumjs/core';
+import { TransportMode } from '../transport';
 /**
  * Redis transport configuration options
  * @author Axel Nana <axel.nana@workbud.com>
  */
 export type RedisTransportOptions = {
     /**
-     * Redis connection name.
-     * If not provided, uses the default connection.
+     * Redis connection name from Elysium config.
      */
     connection?: string;
     /**
@@ -25,11 +25,6 @@ export type RedisTransportOptions = {
      * If not provided, a random name will be generated.
      */
     consumerName?: string;
-    /**
-     * How often to poll Redis for new messages (in milliseconds).
-     * @default 1000
-     */
-    pollInterval?: number;
     /**
      * Maximum number of messages to read in a single poll.
      * @default 10
@@ -57,76 +52,62 @@ export type RedisTransportOptions = {
     maxStreamSize?: number;
 };
 /**
- * Transport implementation using Redis streams for distributed job processing.
+ * Transport implementation using Redis streams and pub/sub for distributed job processing.
  * @author Axel Nana <axel.nana@workbud.com>
  */
 export declare class RedisTransport extends InteractsWithConsole implements Transport {
     /**
-     * Redis client instance.
-     */
-    private client;
-    /**
-     * Options for this transport.
+     * Main Redis client for commands
      */
-    private options;
+    private commandClient;
     /**
-     * Mode this transport is operating in.
+     * Dedicated client for pub/sub subscriptions
      */
-    private mode;
+    private subscriberClient;
     /**
-     * Registered message handlers.
+     * Dedicated client for publishing messages
      */
-    private messageHandlers;
+    private publisherClient;
     /**
-     * Timer for polling Redis in consumer mode.
+     * Options for this transport
      */
-    private pollTimer?;
-    /**
-     * Timer for stream cleanup.
-     */
-    private cleanupTimer?;
+    private options;
     /**
-     * Last processed message IDs by stream.
+     * Mode this transport is operating in
      */
-    private lastIds;
+    private mode;
     /**
-     * Whether to check the logs instead of new messages.
+     * Registered message handlers
      */
-    private checkLogs;
+    private messageHandlers;
     /**
-     * Consumer ID for this instance.
+     * Consumer ID for this instance
      */
     private consumerId;
     /**
-     * Whether the consumer group has been created for each stream.
+     * Whether the transport is currently running
      */
-    private consumerGroupCreated;
+    private isRunning;
     /**
-     * Map of message IDs to completed status for tracking completed jobs.
+     * Timer for cleanup operations
      */
-    private completedMessageIds;
+    private cleanupTimer?;
     /**
-     * Map to track locks on job IDs for NO_OVERLAP jobs.
+     * Map to track job locks for NO_OVERLAP jobs
      */
     private jobLocks;
     /**
-     * Create a new Redis transport.
+     * Create a new Redis transport
      */
     constructor(mode: TransportMode, options?: RedisTransportOptions);
     /**
-     * Helper method to execute Redis commands with proper type conversion
-     * @param command Redis command
-     * @param args Command arguments
-     * @returns Redis response
+     * Initialize Redis clients with proper configuration
      */
-    private redisCommand;
+    private initializeClients;
     /**
-     * Finds job message ID in the stream by job ID
-     * @param jobId The job ID to find
-     * @param queueName The queue to search in
-     * @returns The message ID if found, null otherwise
+     * Setup connection event handlers for all clients
      */
-    private findJobMessageId;
+    private setupConnectionHandlers;
     /**
      * Start the transport
      */
@@ -135,6 +116,22 @@ export declare class RedisTransport extends InteractsWithConsole implements Tran
      * Stop the transport
      */
     stop(): Promise<void>;
+    /**
+     * Subscribe to relevant pub/sub channels
+     */
+    private subscribeToChannels;
+    /**
+     * Process a job from the stream when notified via pub/sub
+     */
+    private processJobFromStream;
+    /**
+     * Handle control messages
+     */
+    private handleControlMessage;
+    /**
+     * Handle worker coordination events
+     */
+    private handleWorkerCoordination;
     /**
      * Send a message through the transport
      */
@@ -144,15 +141,11 @@ export declare class RedisTransport extends InteractsWithConsole implements Tran
      */
     onMessage(handler: (message: TransportEvent) => void | Promise<void>): void;
     /**
-     * Gets the current status of a job.
+     * Gets the current status of a job
      */
     getJobStatus(jobId: string, dispatchId: string, queueName: string): Promise<JobStatusInfo>;
     /**
-     * Updates a job's status directly in Redis without creating a new message.
-     *
-     * @param jobId The ID of the job to update
-     * @param queueName The queue the job belongs to
-     * @param updates The status updates to apply
+     * Updates a job's status
      */
     updateJobStatus(jobId: string, dispatchId: string, queueName: string, updates: Partial<JobStatusInfo>): Promise<void>;
     /**
@@ -164,88 +157,75 @@ export declare class RedisTransport extends InteractsWithConsole implements Tran
      */
     unregisterWorker(workerId: string): Promise<void>;
     /**
-     * Start polling Redis for messages
+     * Index a job for efficient lookups
      */
-    private startPolling;
+    private indexJob;
     /**
-     * Poll for new messages
+     * Update job index when status changes
      */
-    private pollMessages;
+    private updateJobIndex;
     /**
-     * Process messages from Redis stream
+     * Get job from index
      */
-    private processMessages;
+    private getJobFromIndex;
     /**
-     * Get all registered queues
-     */
-    private getRegisteredQueues;
-    /**
-     * Ensure a stream exists
+     * Store job status in Redis
      */
-    private ensureStreamExists;
+    private storeJobStatus;
     /**
-     * Ensure a consumer group exists for a stream
+     * Ensure consumer group exists for a stream
      */
     private ensureConsumerGroupExists;
     /**
-     * Store job status in Redis
+     * Get all registered queues
      */
-    private storeJobStatus;
+    private getRegisteredQueues;
     /**
-     * Serialize a message for Redis
+     * Schedule periodic cleanup operations
      */
-    private serializeMessage;
+    private scheduleCleanup;
     /**
-     * Deserialize a message from Redis
+     * Perform cleanup of old jobs and indices
      */
-    private deserializeMessage;
+    private performCleanup;
     /**
-     * Get Redis key for a stream
+     * Extract queue name from a transport event
      */
-    private getStreamKey;
+    private extractQueueName;
     /**
-     * Get Redis key for job status
+     * Serialize a message for Redis storage
      */
-    private getJobStatusKey;
+    private serializeMessage;
     /**
-     * Get Redis key for worker status
+     * Deserialize a message from Redis
      */
-    private getWorkerKey;
+    private deserializeMessage;
     /**
-     * Track completed message for later cleanup
+     * Notify handlers about lock release
      */
-    private trackCompletedMessage;
+    private notifyLockRelease;
     /**
-     * Schedule periodic stream cleanup
+     * Check if a job is locked
      */
-    private scheduleStreamCleanup;
+    isJobLocked(jobId: string, queue: string): Promise<boolean>;
     /**
-     * Find expired message IDs for cleanup
+     * Acquire a lock for a job
      */
-    private cleanupStreams;
+    acquireJobLock(jobId: string, queue: string, duration?: number): Promise<boolean>;
     /**
-     * Trim all streams to the maximum configured size
+     * Release a job lock
      */
-    private trimStreamsToMaxSize;
+    releaseJobLock(jobId: string, queue: string): Promise<void>;
     /**
-     * Check if a job with the same ID is currently locked
-     * @param jobId The job ID to check
-     * @param queue The queue name
-     * @returns True if the job is locked, false otherwise
+     * Get Redis key for a stream
      */
-    isJobLocked(jobId: string, queue: string): Promise<boolean>;
+    private getStreamKey;
     /**
-     * Acquire a lock for a job ID
-     * @param jobId The job ID to lock
-     * @param queue The queue name
-     * @param duration Duration in milliseconds to lock the job
-     * @returns True if lock was acquired, false otherwise
+     * Get Redis key for job status
      */
-    acquireJobLock(jobId: string, queue: string, duration?: number): Promise<boolean>;
+    private getJobStatusKey;
     /**
-     * Release a lock for a job ID
-     * @param jobId The job ID to unlock
-     * @param queue The queue name
+     * Get Redis key for worker status
      */
-    releaseJobLock(jobId: string, queue: string): Promise<void>;
+    private getWorkerKey;
 }

package/dist/utils/performance-monitor.d.ts

@@ -0,0 +1,172 @@
+import { InteractsWithConsole } from '@elysiumjs/core';
+/**
+ * Histogram for tracking distributions
+ */
+declare class Histogram {
+    private values;
+    private sorted;
+    /**
+     * Add a value to the histogram
+     */
+    record(value: number): void;
+    /**
+     * Get percentile value
+     */
+    percentile(p: number): number;
+    /**
+     * Get average value
+     */
+    average(): number;
+    /**
+     * Get minimum value
+     */
+    min(): number;
+    /**
+     * Get maximum value
+     */
+    max(): number;
+    /**
+     * Clear the histogram
+     */
+    clear(): void;
+    /**
+     * Get the count of values
+     */
+    count(): number;
+}
+/**
+ * Counter for tracking counts
+ */
+declare class Counter {
+    private value;
+    private lastReset;
+    /**
+     * Increment the counter
+     */
+    increment(delta?: number): void;
+    /**
+     * Get the current value
+     */
+    get(): number;
+    /**
+     * Get the rate per second
+     */
+    rate(): number;
+    /**
+     * Reset the counter
+     */
+    reset(): void;
+}
+/**
+ * Gauge for tracking current values
+ */
+declare class Gauge {
+    private value;
+    /**
+     * Set the gauge value
+     */
+    set(value: number): void;
+    /**
+     * Get the current value
+     */
+    get(): number;
+    /**
+     * Increment the gauge
+     */
+    increment(delta?: number): void;
+    /**
+     * Decrement the gauge
+     */
+    decrement(delta?: number): void;
+}
+/**
+ * Performance monitoring system for Heracles
+ * @author Axel Nana <axel.nana@workbud.com>
+ */
+export declare class PerformanceMonitor extends InteractsWithConsole {
+    private client?;
+    private metrics;
+    private timers;
+    private reportingInterval?;
+    private keyPrefix;
+    /**
+     * Job processing metrics
+     */
+    readonly jobProcessingTime: Histogram;
+    readonly jobQueueTime: Histogram;
+    readonly jobRetryCount: Counter;
+    readonly jobSuccessCount: Counter;
+    readonly jobFailureCount: Counter;
+    /**
+     * Queue metrics
+     */
+    readonly queueDepth: Gauge;
+    readonly queueThroughput: Counter;
+    readonly queueBacklog: Gauge;
+    /**
+     * Redis metrics
+     */
+    readonly redisLatency: Histogram;
+    readonly redisCommands: Counter;
+    readonly redisErrors: Counter;
+    /**
+     * Worker metrics
+     */
+    readonly workerUtilization: Gauge;
+    readonly workerConcurrency: Gauge;
+    readonly workerMemoryUsage: Gauge;
+    constructor(options?: {
+        redisUrl?: string;
+        keyPrefix?: string;
+        enableReporting?: boolean;
+        reportingInterval?: number;
+    });
+    /**
+     * Initialize Redis reporting
+     */
+    private initializeReporting;
+    /**
+     * Start a timer
+     */
+    startTimer(name: string): void;
+    /**
+     * End a timer and record the duration
+     */
+    endTimer(name: string, metric?: Histogram): number;
+    /**
+     * Record a job execution
+     */
+    recordJobExecution(_jobId: string, _queue: string, duration: number, success: boolean, retries?: number): void;
+    /**
+     * Record Redis operation
+     */
+    recordRedisOperation(_command: string, duration: number, error?: Error): void;
+    /**
+     * Update worker metrics
+     */
+    updateWorkerMetrics(utilization: number, concurrency: number): void;
+    /**
+     * Update queue metrics
+     */
+    updateQueueMetrics(depth: number, backlog: number): void;
+    /**
+     * Get current metrics summary
+     */
+    getSummary(): Record<string, any>;
+    /**
+     * Report metrics to Redis
+     */
+    private reportMetrics;
+    /**
+     * Get historical metrics
+     */
+    getHistoricalMetrics(metric: string, start: number, end?: number): Promise<Array<{
+        timestamp: number;
+        value: number;
+    }>>;
+    /**
+     * Clean up and stop reporting
+     */
+    shutdown(): Promise<void>;
+}
+export {};

package/dist/worker.d.ts

@@ -77,9 +77,9 @@ export type QueueState = {
      */
     activeJobs: Map<string, QueuedJob>;
     /**
-     * The number of jobs currently being processed.
+     * The jobs that are waiting to be retried.
      */
-    processing: number;
+    retryingJobs: Map<string, QueuedJob>;
     /**
      * Whether the queue is currently paused.
      */

package/dist/workers/base.worker.d.ts

@@ -122,6 +122,7 @@ export declare abstract class BaseWorker<TTransport extends Transport> extends I
      */
     getStats(): Promise<Record<string, {
         waiting: number;
+        retrying: number;
         processing: number;
         paused: boolean;
         draining: boolean;

package/dist/workers/redis.worker.d.ts

@@ -15,14 +15,56 @@ export declare class RedisWorker extends BaseWorker<RedisTransport> {
      * Queues registered with this worker
      */
     private registeredQueues;
+    /**
+     * Job batches pending processing
+     */
+    private pendingBatches;
+    /**
+     * Batch processing timer
+     */
+    private batchTimer?;
+    /**
+     * Worker metrics
+     */
+    private metrics;
+    /**
+     * Metrics reporting client
+     */
+    private metricsClient?;
+    /**
+     * Metrics reporting timer
+     */
+    private metricsTimer?;
+    /**
+     * Maximum batch size for processing
+     */
+    private readonly maxBatchSize;
+    /**
+     * Batch collection interval in milliseconds
+     */
+    private readonly batchInterval;
     /**
      * Creates a new RedisWorker instance
      * @param connectionName Name of the Redis connection to use
-     * @param options Additional options for the Redis transport
+     * @param options Additional options
      */
-    constructor(connectionName?: string, options?: Record<string, any>);
+    constructor(connectionName?: string, options?: {
+        id?: string;
+        maxBatchSize?: number;
+        batchInterval?: number;
+        enableMetrics?: boolean;
+        [key: string]: any;
+    });
     /**
-     * Start the worker and begin processing jobs
+     * Initialize metrics reporting
+     */
+    private initializeMetrics;
+    /**
+     * Report worker metrics
+     */
+    private reportMetrics;
+    /**
+     * Start the worker
      */
     start(): Promise<void>;
     /**
@@ -31,26 +73,59 @@ export declare class RedisWorker extends BaseWorker<RedisTransport> {
      */
     stop(force?: boolean): Promise<void>;
     /**
-     * Process the next job in a queue, respecting overlap settings
-     * @param queueName Name of the queue
+     * Start batch processing timer
      */
-    protected processNextJob(queueName: string): Promise<void>;
+    private startBatchProcessing;
     /**
-     * Create a new queue with the specified options
-     * @param options Queue options
+     * Stop batch processing timer
      */
-    createQueue(options: WorkerQueueOptions): Promise<Worker>;
+    private stopBatchProcessing;
+    /**
+     * Process all pending batches
+     */
+    private processAllPendingBatches;
+    /**
+     * Process batches that are ready
+     */
+    private processBatches;
+    /**
+     * Process a batch of jobs
+     */
+    private processBatch;
+    /**
+     * Process NO_OVERLAP jobs sequentially
+     */
+    private processOverlapJobs;
+    /**
+     * Process a single job internally
+     */
+    private processJobInternal;
+    /**
+     * Handle job failure with retry logic
+     */
+    private handleJobFailure;
     /**
-     * Handle a message from the transport
-     * @param message The message to handle
+     * Add a job to the batch for processing
+     */
+    private addJobToBatch;
+    /**
+     * Process the next job in a queue (overridden for batching)
+     */
+    protected processNextJob(queueName: string): Promise<void>;
+    /**
+     * Handle transport messages
      */
     private handleTransportMessage;
     /**
-     * Handle a job:process message
-     * @param jobName Job class name
-     * @param args Job constructor arguments
-     * @param queueName Queue name
-     * @param options Job options
+     * Handle job process message
      */
     private handleProcessJob;
+    /**
+     * Update worker metrics
+     */
+    private updateMetrics;
+    /**
+     * Create a new queue with the specified options
+     */
+    createQueue(options: WorkerQueueOptions): Promise<Worker>;
 }

package/package.json

@@ -1,45 +1,48 @@
 {
-	"name": "@elysiumjs/heracles",
-	"version": "0.5.1",
-	"description": "Background job processing system for Elysium.js apps.",
-	"module": "dist/index.js",
-	"type": "module",
-	"types": "dist/index.d.ts",
-	"scripts": {
-		"test": "bun test",
-		"dev": "bun --watch src/index.ts",
-		"format": "prettier --write 'src/**/*.{ts,tsx}'",
-		"clean": "rimraf dist",
-		"build:js": "bun build ./src/index.ts --outdir ./dist --target bun --packages external --root ./src --minify",
-		"build:ts": "tsc",
-		"build": "bun run clean && bun run build:js && bun run build:ts",
-		"prepublishOnly": "bun run build"
-	},
-	"keywords": [
-		"elysium",
-		"job",
-		"queue",
-		"worker",
-		"scheduler",
-		"background",
-		"processing",
-		"heracles"
-	],
-	"author": "Workbud Technologies Inc.",
-	"license": "Apache-2.0",
-	"dependencies": {
-		"commander": "^13.1.0",
-		"radash": "^12.1.0",
-		"reflect-metadata": "^0.2.2"
-	},
-	"devDependencies": {
-		"bun-types": "^1.2.15",
-		"prettier": "^3.5.3",
-		"rimraf": "^6.0.1",
-		"type-fest": "^4.41.0"
-	},
-	"peerDependencies": {
-		"@elysiumjs/core": "latest",
-		"typescript": "^5.8.3"
-	}
+  "name": "@elysiumjs/heracles",
+  "version": "0.6.1",
+  "description": "Background job processing system for Elysium.js apps.",
+  "module": "dist/index.js",
+  "type": "module",
+  "types": "dist/index.d.ts",
+  "sideEffects": [
+    "./src/index.ts"
+  ],
+  "scripts": {
+    "test": "bun test",
+    "dev": "bun --watch src/index.ts",
+    "format": "prettier --write 'src/**/*.{ts,tsx}'",
+    "clean": "rimraf dist",
+    "build:js": "bun build ./src/index.ts --outdir ./dist --target bun --packages external --root ./src --minify",
+    "build:ts": "tsc",
+    "build": "bun run clean && bun run build:js && bun run build:ts",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "bun run build",
+    "example:optimized": "bun run examples/optimized_example.ts",
+    "example:worker": "bun run examples/redis_worker_example.ts",
+    "example:comprehensive": "bun run examples/heracles_comprehensive.ts"
+  },
+  "keywords": [
+    "elysium",
+    "job",
+    "queue",
+    "worker",
+    "scheduler",
+    "background",
+    "processing",
+    "heracles"
+  ],
+  "author": "Workbud Technologies Inc.",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "commander": "^14.0.1",
+    "radash": "^12.1.1",
+    "reflect-metadata": "^0.2.2"
+  },
+  "devDependencies": {
+    "@elysiumjs/core": "0.6.1"
+  },
+  "peerDependencies": {
+    "@elysiumjs/core": "^0.6.1"
+  }
 }

package/tsconfig.json

@@ -1,48 +1,12 @@
-// Copyright (c) 2025-present Workbud Technologies Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 {
+	"extends": "../../tsconfig.base.json",
 	"compilerOptions": {
-		// Environment setup & latest features
-		"target": "ESNext",
-		"module": "ESNext",
-		"moduleDetection": "force",
-		"allowJs": true,
-		"types": ["bun-types", "type-fest"],
-		"experimentalDecorators": true,
-		"emitDecoratorMetadata": true,
-		// Compiler setup
 		"outDir": "dist",
 		"declaration": true,
 		"emitDeclarationOnly": true,
 		"declarationMap": false,
 		"declarationDir": "dist",
-		"rootDir": "src",
-		// Bundler mode
-		"moduleResolution": "bundler",
-		"allowImportingTsExtensions": true,
-		"verbatimModuleSyntax": true,
-		// Best practices
-		"strict": true,
-		"skipLibCheck": true,
-		"noFallthroughCasesInSwitch": true,
-		"noUncheckedIndexedAccess": false,
-		"forceConsistentCasingInFileNames": true,
-		// Some stricter flags (disabled by default)
-		"noUnusedLocals": true,
-		"noUnusedParameters": true,
-		"noPropertyAccessFromIndexSignature": false
+		"rootDir": "src"
 	},
-	"exclude": ["node_modules", "dist"],
 	"include": ["src"]
 }