@elysiumjs/heracles 0.4.4 → 0.5.1

package/README.md

@@ -8,11 +8,9 @@ Heracles is a powerful background job processing system for Elysium.js applicati
 
 - **Job queues**: Organize jobs into different queues with custom settings
 - **Transports**: Abstract communication layer between queues and workers
-- **Worker pool**: Efficiently coordinate workers across multiple processes or servers
 - **Scheduling**: Schedule jobs to run at specific times
 - **Retries**: Automatically retry failed jobs with configurable backoff
 - **Prioritization**: Assign priorities to jobs within a queue
-- **CLI tool**: Manage workers through a simple command-line interface
 
 ## Installation
 
@@ -43,12 +41,12 @@ export class SendEmailJob extends Job {
 
 	protected async execute(): Promise<void> {
 		// Implementation for sending an email
-		console.log(`Sending email to ${this.email}: ${this.subject}`);
+		this.info(`Sending email to ${this.email}: ${this.subject}`);
 
 		// Simulate email sending
-		await new Promise((resolve) => setTimeout(resolve, 1000));
+		await Bun.sleep(1000);
 
-		console.log(`Email sent successfully to ${this.email}`);
+		this.success(`Email sent successfully to ${this.email}`);
 	}
 }
 ```
@@ -56,7 +54,7 @@ export class SendEmailJob extends Job {
 ### 2. Dispatch a Job
 
 ```typescript
-import { Queue, RedisTransport } from '@elysiumjs/heracles';
+import { Heracles, Queue, RedisTransport } from '@elysiumjs/heracles';
 
 import { SendEmailJob } from './jobs/send-email.job';
 
@@ -69,14 +67,10 @@ const emailQueue = Queue.get('emails', {
 });
 
 // Dispatch a job
-await emailQueue.dispatch(SendEmailJob, [
-	'user@example.com',
-	'Welcome!',
-	'Welcome to our platform!'
-]);
+await Heracles.dispatch(SendEmailJob, ['user@example.com', 'Welcome!', 'Welcome to our platform!']);
 
 // Dispatch with options
-await emailQueue.dispatch(
+await Heracles.dispatch(
 	SendEmailJob,
 	['user@example.com', 'Welcome!', 'Welcome to our platform!'],
 	{
@@ -89,14 +83,14 @@ await emailQueue.dispatch(
 
 ### 3. Start Workers
 
-Using the CLI:
+You can use [`styx`](https://www.npmjs.com/package/@elysiumjs/styx) to launch Heracles workers through the CLI:
 
 ```bash
 # Start a Redis worker for the 'emails' queue with concurrency of 3 jobs
-bun heracles work --id worker-1 --queues=emails --concurrency=3 --redis=default
+bun styx heracles:work --id worker-1 --queues emails --concurrency 3 --redis default
 
 # Start workers for multiple queues
-bun heracles work --id worker-2 --queues=emails,notifications --redis=default
+bun styx heracles:work --id worker-2 --queues emails,notifications --redis default
 ```
 
 Or programmatically:
@@ -140,7 +134,7 @@ const queue = Queue.get('emails', {
 
 ### Redis Transport Configuration
 
-Heracles uses Redis as its transport layer for job distribution, enabling distributed processing across multiple servers or processes. It uses Redis Streams for reliable message delivery and job coordination.
+Heracles has a bundled Redis transport layer for job distribution, enabling distributed processing across multiple servers or processes. It uses Redis Streams for reliable message delivery and job coordination.
 
 #### 1. Configure Redis Connection
 
@@ -156,6 +150,23 @@ Redis.registerConnection('default', {
 });
 ```
 
+Or during the Application registration:
+
+```typescript
+import { Application } from '@elysiumjs/core';
+
+@Application.register({
+	redis: {
+		default: 'jobs',
+		connections: {
+			jobs: { url: 'redis://localhost:6379' }
+		}
+	}
+	// ...other configurations...
+})
+export class App extends Application {}
+```
+
 #### 2. Configure Queues with Redis Transport
 
 ```typescript
@@ -167,7 +178,6 @@ const queue = Queue.get('emails', {
 		connection: 'default', // Redis connection name
 		keyPrefix: 'myapp:jobs', // Prefix for Redis keys (default: 'elysium:heracles')
 		consumerGroup: 'workers', // Redis consumer group name (default: 'workers')
-		pollInterval: 1000, // Poll interval in ms (default: 1000)
 		statusTTL: 86400, // Job status TTL in seconds (default: 86400 - 24 hours)
 		cleanupCompletedJobs: true, // Auto-cleanup completed jobs (default: true)
 		completedJobRetention: 3600, // Time to retain completed jobs in seconds (default: 3600 - 1 hour)
@@ -186,8 +196,11 @@ import { RedisWorker } from '@elysiumjs/heracles';
 // Create a Redis worker with a unique ID
 const worker = new RedisWorker('default', {
 	id: 'worker-1', // Unique worker ID
+	consumerGroup: 'workers', // Redis consumer group name (default: 'workers')
 	consumerName: 'worker-1', // Unique consumer name (optional)
-	keyPrefix: 'myapp:jobs' // Use same prefix as queue configuration
+	keyPrefix: 'myapp:jobs', // Use same prefix as queue configuration
+	pollInterval: 1000, // Poll interval in ms (default: 1000)
+	batchSize: 10 // Batch size for processing jobs (default: 10)
 });
 
 // Configure queue settings
@@ -220,7 +233,8 @@ Jobs in Heracles go through several statuses:
 
 ```typescript
 // Cancel a specific job
-const cancelled = await queue.cancelJob('job_123456789');
+const jobId = await Heracles.dispatch(HelloWorldJob);
+const cancelled = await queue.cancelJob(jobId);
 
 // Cancel all jobs in a queue
 const cancelledCount = await queue.cancelAllJobs();
@@ -242,78 +256,72 @@ await queue.drain();
 await queue.clear();
 ```
 
-### Worker Pool
-
-The worker pool manages worker availability and distributes jobs using a round-robin algorithm:
-
-```typescript
-import { ThreadWorker, WorkerPool } from '@elysiumjs/heracles';
+## CLI Reference
 
-// Get the worker pool instance
-const pool = WorkerPool.instance;
+Heracles commands are available via the [`styx`](https://www.npmjs.com/package/@elysiumjs/styx) CLI tool:
 
-// Add a worker to the pool
-const worker = new ThreadWorker(['emails']);
-await worker.start();
-await pool.addWorker(worker.getInfo(), ['emails']);
+### Start Workers
 
-// Get workers for a specific queue
-const emailWorkers = pool.getWorkersForQueue('emails');
 ```
+Usage: styx heracles:work [options]
+Description: Start an Heracles worker for specific queues.
 
-## CLI Reference
+Arguments:
+ --id <string> (required)
+    Unique identifier for this worker
 
-Heracles includes a CLI tool for managing workers and Redis maintenance:
+ --queues <array of strings> (required) [default: default]
+    List of queues to work on
 
-```
-Usage: heracles [options] [command]
+ --concurrency <number> (required) [default: 1]
+    Number of concurrent jobs to process
 
-Options:
-  -V, --version                         output the version number
-  -h, --help                            display help for command
+ --redis <string> (required) [default: default]
+    Name of the Redis connection to use
 
-Commands:
-  work [options]                        Start a Redis worker for specific queues
-  cleanup [options]                     Clean up Redis streams
-  help [command]                        display help for command
-```
+ --max-retries <number> (optional) [default: 3]
+    Maximum retries for failed jobs before giving up
 
-### Start Workers
+ --retry-delay <number> (optional) [default: 5000]
+    Delay between retries in milliseconds
 
-```
-Usage: heracles work [options]
-
-Start a Redis worker for specific queues
-
-Options:
-  --id <ID>                   Unique identifier for this worker (required)
-  -q, --queues <queues>       Comma-separated list of queues (default: "default")
-  -c, --concurrency <count>   Number of concurrent jobs per worker (default: "1")
-  -r, --redis <n>             Redis connection name (default: "default")
-  --config <path>             Path to configuration file (default: "")
-  --max-retries <count>       Maximum retries for failed jobs (default: "3")
-  --retry-delay <ms>          Delay between retries in milliseconds (default: "5000")
-  --pause-on-error            Pause queue when error occurs (default: false)
-  -h, --help                  display help for command
+ --pause-on-error <boolean> (optional) [default: false]
+    Pause the worker when an error occurs
+
+ --verbose <boolean> (optional) [default: false]
+    Enable verbose logging
 ```
 
 ### Clean Up Redis Streams
 
 ```
-Usage: heracles cleanup [options]
-
-Clean up Redis streams by removing completed and failed jobs
-
-Options:
-  -r, --redis <name>          Redis connection name (default: "default")
-  -q, --queues <queues>       Comma-separated list of queues (all if not specified)
-  -a, --all                   Remove all entries, not just completed ones (default: false)
-  -k, --keyprefix <prefix>    Redis key prefix (default: "elysium:heracles")
-  -t, --retention <seconds>   Time to retain completed jobs in seconds (default: "3600")
-  -m, --max <count>           Maximum stream size to trim to (default: "1000")
-  -s, --status-only           Only clean up job status keys, not stream entries (default: false)
-  --dry-run                   Show what would be removed without actually removing (default: false)
-  -h, --help                  display help for command
+Usage: styx heracles:clean [options]
+Description: Clean up Redis streams by removing completed and failed jobs.
+
+Arguments:
+ --redis <string> (required) [default: default]
+    Name of the Redis connection to use
+
+ --queues <array of strings> (optional) [default: ]
+    List of queues to clean. If empty, all queues will be cleaned
+
+ --dry-run <boolean> (optional) [default: false]
+    Show what would be removed without actually removing
+
+ --verbose <boolean> (optional) [default: false]
+    Enable verbose logging
+
+ --all <boolean> (optional) [default: false]
+    Remove all entries, not only completed and failed jobs
+
+ --key-prefix <string> (optional) [default: elysium:heracles]
+    Redis key prefix to clean
+
+ --retention <number> (optional) [default: 3600]
+    Number of seconds to retain completed jobs
+
+ --max <number> (optional) [default: 1000]
+    Maximum number of jobs to clean
 ```
 
 ## License

package/dist/heracles.d.ts

@@ -0,0 +1,13 @@
+import type { Job, JobClass } from './job';
+import type { JobDispatchId, JobDispatchOptions } from './queue';
+export declare namespace Heracles {
+    /**
+     * Dispatch a job to be executed asynchronously.
+     * @author Axel Nana <axel.nana@workbud.com>
+     * @param jobClass The job class to dispatch. Should extend the base {@link Job} class and have the {@link Job.register} decorator set.
+     * @param args The arguments to pass to the job constructor.
+     * @param options Additional options for the job dispatch.
+     * @returns The ID of the dispatched job.
+     */
+    const dispatch: <T extends Job, TClass extends JobClass<T> = JobClass<T>>(jobClass: TClass, args?: ConstructorParameters<TClass>, options?: JobDispatchOptions) => Promise<JobDispatchId>;
+}

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+import '@elysiumjs/core';
+export type { JobProps, JobClass, JobMetadata } from './job';
+export type { JobDispatchOptions, QueueOptions, JobDispatchId } from './queue';
+export type { JobStatusInfo, Transport, TransportClass, TransportEvent } from './transport';
+export type { QueuedJob, QueueState, Worker, WorkerInfo, WorkerQueueOptions } from './worker';
+export type { RedisTransportOptions } from './transports/redis.transport';
+export { Heracles } from './heracles';
+export { Job, JobStatus, JobOverlapBehavior } from './job';
+export { Queue } from './queue';
+export { TransportMode } from './transport';
+export { RedisTransport } from './transports/redis.transport';
+export { WorkerStatus } from './worker';
+export { BaseWorker } from './workers/base.worker';
+export { RedisWorker } from './workers/redis.worker';

package/dist/index.js

@@ -0,0 +1,2 @@
+// @bun
+import"@elysiumjs/core";import{uid as b}from"radash";import{Event as W,InteractsWithConsole as N}from"@elysiumjs/core";import{map as p,uid as n}from"radash";import"reflect-metadata";import{InteractsWithConsole as c,Service as j}from"@elysiumjs/core";import{assign as I,uid as v}from"radash";var R;((Q)=>(Q.job=Symbol("elysium:job"),Q.queue=Symbol("elysium:queue"),Q.worker=Symbol("elysium:worker"),Q.transport=Symbol("elysium:transport"),Q.jobTransaction=Symbol("elysium:jobTransaction"),Q.workerPool=Symbol("elysium:workerPool")))(R||={});var A=(L)=>{if(!Reflect.hasMetadata(R.job,L))return null;return Reflect.getMetadata(R.job,L)};var F;((H)=>{H.ALLOW_OVERLAP="allow_overlap";H.NO_OVERLAP="no_overlap"})(F||={});class K extends c{#Y;#H;#L="pending";#U;#z=0;#V;static register(L={}){return function(Y){L=I({name:Y.name,queue:"default",overlapBehavior:"allow_overlap",overlapDelay:0,priority:0},L);let H=`elysium.heracles.job.${L.name}`;j.instance(H,Y);let U={name:H,queue:L.queue,overlapBehavior:L.overlapBehavior,overlapDelay:L.overlapDelay,priority:L.priority,maxRetries:L.maxRetries,retryDelay:L.retryDelay};Reflect.defineMetadata(R.job,U,Y)}}static generateJobId(...L){return`job_${Date.now()}_${v(8)}`}static description="An Heracles job.";id;dispatchId;createdAt;get startedAt(){return this.#Y}get completedAt(){return this.#H}get status(){return this.#L}get lastError(){return this.#U}get retries(){return this.#z}get queueName(){return this.#V}set queueName(L){this.#V=L}incrementRetries(){this.#z++}constructor(L,Y){super();this.id=L??this.constructor.generateJobId(),this.dispatchId=Y??`dispatch_${Date.now()}_${v(8)}`,this.createdAt=new Date}async run(){if(this.#L==="cancelled")return;try{this.#L="running",this.#Y=new Date;let L=this.#V?` in queue '${this.#V}'`:"";if(this.debug(`Job ${this.id}${L} started at ${this.#Y.toISOString()}`),await this.execute(),this.#L!=="cancelled")this.#L="completed",this.#H=new Date,this.debug(`Job ${this.id} completed successfully at ${this.#H.toISOString()} after ${this.retries} retries`)}catch(L){if(this.#U=L instanceof Error?L:new Error(String(L)),this.#L!=="cancelled")this.#L="failed",this.#H=new Date,this.error(`Job ${this.id} failed at ${this.#H.toISOString()} after ${this.retries} retries: ${this.#U.message}`),this.trace(this.#U)}}cancel(){if(this.#L==="completed"||this.#L==="failed")return!1;let L=this.#L==="running";if(this.#L="cancelled",L)this.#H=new Date;return this.info(`Job ${this.id} was cancelled${L?" while running":" before execution"}`),!0}}var k=(L,Y,H)=>{return class extends L{constructor(...U){super(...U);this.id=Y,this.dispatchId=H}}},T;((X)=>{X.PENDING="pending";X.RUNNING="running";X.COMPLETED="completed";X.FAILED="failed";X.CANCELLED="cancelled";X.SCHEDULED_FOR_RETRY="scheduled_for_retry"})(T||={});var x;((H)=>{H.CONSUMER="consumer";H.PRODUCER="producer"})(x||={});class E extends N{static queues=new Map;static exists(L){return this.queues.has(L)}static get(L,Y){if(!this.queues.has(L)){let H=new E({name:L,...Y});this.queues.set(L,H)}return this.queues.get(L)}static getAll(){return new Map(this.queues)}name;transport;transportClass;transportOptions;options;jobs=[];constructor(L){super();this.name=L.name,this.transportClass=L.transport,this.transportOptions=L.transportOptions,this.options={concurrency:L.concurrency??1,maxRetries:L.maxRetries??0,retryDelay:L.retryDelay??1000,pauseOnError:L.pauseOnError??!1,...L},this.transport=new this.transportClass("producer",this.transportOptions),this.transport.onMessage(this.handleTransportMessage.bind(this))}on(L,Y){W.on(`elysium:heracles:queue:${this.name}:${L}`,Y)}once(L,Y){W.once(`elysium:heracles:queue:${this.name}:${L}`,Y)}emit(L,Y,H=null){W.emit(`elysium:heracles:queue:${this.name}:${L}`,Y,H)}async handleTransportMessage(L){switch(L.type){case"job:status":case"job:result":if(L.queue===this.name){if(L.type==="job:result")this.info(`Job ${L.jobId} completed with status: ${L.status}`);let Y=await this.getJobStatus({jobId:L.jobId,dispatchId:L.dispatchId}),H=Y?.createdAt??new Date().toISOString();this.updateJobStatus(L.jobId,L.dispatchId,{jobId:L.jobId,dispatchId:L.dispatchId,queue:L.queue,status:L.status,error:L.error,retries:L.retries??Y?.retries??0,createdAt:H,startedAt:L.startedAt??Y?.startedAt,completedAt:L.completedAt??Y?.completedAt,updatedAt:L.updatedAt??new Date().toISOString()})}break}this.emit(L.type,L,this)}updateJobStatus(L,Y,H){if(H.status==="completed"||H.status==="failed"||H.status==="cancelled")this.debug(`Job ${L} (${Y}) reached terminal state: ${H.status}`)}async dispatch(L,Y,H=[],U={}){let V=new Date,z=V.toISOString(),X=U.jobId??L.generateJobId(...H),P=U.dispatchId??`dispatch_${Date.now()}_${n(8)}`,Z={jobId:X,dispatchId:P,queue:this.name,status:U.scheduledFor&&U.scheduledFor>V?"scheduled_for_retry":"pending",retries:0,createdAt:z,updatedAt:z};this.updateJobStatus(X,P,Z),this.debug(`Dispatching job ${X} (dispatch: ${P}, name: ${Y}) to queue '${this.name}'`);try{let _={type:"job:process",job:Y,jobId:X,dispatchId:P,args:H,queue:this.name,options:{scheduledFor:U.scheduledFor?.toISOString(),priority:U.priority,maxRetries:U.maxRetries,retryDelay:U.retryDelay,overlapBehavior:U.overlapBehavior,overlapDelay:U.overlapDelay}};return await this.transport.send(_),{jobId:Z.jobId,dispatchId:P}}catch(_){throw this.error(`Failed to dispatch job ${Y} to queue '${this.name}': ${_.message}`),this.updateJobStatus(X,P,{jobId:Z.jobId,dispatchId:P,queue:this.name,status:"failed",error:_.message,retries:0,createdAt:Z.createdAt,completedAt:new Date().toISOString(),updatedAt:new Date().toISOString()}),_}}async cancelJob(L){this.debug(`Canceling job ${L.jobId} (${L.dispatchId}) in queue '${this.name}'`);try{let Y={type:"job:cancel",jobId:L.jobId,dispatchId:L.dispatchId,queue:this.name};return await this.transport.send(Y),!0}catch(Y){return this.error(`Failed to cancel job ${L.jobId}: ${Y}`),!1}}async cancelAllJobs(){this.debug(`Canceling all jobs in queue '${this.name}'`);try{let L={type:"job:cancelAll",queue:this.name};await this.transport.send(L)}catch(L){this.error(`Failed to cancel all jobs: ${L}`)}return 0}async getJobStatus(L){try{let Y=await this.transport.getJobStatus(L.jobId,L.dispatchId,this.name);if(Y)return this.debug(`Retrieved job ${L.jobId} (${L.dispatchId}) status from transport: ${Y.status}`),Y}catch(Y){this.warning(`Failed to get job status from transport: ${Y}`)}return null}getAllJobs(){return Array.from(this.jobs)}async getJobsByStatus(L){let Y=[];for(let[H,U]of await p(this.jobs,async(V)=>[V,await this.getJobStatus(V)]))if(U?.status===L)Y.push(H);return Y}async start(){if(this.transport)await this.transport.start(),this.info(`Queue '${this.name}' transport started`)}async stop(){if(this.transport)await this.transport.stop(),this.info(`Queue '${this.name}' transport stopped`)}getOptions(){return{...this.options}}updateOptions(L){this.options={...this.options,...L}}}var y;((f)=>f.dispatch=async(Y,H=[],U={})=>{let V=A(Y);if(!V)throw new Error(`Job ${Y.name} is not marked with @Job.register() decorator`);if(!V.queue)throw new Error(`Job ${Y.name} is not registered with a queue`);if(!E.exists(V.queue))throw new Error(`Queue ${V.queue} does not exist. Please create the queue before dispatching the job.`);let z=E.get(V.queue),P={dispatchId:`dispatch_${Date.now()}_${b(8)}`,priority:V.priority,maxRetries:V.maxRetries,retryDelay:V.retryDelay,overlapBehavior:V.overlapBehavior,overlapDelay:V.overlapDelay,...U};return z.dispatch(Y,V.name,H,P)})(y||={});import{InteractsWithConsole as i,Redis as g}from"@elysiumjs/core";import{uid as S}from"radash";class D extends i{client;options;mode;messageHandlers=[];pollTimer;cleanupTimer;lastIds=new Map;checkLogs=new Map;consumerId;consumerGroupCreated=new Set;completedMessageIds=new Map;jobLocks=new Map;constructor(L,Y={}){super();this.mode=L,this.options={connection:Y.connection??"default",keyPrefix:Y.keyPrefix??"elysium:heracles",consumerGroup:Y.consumerGroup??"workers",consumerName:Y.consumerName??`worker-${S(8)}`,pollInterval:Y.pollInterval??1000,batchSize:Y.batchSize??10,statusTTL:Y.statusTTL??86400,cleanupCompletedJobs:Y.cleanupCompletedJobs!==!1,completedJobRetention:Y.completedJobRetention??3600,maxStreamSize:Y.maxStreamSize??1000},this.client=g.getConnection(this.options.connection),this.consumerId=this.mode==="consumer"?this.options.consumerName:`producer-${S(8)}`}redisCommand(L,Y){let H=Y.map((U)=>{if(typeof U==="number")return U.toString();else if(U===null||U===void 0)return"";else if(Array.isArray(U))return JSON.stringify(U);else if(typeof U==="object")return JSON.stringify(U);return U});try{return this.client.send(L,H)}catch(U){throw new Error(`Redis command failed: ${L} - ${U}`)}}async findJobMessageId(L,Y,H){let U=this.getStreamKey(H);try{let V=await this.redisCommand("XRANGE",[U,"-","+","COUNT","1000"]);for(let[z,X]of V){let P={};for(let Z=0;Z<X.length;Z+=2)P[X[Z]]=X[Z+1];if(P.jobId===L&&P.dispatchId===Y)return z}}catch(V){this.error(`Error finding job message: ${V}`)}return null}async start(){try{if(await this.redisCommand("PING",[]),this.info(`RedisTransport started in ${this.mode} mode with ID: ${this.consumerId}`),this.mode==="consumer"){if(this.startPolling(),this.options.cleanupCompletedJobs)this.scheduleStreamCleanup()}}catch(L){throw this.error(`Failed to start RedisTransport: ${L}`),new Error(`Redis connection failed: ${L}`)}}async stop(){if(this.pollTimer)clearInterval(this.pollTimer),this.pollTimer=void 0;if(this.cleanupTimer)clearInterval(this.cleanupTimer),this.cleanupTimer=void 0;this.info("RedisTransport stopped")}async send(L){try{let Y=["default"];if("queue"in L&&typeof L.queue==="string")Y=[L.queue];else if("queues"in L&&Array.isArray(L.queues))Y=L.queues;for(let H of Y){let U=this.getStreamKey(H),V=this.serializeMessage(L),z="0";if(L.type==="job:status"||L.type==="job:result")if(await this.findJobMessageId(L.jobId,L.dispatchId,H)){let P={status:L.status,updatedAt:Date.now().toString()};if(L.error)P.error=L.error;if(L.retries!==void 0)P.retries=L.retries.toString();if(L.startedAt)P.startedAt=L.startedAt;if(L.completedAt)P.completedAt=L.completedAt;await this.redisCommand("HSET",[this.getJobStatusKey(L.jobId,L.dispatchId,H),...Object.entries(P).flat()]),this.debug(`Updated status for job ${L.jobId} to ${L.status}`),await this.storeJobStatus({jobId:L.jobId,dispatchId:L.dispatchId,queue:H,status:L.status,error:L.error,retries:L.retries??0,createdAt:L.createdAt??new Date().toISOString(),startedAt:L.startedAt,completedAt:L.completedAt});return}else{let{jobId:P,dispatchId:Z}=L,_=this.getJobStatusKey(P,Z,H),$=await this.redisCommand("EXISTS",[_]);if(z=await this.redisCommand("XADD",[U,"*",...V]),$)await this.updateJobStatus(L.jobId,L.dispatchId,H,{status:L.status,error:L.error,retries:L.retries??0,startedAt:L.startedAt,completedAt:L.completedAt,updatedAt:Date.now().toString()});else await this.storeJobStatus({jobId:L.jobId,dispatchId:L.dispatchId,queue:H,status:L.status,error:L.error,retries:L.retries??0,createdAt:L.createdAt??new Date().toISOString(),startedAt:L.startedAt,completedAt:L.completedAt,messageId:z,updatedAt:Date.now().toString()})}else if(L.type==="job:process")z=await this.redisCommand("XADD",[U,"*",...V]),await this.storeJobStatus({jobId:L.jobId,dispatchId:L.dispatchId,queue:H,status:"pending",retries:0,createdAt:new Date().toISOString(),messageId:z,updatedAt:Date.now().toString()});else if(L.type==="job:update")await this.updateJobStatus(L.jobId,L.dispatchId,H,L.updates);else z=await this.redisCommand("XADD",[U,"*",...V]);this.debug(`Sent ${L.type} message to stream ${U} with ID ${z}`)}}catch(Y){throw this.error(`Failed to send message: ${Y}`),new Error(`Failed to send message: ${Y}`)}}onMessage(L){this.messageHandlers.push(L)}async getJobStatus(L,Y,H){try{let U=this.getJobStatusKey(L,Y,H),V=await this.redisCommand("HGETALL",[U]);if(V){let z={jobId:L,dispatchId:V.dispatchId,queue:H,status:V.status||"unknown",error:V.error,retries:parseInt(V.retries||"0",10),createdAt:V.createdAt||new Date().toISOString(),startedAt:V.startedAt,completedAt:V.completedAt,updatedAt:V.updatedAt,messageId:V.messageId};return this.debug(`Retrieved job status for ${L} from Redis: ${z.status}`),z}}catch(U){this.warning(`Failed to get job status from Redis: ${U}`)}try{let U=await this.findJobMessageId(L,Y,H);if(U){let V={jobId:L,dispatchId:Y,queue:H,status:"pending",retries:0,createdAt:new Date().toISOString(),messageId:U,updatedAt:Date.now().toString()};return await this.storeJobStatus(V),V}}catch(U){this.debug(`Error checking job existence in stream: ${U}`)}return{jobId:L,dispatchId:Y,queue:H,status:"unknown",retries:0,createdAt:new Date().toISOString()}}async updateJobStatus(L,Y,H,U){try{let V=this.getJobStatusKey(L,Y,H),z=Date.now().toString();if(!await this.redisCommand("EXISTS",[V])){if(U.status){let _={jobId:L,dispatchId:Y,queue:H,status:U.status,error:U.error,retries:U.retries||0,createdAt:U.createdAt||new Date().toISOString(),startedAt:U.startedAt,completedAt:U.completedAt,updatedAt:z,messageId:U.messageId};if(await this.storeJobStatus(_),U.status==="completed"||U.status==="failed"||U.status==="cancelled")for(let $ of this.messageHandlers)try{$({type:"job:result",jobId:L,dispatchId:Y,queue:H,status:U.status,error:U.error,completedAt:U.completedAt??new Date().toISOString()})}catch(G){this.error(`Error in message handler: ${G}`)}else for(let $ of this.messageHandlers)try{$({type:"job:status",jobId:L,dispatchId:Y,queue:H,status:U.status,error:U.error,updatedAt:U.updatedAt??new Date().toISOString()})}catch(G){this.error(`Error in message handler: ${G}`)}}return}let P=[],Z={};if(U.status)P.push("status",U.status),Z.status=U.status;if(U.error!==void 0)P.push("error",U.error||""),Z.error=U.error||"";if(U.retries!==void 0)P.push("retries",U.retries.toString()),Z.retries=U.retries;if(U.startedAt)P.push("startedAt",U.startedAt),Z.startedAt=U.startedAt;if(U.completedAt)P.push("completedAt",U.completedAt),Z.completedAt=U.completedAt;if(P.push("updatedAt",z),Z.updatedAt=z,P.length>0)if(await this.redisCommand("HSET",[V,...P]),await this.redisCommand("EXPIRE",[V,this.options.statusTTL]),this.debug(`Updated job ${L} status: ${U.status??"fields updated"}`),U.status==="completed"||U.status==="failed"||U.status==="cancelled"){let _=await this.getJobStatus(L,Y,H);if(_.messageId){this.trackCompletedMessage(_.messageId,this.getStreamKey(H));for(let $ of this.messageHandlers)try{$({type:"job:result",jobId:L,dispatchId:Y,queue:H,status:U.status,error:U.error,completedAt:U.completedAt||z,retries:parseInt(_.retries.toString(),10)})}catch(G){this.error(`Error in message handler: ${G}`)}}}else for(let _ of this.messageHandlers)try{_({...Z,type:"job:status",jobId:L,queue:H})}catch($){this.error(`Error in message handler: ${$}`)}}catch(V){throw this.error(`Failed to update job status: ${V}`),new Error(`Failed to update job status: ${V}`)}}async registerWorker(L,Y){try{for(let H of Y){let U=this.getStreamKey(H);await this.ensureStreamExists(U),await this.ensureConsumerGroupExists(U);let V=this.getWorkerKey(L);await this.redisCommand("HSET",[V,"id",L,"status","active","lastSeen",new Date().toISOString(),"queues",JSON.stringify(Y)]),await this.redisCommand("EXPIRE",[V,"60"]),this.debug(`Registered worker ${L} for queue ${H}`)}await this.send({type:"worker:register",workerId:L,queues:Y})}catch(H){throw this.error(`Failed to register worker: ${H}`),new Error(`Failed to register worker: ${H}`)}}async unregisterWorker(L){try{let Y=this.getWorkerKey(L);await this.redisCommand("DEL",[Y]),await this.send({type:"worker:unregister",workerId:L}),this.debug(`Unregistered worker ${L}`)}catch(Y){throw this.error(`Failed to unregister worker: ${Y}`),new Error(`Failed to unregister worker: ${Y}`)}}startPolling(){this.pollTimer=setInterval(()=>{this.pollMessages().catch((L)=>{this.error(`Error polling messages: ${L}`)})},this.options.pollInterval)}async pollMessages(){if(this.mode!=="consumer")return;let L=await this.getRegisteredQueues();for(let Y of L){let H=this.getStreamKey(Y);try{await this.ensureConsumerGroupExists(H);let U;if(this.checkLogs.get(Y)??!0)U=await this.redisCommand("XREADGROUP",["GROUP",this.options.consumerGroup,this.consumerId,"COUNT",this.options.batchSize,"STREAMS",H,0])??{};else U=await this.redisCommand("XREADGROUP",["GROUP",this.options.consumerGroup,this.consumerId,"COUNT",this.options.batchSize,"STREAMS",H,">"])??{};await this.processMessages(U[H],Y)}catch(U){if(console.error(U),!String(U).includes("timeout"))this.error(`Error reading from stream ${H}: ${U}`)}}}async processMessages(L,Y){if(!L||L.length===0){this.checkLogs.set(Y,!(this.checkLogs.get(Y)??!0));return}let H=this.getStreamKey(Y);for(let[U,V]of L)try{let z=this.deserializeMessage(V,Y);if(this.lastIds.set(H,U),z.type==="job:process"){let X=this.getJobStatusKey(z.jobId,z.dispatchId,Y);if(await this.redisCommand("EXISTS",[X])){if((await this.getJobStatus(z.jobId,z.dispatchId,Y)).status!=="pending"){this.debug(`Skipping duplicate job message for ${z.jobId}`),await this.redisCommand("XACK",[H,this.options.consumerGroup,U]);continue}}}for(let X of this.messageHandlers)try{await X(z)}catch(P){this.error(`Error in message handler: ${P}`)}if((z.type==="job:result"||z.type==="job:status")&&(z.status==="completed"||z.status==="failed"||z.status==="cancelled"))this.trackCompletedMessage(U,H);await this.redisCommand("XACK",[H,this.options.consumerGroup,U])}catch(z){this.error(`Error processing message ${U}: ${z}`)}}async getRegisteredQueues(){try{let L=`${this.options.keyPrefix}:stream:*`,Y=await this.redisCommand("KEYS",[L]);if(!Array.isArray(Y))return["default"];return Y.map((H)=>H.replace(`${this.options.keyPrefix}:stream:`,"")).filter(Boolean)}catch(L){return this.error(`Failed to get registered queues: ${L}`),["default"]}}async ensureStreamExists(L){try{if(!await this.redisCommand("XINFO",["STREAM",L]).catch(()=>null)){let H=await this.redisCommand("XADD",[L,"*","init","true"]);await this.redisCommand("XDEL",[L,H]),this.debug(`Created stream ${L}`)}}catch(Y){this.error(`Failed to ensure stream ${L} exists: ${Y}`)}}async ensureConsumerGroupExists(L){if(this.consumerGroupCreated.has(L))return;try{let Y=await this.redisCommand("XINFO",["GROUPS",L]).catch(()=>[]);if(!(Array.isArray(Y)&&Y.some((U)=>U[1]===this.options.consumerGroup)))await this.redisCommand("XGROUP",["CREATE",L,this.options.consumerGroup,"0","MKSTREAM"]),this.debug(`Created consumer group ${this.options.consumerGroup} for stream ${L}`);this.consumerGroupCreated.add(L)}catch(Y){if(Y.toString().includes("BUSYGROUP"))this.debug(`Consumer group ${this.options.consumerGroup} already exists for stream ${L}`),this.consumerGroupCreated.add(L);else throw this.error(`Failed to ensure consumer group exists: ${Y}`),Y}}async storeJobStatus(L){try{let Y=this.getJobStatusKey(L.jobId,L.dispatchId,L.queue),H=["jobId",L.jobId,"queue",L.queue,"status",L.status,"retries",L.retries.toString(),"createdAt",L.createdAt];if(L.startedAt)H.push("startedAt",L.startedAt);if(L.completedAt)H.push("completedAt",L.completedAt);if(L.error)H.push("error",L.error);if(L.messageId)H.push("messageId",L.messageId);let U=L.updatedAt||Date.now().toString();if(H.push("updatedAt",U),await this.redisCommand("HSET",[Y,...H]),await this.redisCommand("EXPIRE",[Y,this.options.statusTTL]),L.status==="pending"||L.status==="scheduled_for_retry")for(let V of this.messageHandlers)try{V({type:"job:status",jobId:L.jobId,dispatchId:L.dispatchId,queue:L.queue,status:L.status,createdAt:L.createdAt,updatedAt:U})}catch(z){this.error(`Error in message handler: ${z}`)}if(L.status==="completed"||L.status==="failed"||L.status==="cancelled"){let V=L.messageId??null;if(!V)V=await this.findJobMessageId(L.jobId,L.dispatchId,L.queue);if(V){let z=this.getStreamKey(L.queue);this.trackCompletedMessage(V,z);for(let X of this.messageHandlers)try{X({type:"job:result",jobId:L.jobId,dispatchId:L.dispatchId,queue:L.queue,status:L.status,error:L.error,completedAt:L.completedAt||new Date().toISOString(),retries:L.retries})}catch(P){this.error(`Error in message handler: ${P}`)}}}}catch(Y){throw this.error(`Failed to store job status: ${Y}`),new Error(`Failed to store job status: ${Y}`)}}serializeMessage(L){let Y=["type",L.type,"timestamp",Date.now().toString()];if("queue"in L&&typeof L.queue==="string")Y.push("queue",L.queue);switch(L.type){case"job:process":if(Y.push("job",L.job,"args",JSON.stringify(L.args||[]),"jobId",L.jobId,"dispatchId",L.dispatchId),L.options)Y.push("options",JSON.stringify(L.options));break;case"job:cancel":case"job:status":case"job:result":if(Y.push("jobId",L.jobId),Y.push("dispatchId",L.dispatchId),L.type!=="job:cancel"&&L.status)Y.push("status",L.status);if(L.type!=="job:cancel"&&L.error)Y.push("error",L.error);if(L.retries!==void 0)Y.push("retries",L.retries);if(L.startedAt)Y.push("startedAt",L.startedAt);if(L.completedAt)Y.push("completedAt",L.completedAt);break;case"job:cancelAll":break;case"worker:register":Y.push("workerId",L.workerId,"queues",JSON.stringify(L.queues??[]));break;case"worker:unregister":Y.push("workerId",L.workerId);break;case"worker:status":Y.push("workerId",L.workerId,"status",L.status,"queues",JSON.stringify(L.queues),"processing",L.processing.toString(),"waiting",L.waiting.toString());break;case"worker:ready":if(Y.push("id",L.id,"queues",JSON.stringify(L.queues)),L.status)Y.push("status",L.status);if(L.timestamp)Y.push("timestamp",L.timestamp.toString());break}return Y}deserializeMessage(L,Y){let H={};for(let z=0;z<L.length;z+=2)H[L[z]]=L[z+1];let U=H.type,V={type:U,queue:H.queue||Y};switch(U){case"job:process":return{...V,type:"job:process",job:H.job,args:JSON.parse(H.args||"[]"),options:H.options?JSON.parse(H.options):void 0,jobId:H.jobId,dispatchId:H.dispatchId};case"job:cancel":return{...V,type:"job:cancel",jobId:H.jobId,dispatchId:H.dispatchId};case"job:cancelAll":return{...V,type:"job:cancelAll"};case"job:status":return{...V,type:"job:status",jobId:H.jobId,dispatchId:H.dispatchId,status:H.status,error:H.error,retries:H.retries?parseInt(H.retries,10):0,startedAt:H.startedAt,completedAt:H.completedAt};case"job:result":return{...V,type:"job:result",jobId:H.jobId,dispatchId:H.dispatchId,status:H.status,error:H.error,completedAt:H.completedAt||new Date().toISOString()};case"worker:register":return{...V,type:"worker:register",workerId:H.workerId,queues:H.queues?JSON.parse(H.queues):[]};case"worker:unregister":return{...V,type:"worker:unregister",workerId:H.workerId};case"worker:status":return{...V,type:"worker:status",workerId:H.workerId,queues:H.queues?JSON.parse(H.queues):[],status:H.status,processing:parseInt(H.processing||"0",10),waiting:parseInt(H.waiting||"0",10)};case"worker:ready":return{...V,type:"worker:ready",id:H.id,queues:H.queues?JSON.parse(H.queues):[],status:H.status,timestamp:H.timestamp?parseInt(H.timestamp,10):void 0};default:return V}}getStreamKey(L){return`${this.options.keyPrefix}:stream:${L||"default"}`}getJobStatusKey(L,Y,H){return`${this.options.keyPrefix}:status:${H}:${L}:${Y}`}getWorkerKey(L){return`${this.options.keyPrefix}:worker:${L}`}trackCompletedMessage(L,Y){if(this.options.cleanupCompletedJobs)this.completedMessageIds.set(L,{streamKey:Y,timestamp:Date.now()}),this.debug(`Marked message ${L} for future cleanup`)}scheduleStreamCleanup(){if(this.cleanupTimer)clearInterval(this.cleanupTimer);let L=300000;this.cleanupTimer=setInterval(()=>{this.cleanupStreams().catch((Y)=>{this.error(`Error during stream cleanup: ${Y}`)})},L),this.debug(`Scheduled stream cleanup every ${L/1000} seconds`)}async cleanupStreams(){if(!this.options.cleanupCompletedJobs)return;this.debug("Starting stream cleanup for completed jobs");let L=Date.now(),Y=this.options.completedJobRetention*1000,H=new Map,U=[],V=await this.getRegisteredQueues();for(let z of V){let X=this.getStreamKey(z);try{let P=`${this.options.keyPrefix}:status:${z}:*`,Z=await this.redisCommand("KEYS",[P]);for(let _ of Z){let $=await this.redisCommand("HGETALL",[_]);if(!$||$.length===0)continue;let G={};for(let B=0;B<$.length;B+=2)G[$[B]]=$[B+1];if((G.status==="completed"||G.status==="failed"||G.status==="cancelled")&&G.updatedAt&&L-parseInt(G.updatedAt)>Y){let B=_.split(":"),J=B.pop(),l=B.pop();if(U.push(_),G.messageId){if(!H.has(X))H.set(X,[]);H.get(X).push(G.messageId)}else{let O=await this.findJobMessageId(l,J,z);if(O){if(!H.has(X))H.set(X,[]);H.get(X).push(O)}}}}}catch(P){this.error(`Error finding completed jobs for queue ${z}: ${P}`)}}for(let[z,X]of H.entries())if(X.length>0)try{for(let P=0;P<X.length;P+=100){let Z=X.slice(P,P+100);await this.redisCommand("XDEL",[z,...Z])}this.debug(`Cleaned up ${X.length} completed messages from stream ${z}`);for(let P of X)this.completedMessageIds.delete(P)}catch(P){this.error(`Error deleting completed messages from stream ${z}: ${P}`)}if(U.length>0)try{for(let z=0;z<U.length;z+=100){let X=U.slice(z,z+100);await this.redisCommand("DEL",X)}this.debug(`Cleaned up ${U.length} job status records`)}catch(z){this.error(`Error deleting expired status keys: ${z}`)}if(this.options.maxStreamSize>0)await this.trimStreamsToMaxSize()}async trimStreamsToMaxSize(){try{let L=await this.getRegisteredQueues();for(let Y of L){let H=this.getStreamKey(Y),U=await this.redisCommand("XLEN",[H]);if(U>this.options.maxStreamSize)await this.redisCommand("XTRIM",[H,"MAXLEN","~",this.options.maxStreamSize.toString()]),this.debug(`Trimmed stream ${H} from ${U} to ~${this.options.maxStreamSize} entries`)}}catch(L){this.error(`Error trimming streams to max size: ${L}`)}}async isJobLocked(L,Y){let H=this.jobLocks.get(L);if(H&&H.lockedUntil>Date.now())return!0;let U=`${this.options.keyPrefix}:lock:${Y}:${L}`;return!!await this.redisCommand("EXISTS",[U])}async acquireJobLock(L,Y,H=60000){let U=`${this.options.keyPrefix}:lock:${Y}:${L}`;if(await this.redisCommand("SET",[U,Date.now().toString(),"PX",H.toString(),"NX"])==="OK")return this.jobLocks.set(L,{lockedUntil:Date.now()+H,queue:Y}),!0;return!1}async releaseJobLock(L,Y){let H=`${this.options.keyPrefix}:lock:${Y}:${L}`;await this.redisCommand("DEL",[H]),this.jobLocks.delete(L)}}var w;((P)=>{P.IDLE="idle";P.ACTIVE="active";P.PAUSED="paused";P.DRAINING="draining";P.STOPPING="stopping";P.STOPPED="stopped";P.DISCONNECTED="disconnected"})(w||={});import{InteractsWithConsole as q}from"@elysiumjs/core";import{sort as C,uid as r}from"radash";class M extends q{#Y;transport=null;queues=new Map;defaultQueueName="default";defaultTransport;workerStatus="idle";startedAt=new Date;scheduledJobInterval=1000;scheduledJobTimer;constructor(L){super();this.#Y=L??r(24),this.createQueue({name:this.defaultQueueName})}get id(){return this.#Y}get status(){return this.workerStatus}async createQueue(L){if(this.queues.has(L.name))return this.debug(`Queue with name '${L.name}' already exists`),this;let Y={options:{...L,concurrency:L.concurrency||1,maxRetries:L.maxRetries||0,retryDelay:L.retryDelay||1000,pauseOnError:L.pauseOnError||!1},jobs:[],activeJobs:new Map,processing:0,paused:!1,draining:!1};return this.queues.set(L.name,Y),this.debug(`Created queue '${L.name}'`),this}async addJob(L,Y=this.defaultQueueName,H={}){let U=this.queues.get(Y);if(!U)throw this.error(`Queue '${Y}' does not exist`),new Error(`Queue '${Y}' does not exist`);if(U.draining)throw this.warning(`Queue '${Y}' is draining, job ${L.id} will not be added`),new Error(`Queue '${Y}' is draining`);L.queueName=Y;let V={job:L,retries:L.retries,queueName:Y,enqueuedAt:new Date,scheduledFor:H.scheduledFor,priority:H.priority??10,maxRetries:H.maxRetries,retryDelay:H.retryDelay,overlapBehavior:H.overlapBehavior,overlapDelay:H.overlapDelay};if(U.jobs.push(V),U.jobs=C(C(C(U.jobs,(z)=>z.enqueuedAt.getTime()),(z)=>z.scheduledFor?z.scheduledFor.getTime():0),(z)=>z.priority??10),this.debug(`Added job ${L.id} to queue '${Y}'`),this.workerStatus==="idle")this.workerStatus="active";if(!U.paused&&U.processing<U.options.concurrency)try{this.processQueueJobs(Y)}catch(z){this.error(`Error processing next job in queue '${Y}': ${z}`)}return this}async pause(L){if(L){let Y=this.queues.get(L);if(Y)Y.paused=!0,this.info(`Paused queue '${L}'`);else this.warning(`Cannot pause non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.pause(Y);if(Array.from(this.queues.values()).every((Y)=>Y.paused)&&Array.from(this.queues.values()).every((Y)=>Y.processing===0))this.workerStatus="paused",this.info(`Worker ${this.id} is now paused`)}return this}async resume(L){if(L){let Y=this.queues.get(L);if(Y){if(Y.paused=!1,this.info(`Resumed queue '${L}'`),this.workerStatus==="paused")this.workerStatus="active";await this.processQueueJobs(L)}else this.warning(`Cannot resume non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.resume(Y);if(Array.from(this.queues.values()).some((Y)=>Y.jobs.length>0||Y.processing>0))this.workerStatus="active",this.info(`Worker ${this.id} is now active`)}return this}async drain(L){if(L){let Y=this.queues.get(L);if(Y)Y.draining=!0,this.info(`Draining queue '${L}'`);else this.warning(`Cannot drain non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.drain(Y);this.workerStatus="draining",this.info(`Worker ${this.id} is now draining`)}return this}async clear(L){if(L){let Y=this.queues.get(L);if(Y){let H=Y.jobs.length;for(let U of Y.jobs)U.job.cancel();Y.jobs=[],this.info(`Cleared ${H} jobs from queue '${L}'`)}else this.warning(`Cannot clear non-existent queue '${L}'`)}else{for(let[Y,H]of this.queues.entries()){let U=H.jobs.length;for(let V of H.jobs)V.job.cancel();H.jobs=[],this.info(`Cleared ${U} jobs from queue '${Y}'`)}if(Array.from(this.queues.values()).every((Y)=>Y.jobs.length===0&&Y.processing===0))this.workerStatus="idle"}return this}async size(L=this.defaultQueueName){let Y=this.queues.get(L);if(!Y)throw new Error(`Queue '${L}' does not exist`);return Y.jobs.length+Y.processing}async totalSize(){let L=0;for(let Y of this.queues.values())L+=Y.jobs.length+Y.processing;return L}async getStats(){let L={};for(let[Y,H]of this.queues.entries())L[Y]={waiting:H.jobs.length,processing:H.processing,paused:H.paused,draining:H.draining};return L}getInfo(){let L=0;for(let V of this.queues.values())L+=V.processing;let Y=Array.from(this.queues.keys()),H=this.defaultTransport?this.defaultTransport.constructor.name:"None",U=async(V,z)=>{try{return await this.addJob(V,z),!0}catch(X){return!1}};return{id:this.id,status:this.workerStatus,queues:Y,activeJobs:L,transport:H,startedAt:this.startedAt,addJob:U,cancelJob:this.cancelJob.bind(this),cancelAllJobs:this.cancelAllJobs.bind(this)}}async cancelJob(L,Y){if(Y)return this.cancelJobInQueue(L,Y);else{for(let H of this.queues.keys())if(await this.cancelJobInQueue(L,H))return!0;return!1}}async cancelAllJobs(L){let Y=0;if(L){let H=this.queues.get(L);if(H){Y=H.jobs.length;for(let U of H.jobs)U.job.cancel();for(let[U,V]of H.activeJobs.entries())if(V.job.cancel())Y++;this.info(`Canceled ${Y} jobs in queue '${L}'`)}}else for(let[H,U]of this.queues.entries()){let V=U.jobs.length;for(let z of U.jobs)z.job.cancel();for(let[z,X]of U.activeJobs.entries())if(X.job.cancel())V++;Y+=V,this.info(`Canceled ${V} jobs in queue '${H}'`)}if(Y>0&&Array.from(this.queues.values()).every((H)=>H.jobs.length===0&&H.activeJobs.size===0))this.workerStatus="idle";return Y}async getJob(L,Y){if(Y){let H=this.queues.get(Y);if(H){let U=H.jobs.find((z)=>z.job.id===L);if(U)return U.job;let V=H.activeJobs.get(L);if(V)return V.job}}else for(let H of this.queues.values()){let U=H.jobs.find((z)=>z.job.id===L);if(U)return U.job;let V=H.activeJobs.get(L);if(V)return V.job}return null}async start(){if(this.workerStatus="active",this.startedAt=new Date,this.defaultTransport)await this.defaultTransport.start();for(let L of this.queues.keys())await this.processQueueJobs(L);this.startScheduledJobTimer(),this.info(`Worker ${this.id} started`)}async stop(L=!1){if(this.workerStatus="stopping",this.stopScheduledJobTimer(),L){await this.cancelAllJobs();for(let Y of this.queues.keys())await this.clear(Y)}else{for(let H of this.queues.keys())await this.drain(H);let Y=()=>{return Array.from(this.queues.values()).every((H)=>H.processing===0)};if(!Y()){this.info(`Worker ${this.id} waiting for ${Array.from(this.queues.values()).reduce((H,U)=>H+U.processing,0)} jobs to complete`);while(!Y())await new Promise((H)=>setTimeout(H,100))}}if(this.defaultTransport)await this.defaultTransport.stop();this.workerStatus="stopped",this.info(`Worker ${this.id} stopped`)}async setConcurrency(L,Y){let H=this.queues.get(L);if(!H)throw this.error(`Cannot set concurrency: Queue '${L}' does not exist`),new Error(`Queue '${L}' does not exist`);if(Y<1)this.warning(`Invalid concurrency value ${Y}, using 1 instead`),Y=1;return H.options.concurrency=Y,this.info(`Set concurrency for queue '${L}' to ${Y}`),this.processQueueJobs(L)}startScheduledJobTimer(){if(this.scheduledJobTimer)clearInterval(this.scheduledJobTimer);this.scheduledJobTimer=setInterval(()=>{this.checkScheduledJobs()},this.scheduledJobInterval)}stopScheduledJobTimer(){if(this.scheduledJobTimer)clearInterval(this.scheduledJobTimer),this.scheduledJobTimer=void 0}checkScheduledJobs(){let L=new Date;for(let[Y,H]of this.queues.entries()){if(H.paused||H.draining)continue;let U=H.jobs.filter((V)=>V.scheduledFor&&V.scheduledFor<=L);for(let V of U)V.scheduledFor=void 0;if(U.length>0&&H.processing<H.options.concurrency)this.processQueueJobs(Y)}}async processQueueJobs(L){let Y=this.queues.get(L);if(!Y||Y.paused)return;let H=[];while(H.length<Y.options.concurrency)H.push(this.processNextJob(L));await Promise.all(H);let U=Array.from(this.queues.values());if(U.some((V)=>V.processing>0))this.workerStatus="active";else if(U.every((V)=>V.jobs.length===0&&V.processing===0))this.workerStatus="idle";if(U.some((V)=>V.jobs.length>0))await Bun.sleep(10),setImmediate(()=>this.processQueueJobs(L))}async sendJobStatusUpdate(L,Y,H,U,V){try{await this.transport.updateJobStatus(L,Y,H,{dispatchId:Y,status:U,error:V,retries:0,startedAt:U==="running"?new Date().toISOString():void 0,completedAt:U==="completed"||U==="failed"||U==="cancelled"?new Date().toISOString():void 0,updatedAt:Date.now().toString()})}catch(z){this.error(`Error sending job status update: ${z}`)}}async cancelJobInQueue(L,Y){let H=this.queues.get(Y);if(!H)return!1;let U=H.jobs.findIndex((V)=>V.job.id===L);if(U>=0){if(H.jobs[U].job.cancel())return H.jobs.splice(U,1),this.info(`Canceled job ${L} in queue '${Y}'`),!0}if(H.activeJobs.has(L)){if(H.activeJobs.get(L).job.cancel())return this.info(`Canceled running job ${L} in queue '${Y}'`),!0}return!1}}import{Service as d}from"@elysiumjs/core";class h extends M{isRunning=!1;registeredQueues=new Set;constructor(L="default",Y={}){super(Y.id);this.transport=new D("consumer",{connection:L,consumerName:`worker-${this.id}`,...Y}),this.transport.onMessage(this.handleTransportMessage.bind(this)),this.debug(`RedisWorker ${this.id} created with connection: ${L}`)}async start(){if(await super.start(),this.isRunning)return;this.isRunning=!0;for(let L of this.queues.keys())this.registeredQueues.add(L);await this.transport.start(),await this.transport.registerWorker(this.id,Array.from(this.registeredQueues)),this.info(`Redis worker ${this.id} started for queues: ${Array.from(this.registeredQueues).join(", ")}`)}async stop(L=!1){this.isRunning=!1,await super.stop(L),await this.transport.unregisterWorker(this.id),await this.transport.stop(),this.info(`Redis worker ${this.id} stopped`)}async processNextJob(L){let Y=this.queues.get(L);if(!Y||Y.paused||Y.processing>=Y.options.concurrency)return;let H=-1,U=new Date;for(let P=0;P<Y.jobs.length;P++){let Z=Y.jobs[P];if(Z.scheduledFor&&Z.scheduledFor>U)continue;if(Z.job.status==="cancelled"){H=P;break}let _=Z.job.constructor;if(A(_)?.overlapBehavior==="no_overlap"){if(await this.transport.isJobLocked(Z.job.id,L))continue}H=P;break}if(H===-1)return;let V=Y.jobs.splice(H,1)[0];if(!V)return;if(V.job.status==="cancelled")return this.info(`Skipping cancelled job ${V.job.id}`),await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"cancelled"),this.processQueueJobs(L);let z=V.job.constructor,X=A(z);if(X?.overlapBehavior==="no_overlap"){if(!await this.transport.acquireJobLock(V.job.id,L)){Y.jobs.push(V);return}}Y.activeJobs.set(V.job.id,V),Y.processing++,await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"running",void 0);try{if(this.debug(`Processing job ${V.job.id} (dispatch: ${V.job.dispatchId}) from queue '${L}'`),await V.job.run(),V.job.status==="failed"){let P=V.maxRetries!==void 0?V.maxRetries:Y.options.maxRetries;if(V.retries<P){V.retries++,V.job.incrementRetries(),this.warning(`Job ${V.job.id} failed, retrying (${V.retries}/${P})`);let Z=V.retryDelay!==void 0?V.retryDelay:Y.options.retryDelay;await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"scheduled_for_retry",V.job.lastError?.message),setTimeout(()=>{if(V.job.status!=="cancelled")Y.jobs.push(V),this.processQueueJobs(L);else this.info(`Skipping cancelled job ${V.job.id} during retry delay`),this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"cancelled")},Z)}else if(this.error(`Job ${V.job.id} failed after ${V.retries} retries (max: ${P}): ${V.job.lastError?.message}`),await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"failed",V.job.lastError?.message),Y.options.pauseOnError)this.pause(L),this.warning(`Queue '${L}' paused due to job failure`)}else if(V.job.status==="completed")this.success(`Job ${V.job.id} completed successfully`),await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"completed");else if(V.job.status==="cancelled")this.info(`Job ${V.job.id} cancelled while running`),await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"cancelled")}catch(P){if(this.error(`Unexpected error processing job ${V.job.id}: ${P}`),await this.sendJobStatusUpdate(V.job.id,V.job.dispatchId,L,"failed",P instanceof Error?P.message:String(P)),Y.options.pauseOnError)this.pause(L),this.warning(`Queue '${L}' paused due to unexpected error`)}finally{if(X?.overlapBehavior==="no_overlap")if(X.overlapDelay&&X.overlapDelay>0)setTimeout(()=>{this.transport.releaseJobLock(V.job.id,L)},X.overlapDelay);else await this.transport.releaseJobLock(V.job.id,L);Y.activeJobs.delete(V.job.id),Y.processing--,this.processQueueJobs(L)}}async createQueue(L){let Y=await super.createQueue(L);if(this.isRunning&&!this.registeredQueues.has(L.name))this.registeredQueues.add(L.name),await this.transport.registerWorker(this.id,Array.from(this.registeredQueues));return Y}async handleTransportMessage(L){try{switch(L.type){case"job:process":{let{job:Y,args:H=[],queue:U="default",options:V={},jobId:z,dispatchId:X}=L;await this.handleProcessJob(z,X,Y,H,U,V);break}case"job:cancel":{let{jobId:Y,dispatchId:H,queue:U="default"}=L;if(await this.cancelJob(Y,U))await this.sendJobStatusUpdate(Y,H,U,"cancelled");break}case"job:cancelAll":{let{queue:Y="default"}=L;await this.cancelAllJobs(Y);break}}}catch(Y){this.error(`Error handling transport message: ${Y}`)}}async handleProcessJob(L,Y,H,U,V,z){try{let X=d.get(H);if(!X)throw new Error(`Job class not found: ${H}`);let P=new(k(X,L,Y))(...U||[]);P.queueName=V;let Z={};if(z.scheduledFor)Z.scheduledFor=typeof z.scheduledFor==="string"?new Date(z.scheduledFor):z.scheduledFor;if(z.priority!==void 0)Z.priority=Number(z.priority);if(z.maxRetries!==void 0)Z.maxRetries=Number(z.maxRetries);if(z.retryDelay!==void 0)Z.retryDelay=Number(z.retryDelay);if(Z.scheduledFor&&Z.scheduledFor>new Date)await this.sendJobStatusUpdate(P.id,P.dispatchId,V,"scheduled_for_retry"),this.debug(`Scheduled job ${P.id} for future execution at ${Z.scheduledFor.toISOString()}`);else await this.sendJobStatusUpdate(P.id,P.dispatchId,V,"pending"),this.debug(`Added job ${P.id} to queue ${V} with initial status: pending`);await this.addJob(P,V,Z)}catch(X){throw this.error(`Error processing job ${H}: ${X.message}`),this.sendJobStatusUpdate(L,Y,V,"failed",X.message),X}}}export{w as WorkerStatus,x as TransportMode,h as RedisWorker,D as RedisTransport,E as Queue,T as JobStatus,F as JobOverlapBehavior,K as Job,y as Heracles,M as BaseWorker};

package/dist/job.d.ts

@@ -0,0 +1,249 @@
+import 'reflect-metadata';
+import type { Class } from 'type-fest';
+import { InteractsWithConsole } from '@elysiumjs/core';
+/**
+ * Defines how a job should behave when multiple instances with the same ID are dispatched.
+ * @author Axel Nana <axel.nana@workbud.com>
+ */
+export declare enum JobOverlapBehavior {
+    /**
+     * Allow jobs with the same ID to execute concurrently.
+     */
+    ALLOW_OVERLAP = "allow_overlap",
+    /**
+     * Only allow one job with a given ID to execute at a time.
+     * Additional jobs with the same ID will be queued and executed in sequence.
+     */
+    NO_OVERLAP = "no_overlap"
+}
+/**
+ * Represents job metadata when registered.
+ * @author Axel Nana <axel.nana@workbud.com>
+ */
+export interface JobMetadata {
+    /**
+     * The service name of the job in the container.
+     */
+    name: string;
+    /**
+     * The name of the queue the job is assigned to.
+     */
+    queue: string;
+    /**
+     * Defines how the job handles overlap with other instances of the same job ID.
+     */
+    overlapBehavior: JobOverlapBehavior;
+    /**
+     * When using NO_OVERLAP behavior, specifies the delay in milliseconds
+     * between sequential executions of jobs with the same ID.
+     */
+    overlapDelay: number;
+    /**
+     * Priority of this job (lower number = higher priority).
+     */
+    priority: number;
+    /**
+     * The maximum number of retries for this job type.
+     * This overrides queue-level retry settings.
+     * @default undefined - Uses queue setting
+     */
+    maxRetries?: number;
+    /**
+     * The delay between retries in milliseconds.
+     * This overrides queue-level retry delay settings.
+     * @default undefined - Uses queue setting
+     */
+    retryDelay?: number;
+}
+/**
+ * Properties required when declaring a job using the `@Job.register()` decorator.
+ * @author Axel Nana <axel.nana@workbud.com>
+ */
+export type JobProps = {
+    /**
+     * The name of the job.
+     */
+    name?: string;
+    /**
+     * The queue on which the job should be processed.
+     */
+    queue?: string;
+    /**
+     * Priority of this job (lower number = higher priority).
+     * @default 0
+     */
+    priority?: number;
+    /**
+     * The maximum number of retries for this job type.
+     * This overrides queue-level retry settings.
+     * @default undefined - Uses queue setting
+     */
+    maxRetries?: number;
+    /**
+     * The delay between retries in milliseconds.
+     * This overrides queue-level retry delay settings.
+     * @default undefined - Uses queue setting
+     */
+    retryDelay?: number;
+    /**
+     * Defines how the job handles overlap with other instances of the same job ID.
+     * @default JobOverlapBehavior.ALLOW_OVERLAP
+     */
+    overlapBehavior?: JobOverlapBehavior;
+    /**
+     * When using NO_OVERLAP behavior, specifies the delay in milliseconds
+     * between sequential executions of jobs with the same ID.
+     * @default 0
+     */
+    overlapDelay?: number;
+};
+/**
+ * Type for any job classes.
+ * @author Axel Nana <axel.nana@workbud.com>
+ */
+export type JobClass<T extends Job = Job, TClass extends Class<T> = Class<T>> = {
+    /**
+     * Description of the job.
+     */
+    readonly description: string;
+    /**
+     * Generates the ID for instances of this Job.
+     */
+    generateJobId(...args: ConstructorParameters<TClass>): string;
+    new (...args: ConstructorParameters<TClass>): T;
+};
+/**
+ * Base class for background jobs.
+ *
+ * Jobs are classes that can be executed in the background. They provide
+ * an abstract execute method that serves as the entry point for the job.
+ *
+ * @author Axel Nana <axel.nana@workbud.com>
+ */
+export declare abstract class Job extends InteractsWithConsole {
+    #private;
+    /**
+     * Marks a class as a job.
+     * @author Axel Nana <axel.nana@workbud.com>
+     * @param options The decorator options.
+     */
+    static register(options?: JobProps): (target: JobClass) => void;
+    /**
+     * Generate a random job ID.
+     * @param args Arguments passed to the constructor.
+     * @returns A random job ID.
+     */
+    static generateJobId(..._args: any[]): string;
+    /**
+     * Description of the job.
+     */
+    static readonly description: string;
+    /**
+     * Unique identifier for the job.
+     */
+    readonly id: string;
+    /**
+     * The dispatch ID for this job instance.
+     * This is unique for each dispatch of the job, even if the job ID is the same.
+     */
+    readonly dispatchId: string;
+    /**
+     * The timestamp when the job was created.
+     */
+    readonly createdAt: Date;
+    /**
+     * The timestamp when the job started execution.
+     */
+    get startedAt(): Date | undefined;
+    /**
+     * The timestamp when the job completed execution.
+     */
+    get completedAt(): Date | undefined;
+    /**
+     * The current status of the job.
+     */
+    get status(): JobStatus;
+    /**
+     * The error that occurred during job execution, if any.
+     */
+    get lastError(): Error | undefined;
+    /**
+     * The number of times this job has been retried.
+     */
+    get retries(): number;
+    /**
+     * The name of the queue this job is assigned to.
+     */
+    get queueName(): string | undefined;
+    /**
+     * Set the queue for this job instance.
+     * @internal This should only be called by the queue implementation.
+     */
+    set queueName(value: string | undefined);
+    /**
+     * Increment the retry count for this job.
+     * @internal This should only be called by the worker implementation.
+     */
+    incrementRetries(): void;
+    /**
+     * Creates a new job instance.
+     *
+     * @param id Optional job ID. If not provided, a random ID will be generated.
+     */
+    constructor(id?: string, dispatchId?: string);
+    /**
+     * Run the job.
+     * This method handles the job lifecycle and calls the execute method.
+     * @returns A promise that resolves when the job completes (successfully or not).
+     */
+    run(): Promise<void>;
+    /**
+     * Cancels the job.
+     * @returns True if the job was cancelled, false if it was already in a terminal state.
+     */
+    cancel(): boolean;
+    /**
+     * The main execution method for the job.
+     * This must be implemented by all job classes.
+     */
+    protected abstract execute(): Promise<void>;
+}
+/**
+ * Wraps a Job class to assign a specific ID to all instances.
+ * @author Axel Nana <axel.nana@workbud.com>
+ * @internal
+ * @param J The Job class.
+ * @param id The ID to assign to the job.
+ * @returns A Job class which creates job instances with the specified ID.
+ */
+export declare const WithId: <T extends Job, TClass extends Class<T> = Class<T>>(J: TClass, id: string, dispatchId: string) => TClass;
+/**
+ * Possible statuses for a job.
+ * @author Axel Nana <axel.nana@workbud.com>
+ */
+export declare enum JobStatus {
+    /**
+     * The job is in pending state; queued to run, but not started.
+     */
+    PENDING = "pending",
+    /**
+     * The job is currently running.
+     */
+    RUNNING = "running",
+    /**
+     * The job has completed successfully.
+     */
+    COMPLETED = "completed",
+    /**
+     * The job has failed to complete successfully. No retries left.
+     */
+    FAILED = "failed",
+    /**
+     * The job was cancelled and won't be retried or completed.
+     */
+    CANCELLED = "cancelled",
+    /**
+     * The job is scheduled to be retried after a failure.
+     */
+    SCHEDULED_FOR_RETRY = "scheduled_for_retry"
+}