@elysiumjs/heracles 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/dist/index.js +1 -1
  2. package/package.json +46 -44
  3. package/.turbo/turbo-build.log +0 -8
package/dist/index.js CHANGED
@@ -1,2 +1,2 @@
1
1
  // @bun
2
- import"@elysiumjs/core";import"reflect-metadata";import{InteractsWithConsole as S,Service as h}from"@elysiumjs/core";import{assign as l,uid as W}from"radash";var E;(($)=>($.job=Symbol("elysium:job"),$.queue=Symbol("elysium:queue"),$.worker=Symbol("elysium:worker"),$.transport=Symbol("elysium:transport"),$.jobTransaction=Symbol("elysium:jobTransaction"),$.workerPool=Symbol("elysium:workerPool")))(E||={});var Q=(L)=>{if(!Reflect.hasMetadata(E.job,L))return null;return Reflect.getMetadata(E.job,L)};var G;((F)=>{F.ALLOW_OVERLAP="allow_overlap";F.NO_OVERLAP="no_overlap"})(G||={});class O extends S{#Y;#F;#L="pending";#H;#z=0;#U;static register(L={}){return function(Y){L=l({name:Y.name,queue:"default",overlapBehavior:"allow_overlap",overlapDelay:0,priority:0},L);let F=`elysium.heracles.job.${L.name}`;h.instance(F,Y);let H={name:F,queue:L.queue,overlapBehavior:L.overlapBehavior,overlapDelay:L.overlapDelay,priority:L.priority,maxRetries:L.maxRetries,retryDelay:L.retryDelay};Reflect.defineMetadata(E.job,H,Y)}}static generateJobId(...L){return`job_${Date.now()}_${W(8)}`}static description="An Heracles job.";id;dispatchId;createdAt;get startedAt(){return this.#Y}get completedAt(){return this.#F}get status(){return this.#L}get lastError(){return this.#H}get retries(){return this.#z}get queueName(){return this.#U}set queueName(L){this.#U=L}incrementRetries(){this.#z++}constructor(L,Y){super();this.id=L??this.constructor.generateJobId(),this.dispatchId=Y??`dispatch_${Date.now()}_${W(8)}`,this.createdAt=new Date}async run(){if(this.#L==="cancelled")return;try{this.#L="running",this.#Y=new Date;let L=this.#U?` in queue '${this.#U}'`:"";if(this.debug(`Job ${this.id}${L} started at ${this.#Y.toISOString()}`),await this.execute(),this.#L==="running")this.#L="completed",this.#F=new Date,this.debug(`Job ${this.id} completed successfully at ${this.#F.toISOString()} after ${this.retries} retries`)}catch(L){this.fail(L instanceof Error?L:Error(String(L)))}}fail(L){if(this.#H=L,this.#L!=="cancelled")this.#L="failed",this.#F=new Date,this.error(`Job ${this.id} failed at ${this.#F.toISOString()} after ${this.retries} retries: ${this.#H.message}`),this.trace(this.#H)}cancel(){if(this.#L==="completed"||this.#L==="failed")return!1;let L=this.#L==="running";if(this.#L="cancelled",L)this.#F=new Date;return this.info(`Job ${this.id} was cancelled${L?" while running":" before execution"}`),!0}}var C=(L,Y,F)=>{return class extends L{constructor(...H){super(...H);this.id=Y,this.dispatchId=F}}},A;((V)=>{V.PENDING="pending";V.RUNNING="running";V.COMPLETED="completed";V.FAILED="failed";V.CANCELLED="cancelled";V.SCHEDULED_FOR_RETRY="scheduled_for_retry"})(A||={});import{Redis as c,Service as j}from"@elysiumjs/core";var K;((F)=>{F.CONSUMER="consumer";F.PRODUCER="producer"})(K||={});import{InteractsWithConsole as J,Redis as T}from"@elysiumjs/core";import{uid as w}from"radash";class B extends J{commandClient=null;subscriberClient=null;publisherClient=null;options;mode;messageHandlers=[];consumerId;isRunning=!1;cleanupTimer;jobLocks=new Map;constructor(L,Y={}){super();this.mode=L,this.options={connection:Y.connection||"default",keyPrefix:Y.keyPrefix||"elysium:heracles",consumerGroup:Y.consumerGroup||"workers",consumerName:Y.consumerName||`worker-${w(8)}`,batchSize:Y.batchSize||10,statusTTL:Y.statusTTL||86400,cleanupCompletedJobs:Y.cleanupCompletedJobs!==!1,completedJobRetention:Y.completedJobRetention||3600,maxStreamSize:Y.maxStreamSize||1000},this.consumerId=this.mode==="consumer"?this.options.consumerName:`producer-${w(8)}`}async initializeClients(){this.commandClient=await T.getConnection(this.options.connection).duplicate(),this.subscriberClient=await T.getConnection(this.options.connection).duplicate(),this.publisherClient=await T.getConnection(this.options.connection).duplicate(),this.setupConnectionHandlers()}setupConnectionHandlers(){this.commandClient.onconnect=()=>{this.debug("Command client connected")},this.commandClient.onclose=(L)=>{this.error(`Command client disconnected: ${L}`)},this.subscriberClient.onconnect=()=>{this.debug("Subscriber client connected")},this.subscriberClient.onclose=(L)=>{this.error(`Subscriber client disconnected: ${L}`)},this.publisherClient.onconnect=()=>{this.debug("Publisher client connected")},this.publisherClient.onclose=(L)=>{this.error(`Publisher client disconnected: ${L}`)}}async start(){if(this.isRunning)return;try{if(await this.initializeClients(),await Promise.all([this.commandClient.connect(),this.subscriberClient.connect(),this.publisherClient.connect()]),await this.commandClient.ping(),await this.publisherClient.ping(),this.info(`RedisTransport started in ${this.mode} mode with ID: ${this.consumerId}`),this.mode==="consumer"){if(await this.subscribeToChannels(),this.options.cleanupCompletedJobs)this.scheduleCleanup()}this.isRunning=!0}catch(L){throw this.error(`Failed to start RedisTransport: ${L}`),Error(`Redis connection failed: ${L}`)}}async stop(){if(this.isRunning=!1,this.cleanupTimer)clearInterval(this.cleanupTimer),this.cleanupTimer=void 0;if(this.mode==="consumer")await this.subscriberClient.unsubscribe();this.commandClient.close(),this.subscriberClient.close(),this.publisherClient.close(),this.info("RedisTransport stopped")}async subscribeToChannels(){let L=await this.getRegisteredQueues();for(let Y of L)await this.subscriberClient.subscribe(`${this.options.keyPrefix}:queue:${Y}:new`,async(F,H)=>{this.debug(`New job notification for queue ${Y}: ${F}`),await this.processJobFromStream(Y,F)}),await this.subscriberClient.subscribe(`${this.options.keyPrefix}:queue:${Y}:control`,async(F,H)=>{try{let U=JSON.parse(F);await this.handleControlMessage(U)}catch(U){this.error(`Error parsing control message: ${U}`)}}),await this.subscriberClient.subscribe(`${this.options.keyPrefix}:lock:${Y}:released`,(F,H)=>{this.debug(`Lock released for job ${F} in queue ${Y}`),this.notifyLockRelease(F,Y)});await this.subscriberClient.subscribe(`${this.options.keyPrefix}:worker:coordination`,async(Y,F)=>{try{let H=JSON.parse(Y);await this.handleWorkerCoordination(H)}catch(H){this.error(`Error handling worker coordination: ${H}`)}})}async processJobFromStream(L,Y){let F=this.getStreamKey(L);try{let H=await this.commandClient.send("XRANGE",[F,Y,Y]);if(H&&H.length>0){let[U,z]=H[0],V=this.deserializeMessage(z,L);for(let X of this.messageHandlers)try{await X(V)}catch(Z){this.error(`Error in message handler: ${Z}`)}await this.commandClient.send("XACK",[F,this.options.consumerGroup,Y])}}catch(H){this.error(`Error processing job from stream: ${H}`)}}async handleControlMessage(L){switch(L.type){case"pause":this.info(`Queue ${L.queue} paused`);break;case"resume":this.info(`Queue ${L.queue} resumed`);break;case"drain":this.info(`Queue ${L.queue} draining`);break;default:this.debug(`Unknown control message type: ${L.type}`)}}async handleWorkerCoordination(L){switch(L.type){case"worker:joined":this.debug(`Worker ${L.workerId} joined`);break;case"worker:left":this.debug(`Worker ${L.workerId} left`);break;case"rebalance":this.debug("Rebalancing work across workers");break}}async send(L){try{let Y=this.extractQueueName(L),F=this.getStreamKey(Y);if(L.type==="job:process"){let H=this.serializeMessage(L),U=await this.commandClient.send("XADD",[F,"*",...H]);await this.indexJob({jobId:L.jobId,dispatchId:L.dispatchId,queue:Y,status:"pending",priority:L.options?.priority||10,timestamp:Date.now()}),await this.storeJobStatus({jobId:L.jobId,dispatchId:L.dispatchId,queue:Y,status:"pending",retries:0,createdAt:new Date().toISOString(),updatedAt:new Date().toISOString(),messageId:U}),await this.publisherClient.publish(`${this.options.keyPrefix}:queue:${Y}:new`,U),this.debug(`Job ${L.jobId} dispatched to queue ${Y} with ID ${U}`)}else if(L.type==="job:status"||L.type==="job:result")await this.updateJobStatus(L.jobId,L.dispatchId,Y,{status:L.status,error:L.error,retries:L.retries,completedAt:L.completedAt,updatedAt:new Date().toISOString()}),await this.updateJobIndex(L.jobId,L.dispatchId,Y,L.status),await this.publisherClient.publish(`${this.options.keyPrefix}:job:${L.jobId}:status`,JSON.stringify({status:L.status,error:L.error,timestamp:Date.now()}));else if(L.type==="job:cancel")await this.publisherClient.publish(`${this.options.keyPrefix}:queue:${Y}:control`,JSON.stringify({type:"cancel",jobId:L.jobId,dispatchId:L.dispatchId}));else{let H=this.serializeMessage(L);await this.commandClient.send("XADD",[F,"*",...H])}}catch(Y){throw this.error(`Failed to send message: ${Y}`),Error(`Failed to send message: ${Y}`)}}onMessage(L){this.messageHandlers.push(L)}async getJobStatus(L,Y,F){try{let H=this.getJobStatusKey(L,Y,F),U=await this.commandClient.hmget(H,["jobId","dispatchId","queue","status","error","retries","createdAt","startedAt","completedAt","updatedAt","messageId"]);if(U&&U[0])return{jobId:U[0],dispatchId:U[1],queue:U[2]||F,status:U[3]||"unknown",error:U[4],retries:parseInt(U[5]||"0",10),createdAt:U[6]||new Date().toISOString(),startedAt:U[7],completedAt:U[8],updatedAt:U[9],messageId:U[10]};let z=await this.getJobFromIndex(L,Y,F);if(z)return{jobId:L,dispatchId:Y,queue:F,status:z.status,retries:0,createdAt:new Date(z.timestamp).toISOString()};return{jobId:L,dispatchId:Y,queue:F,status:"unknown",retries:0,createdAt:new Date().toISOString()}}catch(H){return this.warning(`Failed to get job status: ${H}`),{jobId:L,dispatchId:Y,queue:F,status:"unknown",retries:0,createdAt:new Date().toISOString()}}}async updateJobStatus(L,Y,F,H){try{let U=this.getJobStatusKey(L,Y,F),z=[];if(H.status)z.push("status",H.status);if(H.error!==void 0)z.push("error",H.error||"");if(H.retries!==void 0)z.push("retries",H.retries.toString());if(H.startedAt)z.push("startedAt",H.startedAt);if(H.completedAt)z.push("completedAt",H.completedAt);if(H.updatedAt)z.push("updatedAt",H.updatedAt);if(z.length>0){if(await this.commandClient.hmset(U,z),await this.commandClient.expire(U,this.options.statusTTL),H.status)await this.updateJobIndex(L,Y,F,H.status);await this.publisherClient.publish(`${this.options.keyPrefix}:job:${L}:status`,JSON.stringify({...H,timestamp:Date.now()})),this.debug(`Updated job ${L} status: ${H.status||"fields updated"}`)}}catch(U){throw this.error(`Failed to update job status: ${U}`),U}}async registerWorker(L,Y){try{for(let H of Y){let U=this.getStreamKey(H);await this.ensureConsumerGroupExists(U)}let F=this.getWorkerKey(L);await this.commandClient.hmset(F,["id",L,"status","active","lastSeen",new Date().toISOString(),"queues",JSON.stringify(Y)]),await this.commandClient.expire(F,60),await this.publisherClient.publish(`${this.options.keyPrefix}:worker:coordination`,JSON.stringify({type:"worker:joined",workerId:L,queues:Y,timestamp:Date.now()})),this.debug(`Registered worker ${L} for queues: ${Y.join(", ")}`)}catch(F){throw this.error(`Failed to register worker: ${F}`),F}}async unregisterWorker(L){try{let Y=this.getWorkerKey(L);await this.commandClient.del(Y),await this.publisherClient.publish(`${this.options.keyPrefix}:worker:coordination`,JSON.stringify({type:"worker:left",workerId:L,timestamp:Date.now()})),this.debug(`Unregistered worker ${L}`)}catch(Y){throw this.error(`Failed to unregister worker: ${Y}`),Y}}async indexJob(L){let Y=L.timestamp,F=`${L.jobId}:${L.dispatchId}`;await Promise.all([this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:queue:${L.queue}`,Y.toString(),F]),this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:status:${L.status}`,Y.toString(),F]),this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:priority`,L.priority.toString(),F])])}async updateJobIndex(L,Y,F,H){let U=`${L}:${Y}`,z=Date.now(),X=["pending","running","completed","failed","cancelled","scheduled_for_retry"].filter((_)=>_!==H).map((_)=>this.commandClient.send("ZREM",[`${this.options.keyPrefix}:idx:status:${_}`,U])),Z=this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:status:${H}`,z.toString(),U]);await Promise.all([...X,Z])}async getJobFromIndex(L,Y,F){let H=`${L}:${Y}`,U=await this.commandClient.zscore(`${this.options.keyPrefix}:idx:queue:${F}`,H);if(U){let z=["pending","running","completed","failed","cancelled","scheduled_for_retry"];for(let V of z)if(await this.commandClient.zscore(`${this.options.keyPrefix}:idx:status:${V}`,H))return{jobId:L,dispatchId:Y,queue:F,status:V,priority:10,timestamp:parseInt(U,10)}}return null}async storeJobStatus(L){let Y=this.getJobStatusKey(L.jobId,L.dispatchId,L.queue),F=["jobId",L.jobId,"dispatchId",L.dispatchId,"queue",L.queue,"status",L.status,"retries",L.retries.toString(),"createdAt",L.createdAt,"updatedAt",L.updatedAt||Date.now().toString()];if(L.error)F.push("error",L.error);if(L.startedAt)F.push("startedAt",L.startedAt);if(L.completedAt)F.push("completedAt",L.completedAt);if(L.messageId)F.push("messageId",L.messageId);await this.commandClient.hmset(Y,F),await this.commandClient.expire(Y,this.options.statusTTL)}async ensureConsumerGroupExists(L){try{await this.commandClient.send("XGROUP",["CREATE",L,this.options.consumerGroup,"0","MKSTREAM"]),this.debug(`Created consumer group ${this.options.consumerGroup} for ${L}`)}catch(Y){if(!Y.message?.includes("BUSYGROUP"))this.error(`Error creating consumer group: ${Y}`)}}async getRegisteredQueues(){try{let L=`${this.options.keyPrefix}:stream:*`,Y=await this.commandClient.keys(L);if(!Array.isArray(Y)||Y.length===0)return["default"];return Y.map((F)=>F.replace(`${this.options.keyPrefix}:stream:`,"")).filter(Boolean)}catch(L){return this.error(`Failed to get registered queues: ${L}`),["default"]}}scheduleCleanup(){this.cleanupTimer=setInterval(async()=>{try{await this.performCleanup()}catch(Y){this.error(`Error during cleanup: ${Y}`)}},300000),this.debug("Scheduled cleanup every 300 seconds")}async performCleanup(){this.debug("Starting cleanup operation");let L=Date.now()-this.options.completedJobRetention*1000,Y=await this.getRegisteredQueues(),F=["completed","failed","cancelled"];for(let z of F){let V=`${this.options.keyPrefix}:idx:status:${z}`;await this.commandClient.send("ZREMRANGEBYSCORE",[V,"-inf",L.toString()])}for(let z of Y){let V=this.getStreamKey(z),X=await this.commandClient.send("XLEN",[V]);if(X>this.options.maxStreamSize)await this.commandClient.send("XTRIM",[V,"MAXLEN","~",this.options.maxStreamSize.toString()]),this.debug(`Trimmed stream ${V} from ${X} to ~${this.options.maxStreamSize}`)}let H=`${this.options.keyPrefix}:status:*`,U=await this.commandClient.keys(H);for(let z of U){let V=await this.commandClient.ttl(z);if(V===-1||V===-2)await this.commandClient.del(z)}this.debug("Cleanup operation completed")}extractQueueName(L){if("queue"in L&&typeof L.queue==="string")return L.queue;return"default"}serializeMessage(L){let Y=["type",L.type,"timestamp",Date.now().toString()];if("queue"in L&&L.queue)Y.push("queue",L.queue);switch(L.type){case"job:process":if(Y.push("job",L.job,"args",JSON.stringify(L.args||[]),"jobId",L.jobId,"dispatchId",L.dispatchId),L.options)Y.push("options",JSON.stringify(L.options));break;case"job:cancel":case"job:status":case"job:result":if(Y.push("jobId",L.jobId),Y.push("dispatchId",L.dispatchId),L.type!=="job:cancel"){if(L.status)Y.push("status",L.status);if(L.error)Y.push("error",L.error);if(L.retries!==void 0)Y.push("retries",L.retries.toString());if("completedAt"in L&&L.completedAt)Y.push("completedAt",L.completedAt)}break;case"worker:register":if(Y.push("workerId",L.workerId),L.queues)Y.push("queues",JSON.stringify(L.queues));break;case"worker:unregister":Y.push("workerId",L.workerId);break}return Y}deserializeMessage(L,Y){let F={};for(let z=0;z<L.length;z+=2)F[L[z]]=L[z+1];let H=F.type,U={type:H,queue:F.queue||Y};switch(H){case"job:process":return{...U,type:"job:process",job:F.job,args:JSON.parse(F.args||"[]"),jobId:F.jobId,dispatchId:F.dispatchId,options:F.options?JSON.parse(F.options):void 0};case"job:cancel":return{...U,type:"job:cancel",jobId:F.jobId,dispatchId:F.dispatchId};case"job:status":return{...U,type:"job:status",jobId:F.jobId,dispatchId:F.dispatchId,status:F.status,error:F.error,retries:F.retries?parseInt(F.retries,10):void 0};case"job:result":return{...U,type:"job:result",jobId:F.jobId,dispatchId:F.dispatchId,status:F.status,error:F.error,completedAt:F.completedAt||new Date().toISOString()};default:return U}}notifyLockRelease(L,Y){this.jobLocks.delete(L);for(let F of this.messageHandlers)try{F({type:"job:update",jobId:L,dispatchId:"",queue:Y,status:"lock_released",updates:{}})}catch(H){this.error(`Error notifying lock release: ${H}`)}}async isJobLocked(L,Y){let F=this.jobLocks.get(L);if(F&&F.lockedUntil>Date.now())return!0;let H=`${this.options.keyPrefix}:lock:${Y}:${L}`;return await this.commandClient.exists(H)}async acquireJobLock(L,Y,F=60000){let H=`${this.options.keyPrefix}:lock:${Y}:${L}`,U=`${this.consumerId}:${Date.now()}`;if(await this.commandClient.set(H,U,"PX",F.toString(),"NX")==="OK")return this.jobLocks.set(L,{lockedUntil:Date.now()+F,queue:Y}),this.debug(`Acquired lock for job ${L} in queue ${Y}`),!0;return!1}async releaseJobLock(L,Y){let F=`${this.options.keyPrefix}:lock:${Y}:${L}`;await this.commandClient.del(F),this.jobLocks.delete(L),await this.publisherClient.publish(`${this.options.keyPrefix}:lock:${Y}:released`,L),this.debug(`Released lock for job ${L} in queue ${Y}`)}getStreamKey(L){return`${this.options.keyPrefix}:stream:${L||"default"}`}getJobStatusKey(L,Y,F){return`${this.options.keyPrefix}:status:${F}:${L}:${Y}`}getWorkerKey(L){return`${this.options.keyPrefix}:worker:${L}`}}import{InteractsWithConsole as I}from"@elysiumjs/core";import{sort as M,uid as N}from"radash";var v;((X)=>{X.IDLE="idle";X.ACTIVE="active";X.PAUSED="paused";X.DRAINING="draining";X.STOPPING="stopping";X.STOPPED="stopped";X.DISCONNECTED="disconnected"})(v||={});class P extends I{#Y;transport=null;queues=new Map;defaultQueueName="default";defaultTransport;workerStatus="idle";startedAt=new Date;scheduledJobInterval=1000;scheduledJobTimer;constructor(L){super();this.#Y=L??N(24),this.createQueue({name:this.defaultQueueName})}get id(){return this.#Y}get status(){return this.workerStatus}async createQueue(L){if(this.queues.has(L.name))return this.debug(`Queue with name '${L.name}' already exists`),this;let Y={options:{...L,concurrency:L.concurrency||1,maxRetries:L.maxRetries||0,retryDelay:L.retryDelay||1000,pauseOnError:L.pauseOnError||!1},jobs:[],activeJobs:new Map,retryingJobs:new Map,paused:!1,draining:!1};return this.queues.set(L.name,Y),this.debug(`Created queue '${L.name}'`),this}async addJob(L,Y=this.defaultQueueName,F={}){let H=this.queues.get(Y);if(!H)throw this.error(`Queue '${Y}' does not exist`),Error(`Queue '${Y}' does not exist`);if(H.draining)throw this.warning(`Queue '${Y}' is draining, job ${L.id} will not be added`),Error(`Queue '${Y}' is draining`);L.queueName=Y;let U={job:L,retries:L.retries,queueName:Y,enqueuedAt:new Date,scheduledFor:F.scheduledFor,priority:F.priority??10,maxRetries:F.maxRetries,retryDelay:F.retryDelay,overlapBehavior:F.overlapBehavior,overlapDelay:F.overlapDelay};if(H.jobs.push(U),H.jobs=M(M(M(H.jobs,(z)=>z.enqueuedAt.getTime()),(z)=>z.scheduledFor?z.scheduledFor.getTime():0),(z)=>z.priority??10),this.debug(`Added job ${L.id} to queue '${Y}'`),this.workerStatus==="idle")this.workerStatus="active";if(!H.paused&&H.activeJobs.size<H.options.concurrency)try{this.processQueueJobs(Y)}catch(z){this.error(`Error processing next job in queue '${Y}': ${z}`)}return this}async pause(L){if(L){let Y=this.queues.get(L);if(Y)Y.paused=!0,this.info(`Paused queue '${L}'`);else this.warning(`Cannot pause non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.pause(Y);if(Array.from(this.queues.values()).every((Y)=>Y.paused)&&Array.from(this.queues.values()).every((Y)=>Y.activeJobs.size===0))this.workerStatus="paused",this.info(`Worker ${this.id} is now paused`)}return this}async resume(L){if(L){let Y=this.queues.get(L);if(Y){if(Y.paused=!1,this.info(`Resumed queue '${L}'`),this.workerStatus==="paused")this.workerStatus="active";await this.processQueueJobs(L)}else this.warning(`Cannot resume non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.resume(Y);if(Array.from(this.queues.values()).some((Y)=>Y.jobs.length>0||Y.activeJobs.size>0))this.workerStatus="active",this.info(`Worker ${this.id} is now active`)}return this}async drain(L){if(L){let Y=this.queues.get(L);if(Y)Y.draining=!0,this.info(`Draining queue '${L}'`);else this.warning(`Cannot drain non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.drain(Y);this.workerStatus="draining",this.info(`Worker ${this.id} is now draining`)}return this}async clear(L){if(L){let Y=this.queues.get(L);if(Y){let F=Y.jobs.length;for(let H of Y.jobs)H.job.cancel();Y.jobs=[],this.info(`Cleared ${F} jobs from queue '${L}'`)}else this.warning(`Cannot clear non-existent queue '${L}'`)}else{for(let[Y,F]of this.queues.entries()){let H=F.jobs.length;for(let U of F.jobs)U.job.cancel();F.jobs=[],this.info(`Cleared ${H} jobs from queue '${Y}'`)}if(Array.from(this.queues.values()).every((Y)=>Y.jobs.length===0&&Y.activeJobs.size===0))this.workerStatus="idle"}return this}async size(L=this.defaultQueueName){let Y=this.queues.get(L);if(!Y)throw Error(`Queue '${L}' does not exist`);return Y.jobs.length+Y.activeJobs.size}async totalSize(){let L=0;for(let Y of this.queues.values())L+=Y.jobs.length+Y.activeJobs.size;return L}async getStats(){let L={};for(let[Y,F]of this.queues.entries())L[Y]={waiting:F.jobs.length,retrying:F.retryingJobs.size,processing:F.activeJobs.size,paused:F.paused,draining:F.draining};return L}getInfo(){let L=0;for(let U of this.queues.values())L+=U.activeJobs.size;let Y=Array.from(this.queues.keys()),F=this.defaultTransport?this.defaultTransport.constructor.name:"None",H=async(U,z)=>{try{return await this.addJob(U,z),!0}catch(V){return!1}};return{id:this.id,status:this.workerStatus,queues:Y,activeJobs:L,transport:F,startedAt:this.startedAt,addJob:H,cancelJob:this.cancelJob.bind(this),cancelAllJobs:this.cancelAllJobs.bind(this)}}async cancelJob(L,Y){if(Y)return this.cancelJobInQueue(L,Y);else{for(let F of this.queues.keys())if(await this.cancelJobInQueue(L,F))return!0;return!1}}async cancelAllJobs(L){let Y=0;if(L){let F=this.queues.get(L);if(F){Y=F.jobs.length;for(let H of F.jobs)H.job.cancel();for(let[H,U]of F.activeJobs.entries())if(U.job.cancel())Y++;this.info(`Canceled ${Y} jobs in queue '${L}'`)}}else for(let[F,H]of this.queues.entries()){let U=H.jobs.length;for(let z of H.jobs)z.job.cancel();for(let[z,V]of H.activeJobs.entries())if(V.job.cancel())U++;Y+=U,this.info(`Canceled ${U} jobs in queue '${F}'`)}if(Y>0&&Array.from(this.queues.values()).every((F)=>F.jobs.length===0&&F.activeJobs.size===0))this.workerStatus="idle";return Y}async getJob(L,Y){if(Y){let F=this.queues.get(Y);if(F){let H=F.jobs.find((z)=>z.job.id===L);if(H)return H.job;let U=F.activeJobs.get(L);if(U)return U.job}}else for(let F of this.queues.values()){let H=F.jobs.find((z)=>z.job.id===L);if(H)return H.job;let U=F.activeJobs.get(L);if(U)return U.job}return null}async start(){if(this.workerStatus="active",this.startedAt=new Date,this.defaultTransport)await this.defaultTransport.start();for(let L of this.queues.keys())await this.processQueueJobs(L);this.startScheduledJobTimer(),this.info(`Worker ${this.id} started`)}async stop(L=!1){if(this.workerStatus="stopping",this.stopScheduledJobTimer(),L){await this.cancelAllJobs();for(let Y of this.queues.keys())await this.clear(Y)}else{for(let F of this.queues.keys())await this.drain(F);let Y=()=>{return Array.from(this.queues.values()).every((F)=>F.activeJobs.size===0)};if(!Y()){this.info(`Worker ${this.id} waiting for ${Array.from(this.queues.values()).reduce((F,H)=>F+H.activeJobs.size,0)} jobs to complete`);while(!Y())await new Promise((F)=>setTimeout(F,100))}}if(this.defaultTransport)await this.defaultTransport.stop();this.workerStatus="stopped",this.info(`Worker ${this.id} stopped`)}async setConcurrency(L,Y){let F=this.queues.get(L);if(!F)throw this.error(`Cannot set concurrency: Queue '${L}' does not exist`),Error(`Queue '${L}' does not exist`);if(Y<1)this.warning(`Invalid concurrency value ${Y}, using 1 instead`),Y=1;return F.options.concurrency=Y,this.info(`Set concurrency for queue '${L}' to ${Y}`),this.processQueueJobs(L)}startScheduledJobTimer(){if(this.scheduledJobTimer)clearInterval(this.scheduledJobTimer);this.scheduledJobTimer=setInterval(()=>{this.checkScheduledJobs()},this.scheduledJobInterval)}stopScheduledJobTimer(){if(this.scheduledJobTimer)clearInterval(this.scheduledJobTimer),this.scheduledJobTimer=void 0}checkScheduledJobs(){let L=new Date;for(let[Y,F]of this.queues.entries()){if(F.paused||F.draining)continue;let H=F.jobs.filter((U)=>U.scheduledFor&&U.scheduledFor<=L);for(let U of H)U.scheduledFor=void 0;if(H.length>0&&F.activeJobs.size<F.options.concurrency)this.processQueueJobs(Y)}}async processQueueJobs(L){let Y=this.queues.get(L);if(!Y||Y.paused)return;let F=[];while(F.length<Y.options.concurrency)F.push(this.processNextJob(L));await Promise.all(F);let H=Array.from(this.queues.values());if(H.some((U)=>U.activeJobs.size>0))this.workerStatus="active";else if(H.every((U)=>U.jobs.length===0&&U.activeJobs.size===0))this.workerStatus="idle";if(H.some((U)=>U.jobs.length>0))await Bun.sleep(10),setImmediate(()=>this.processQueueJobs(L))}async sendJobStatusUpdate(L,Y,F,H,U){try{await this.transport.updateJobStatus(L,Y,F,{dispatchId:Y,status:H,error:U,retries:0,startedAt:H==="running"?new Date().toISOString():void 0,completedAt:H==="completed"||H==="failed"||H==="cancelled"?new Date().toISOString():void 0,updatedAt:Date.now().toString()})}catch(z){this.error(`Error sending job status update: ${z}`)}}async cancelJobInQueue(L,Y){let F=this.queues.get(Y);if(!F)return!1;let H=F.jobs.findIndex((U)=>U.job.id===L);if(H>=0){if(F.jobs[H].job.cancel())return F.jobs.splice(H,1),this.info(`Canceled job ${L} in queue '${Y}'`),!0}if(F.activeJobs.has(L)){if(F.activeJobs.get(L).job.cancel())return this.info(`Canceled running job ${L} in queue '${Y}'`),!0}return!1}}class f extends P{isRunning=!1;registeredQueues=new Set;pendingBatches=new Map;batchTimer;metrics={jobsProcessed:0,jobsFailed:0,jobsRetried:0,averageProcessingTime:0};metricsClient;metricsTimer;maxBatchSize;batchInterval;constructor(L="default",Y={}){super(Y.id);if(this.maxBatchSize=Y.maxBatchSize||10,this.batchInterval=Y.batchInterval||100,this.transport=new B("consumer",{connection:L,consumerName:`worker-${this.id}`,batchSize:this.maxBatchSize,...Y}),this.transport.onMessage(this.handleTransportMessage.bind(this)),Y.enableMetrics)this.initializeMetrics(L);this.debug(`RedisWorker ${this.id} created with batch size: ${this.maxBatchSize}`)}async initializeMetrics(L){this.metricsClient=await c.getConnection(L).duplicate()}async reportMetrics(){if(!this.metricsClient||!this.metricsClient.connected)return;let L=`worker:${this.id}:metrics`;try{await this.metricsClient.hmset(L,["jobsProcessed",this.metrics.jobsProcessed.toString(),"jobsFailed",this.metrics.jobsFailed.toString(),"jobsRetried",this.metrics.jobsRetried.toString(),"averageProcessingTime",this.metrics.averageProcessingTime.toString(),"lastProcessedAt",(this.metrics.lastProcessedAt||new Date).toISOString(),"timestamp",Date.now().toString()]),await this.metricsClient.expire(L,3600)}catch(Y){this.error(`Failed to report metrics: ${Y}`)}}async start(){if(this.isRunning)return;await super.start(),this.isRunning=!0;for(let L of this.queues.keys())this.registeredQueues.add(L);if(await this.transport.start(),this.metricsClient)this.metricsTimer=setInterval(()=>{this.reportMetrics()},1e4),await this.metricsClient.connect();await this.transport.registerWorker(this.id,Array.from(this.registeredQueues)),this.startBatchProcessing(),this.info(`Redis worker ${this.id} started for queues: ${Array.from(this.registeredQueues).join(", ")}`)}async stop(L=!1){if(this.isRunning=!1,this.stopBatchProcessing(),!L)await this.processAllPendingBatches();if(await super.stop(L),await this.transport.unregisterWorker(this.id),await this.transport.stop(),this.metricsClient)this.metricsClient.close(),clearInterval(this.metricsTimer),this.metricsClient=void 0;this.info(`Redis worker ${this.id} stopped`)}startBatchProcessing(){this.batchTimer=setInterval(()=>{this.processBatches()},this.batchInterval)}stopBatchProcessing(){if(this.batchTimer)clearInterval(this.batchTimer),this.batchTimer=void 0}async processAllPendingBatches(){for(let[L,Y]of this.pendingBatches)if(Y.jobs.length>0)await this.processBatch(Y);this.pendingBatches.clear()}async processBatches(){let L=Date.now();for(let[Y,F]of this.pendingBatches){if(!this.queues.get(Y))continue;if(F.jobs.length>=this.maxBatchSize||F.jobs.length>0&&L-F.timestamp>this.batchInterval)this.pendingBatches.delete(Y),await this.processBatch(F)}}async processBatch(L){let Y=this.queues.get(L.queue);if(!Y||Y.paused)return;let F=Date.now(),H=new Map,U=[];for(let Z of L.jobs)if(Q(Z.job.constructor)?.overlapBehavior==="no_overlap"){let R=Z.job.id;if(!H.has(R))H.set(R,[]);H.get(R).push(Z)}else U.push(Z);let z=U.map((Z)=>this.processJobInternal(Z,L.queue)),V=[];for(let[Z,_]of H)V.push(this.processOverlapJobs(_,L.queue));await Promise.all([...z,...V]);let X=Date.now()-F;this.updateMetrics(L.jobs.length,X),this.debug(`Processed batch of ${L.jobs.length} jobs from queue '${L.queue}' in ${X}ms`)}async processOverlapJobs(L,Y){for(let F of L)if(await this.transport.acquireJobLock(F.job.id,Y))try{await this.processJobInternal(F,Y)}finally{let U=Q(F.job.constructor);if(U?.overlapDelay&&U.overlapDelay>0)setTimeout(()=>{this.transport.releaseJobLock(F.job.id,Y)},U.overlapDelay);else await this.transport.releaseJobLock(F.job.id,Y)}else{let U=this.queues.get(Y);if(U)U.jobs.push(F)}}async processJobInternal(L,Y){let F=this.queues.get(Y);if(!F)return;F.activeJobs.set(L.job.id,L),await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,Y,"running");try{if(await L.job.run(),L.job.status==="completed")this.metrics.jobsProcessed++,await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,Y,"completed");else if(L.job.status==="failed")await this.handleJobFailure(L,F,Y);else if(L.job.status==="cancelled")await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,Y,"cancelled")}catch(H){this.error(`Error processing job ${L.job.id}: ${H}`),L.job.fail(H instanceof Error?H:Error(String(H))),await this.handleJobFailure(L,F,Y)}finally{F.activeJobs.delete(L.job.id),this.metrics.lastProcessedAt=new Date}}async handleJobFailure(L,Y,F){let H=L.maxRetries!==void 0?L.maxRetries:Y.options.maxRetries;if(L.retries<H){L.retries++,L.job.incrementRetries(),this.metrics.jobsRetried++;let U=L.retryDelay!==void 0?L.retryDelay:Y.options.retryDelay;await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,F,"scheduled_for_retry",L.job.lastError?.message),Y.retryingJobs.set(L.job.id,L),setTimeout(()=>{if(L.job.status!=="cancelled")this.addJobToBatch(L,F);Y.retryingJobs.delete(L.job.id)},U)}else if(this.metrics.jobsFailed++,await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,F,"failed",L.job.lastError?.message),Y.options.pauseOnError)await this.pause(F)}addJobToBatch(L,Y){if(!this.pendingBatches.has(Y))this.pendingBatches.set(Y,{queue:Y,jobs:[],timestamp:Date.now()});let F=this.pendingBatches.get(Y);if(F.jobs.push(L),F.jobs.length>=this.maxBatchSize)this.pendingBatches.delete(Y),this.processBatch(F)}async processNextJob(L){let Y=this.queues.get(L);if(!Y||Y.paused||Y.activeJobs.size>=Y.options.concurrency)return;let F=new Date,H=[];for(let U=0;U<Y.jobs.length&&H.length<this.maxBatchSize;U++){let z=Y.jobs[U];if(z.scheduledFor&&z.scheduledFor>F)continue;if(z.job.status==="cancelled"){Y.jobs.splice(U,1),U--;continue}H.push(z)}for(let U of H){let z=Y.jobs.indexOf(U);if(z>=0)Y.jobs.splice(z,1)}if(H.length>0){let U={queue:L,jobs:H,timestamp:Date.now()};if(H.length===1)await this.processBatch(U);else this.pendingBatches.set(L,U)}}async handleTransportMessage(L){try{switch(L.type){case"job:process":{let{job:Y,args:F=[],queue:H="default",options:U={},jobId:z,dispatchId:V}=L;await this.handleProcessJob(z,V,Y,F,H,U);break}case"job:cancel":{let{jobId:Y,queue:F="default"}=L;if(await this.cancelJob(Y,F))await this.sendJobStatusUpdate(Y,L.dispatchId,F,"cancelled");break}case"job:cancelAll":{let{queue:Y="default"}=L;await this.cancelAllJobs(Y);break}case"job:update":{if(L.status==="lock_released"){let Y=L.queue;await this.processQueueJobs(Y)}break}}}catch(Y){this.error(`Error handling transport message: ${Y}`)}}async handleProcessJob(L,Y,F,H,U,z){try{let V=j.get(F);if(!V)throw Error(`Job class not found: ${F}`);let X=new(C(V,L,Y))(...H||[]);X.queueName=U;let Z={};if(z.scheduledFor)Z.scheduledFor=typeof z.scheduledFor==="string"?new Date(z.scheduledFor):z.scheduledFor;if(z.priority!==void 0)Z.priority=Number(z.priority);if(z.maxRetries!==void 0)Z.maxRetries=Number(z.maxRetries);if(z.retryDelay!==void 0)Z.retryDelay=Number(z.retryDelay);let _=Z.scheduledFor&&Z.scheduledFor>new Date?"scheduled_for_retry":"pending";await this.sendJobStatusUpdate(X.id,X.dispatchId,U,_),await this.addJob(X,U,Z)}catch(V){this.error(`Error processing job ${F}: ${V.message}`),await this.sendJobStatusUpdate(L,Y,U,"failed",V.message)}}updateMetrics(L,Y){let F=this.metrics.averageProcessingTime,H=this.metrics.jobsProcessed+this.metrics.jobsFailed;this.metrics.averageProcessingTime=(F*H+Y)/(H+L)}async createQueue(L){let Y=await super.createQueue(L);if(this.isRunning&&!this.registeredQueues.has(L.name))this.registeredQueues.add(L.name),await this.transport.registerWorker(this.id,Array.from(this.registeredQueues));return Y}}import{uid as i}from"radash";import{Event as D,InteractsWithConsole as p}from"@elysiumjs/core";import{map as g,uid as n}from"radash";class x extends p{static queues=new Map;static exists(L){return this.queues.has(L)}static get(L,Y){if(!this.queues.has(L)){let F=new x({name:L,...Y});this.queues.set(L,F)}return this.queues.get(L)}static getAll(){return new Map(this.queues)}name;transport;transportClass;transportOptions;options;jobs=[];constructor(L){super();this.name=L.name,this.transportClass=L.transport,this.transportOptions=L.transportOptions,this.options={concurrency:L.concurrency??1,maxRetries:L.maxRetries??0,retryDelay:L.retryDelay??1000,pauseOnError:L.pauseOnError??!1,...L},this.transport=new this.transportClass("producer",this.transportOptions),this.transport.onMessage(this.handleTransportMessage.bind(this))}on(L,Y){D.on(`elysium:heracles:queue:${this.name}:${L}`,Y)}once(L,Y){D.once(`elysium:heracles:queue:${this.name}:${L}`,Y)}emit(L,Y,F=null){D.emit(`elysium:heracles:queue:${this.name}:${L}`,Y,F)}async handleTransportMessage(L){switch(L.type){case"job:status":case"job:result":if(L.queue===this.name){if(L.type==="job:result")this.info(`Job ${L.jobId} completed with status: ${L.status}`);let Y=await this.getJobStatus({jobId:L.jobId,dispatchId:L.dispatchId}),F=Y?.createdAt??new Date().toISOString();this.updateJobStatus(L.jobId,L.dispatchId,{jobId:L.jobId,dispatchId:L.dispatchId,queue:L.queue,status:L.status,error:L.error,retries:L.retries??Y?.retries??0,createdAt:F,startedAt:L.startedAt??Y?.startedAt,completedAt:L.completedAt??Y?.completedAt,updatedAt:L.updatedAt??new Date().toISOString()})}break}this.emit(L.type,L,this)}updateJobStatus(L,Y,F){if(F.status==="completed"||F.status==="failed"||F.status==="cancelled")this.debug(`Job ${L} (${Y}) reached terminal state: ${F.status}`)}async dispatch(L,Y,F=[],H={}){let U=new Date,z=U.toISOString(),V=H.jobId??L.generateJobId(...F),X=H.dispatchId??`dispatch_${Date.now()}_${n(8)}`,Z={jobId:V,dispatchId:X,queue:this.name,status:H.scheduledFor&&H.scheduledFor>U?"scheduled_for_retry":"pending",retries:0,createdAt:z,updatedAt:z};this.updateJobStatus(V,X,Z),this.debug(`Dispatching job ${V} (dispatch: ${X}, name: ${Y}) to queue '${this.name}'`);try{let _={type:"job:process",job:Y,jobId:V,dispatchId:X,args:F,queue:this.name,options:{scheduledFor:H.scheduledFor?.toISOString(),priority:H.priority,maxRetries:H.maxRetries,retryDelay:H.retryDelay,overlapBehavior:H.overlapBehavior,overlapDelay:H.overlapDelay}};return await this.transport.send(_),{jobId:Z.jobId,dispatchId:X}}catch(_){throw this.error(`Failed to dispatch job ${Y} to queue '${this.name}': ${_.message}`),this.updateJobStatus(V,X,{jobId:Z.jobId,dispatchId:X,queue:this.name,status:"failed",error:_.message,retries:0,createdAt:Z.createdAt,completedAt:new Date().toISOString(),updatedAt:new Date().toISOString()}),_}}async cancelJob(L){this.debug(`Canceling job ${L.jobId} (${L.dispatchId}) in queue '${this.name}'`);try{let Y={type:"job:cancel",jobId:L.jobId,dispatchId:L.dispatchId,queue:this.name};return await this.transport.send(Y),!0}catch(Y){return this.error(`Failed to cancel job ${L.jobId}: ${Y}`),!1}}async cancelAllJobs(){this.debug(`Canceling all jobs in queue '${this.name}'`);try{let L={type:"job:cancelAll",queue:this.name};await this.transport.send(L)}catch(L){this.error(`Failed to cancel all jobs: ${L}`)}return 0}async getJobStatus(L){try{let Y=await this.transport.getJobStatus(L.jobId,L.dispatchId,this.name);if(Y)return this.debug(`Retrieved job ${L.jobId} (${L.dispatchId}) status from transport: ${Y.status}`),Y}catch(Y){this.warning(`Failed to get job status from transport: ${Y}`)}return null}getAllJobs(){return Array.from(this.jobs)}async getJobsByStatus(L){let Y=[];for(let[F,H]of await g(this.jobs,async(U)=>[U,await this.getJobStatus(U)]))if(H?.status===L)Y.push(F);return Y}async start(){if(this.transport)await this.transport.start(),this.info(`Queue '${this.name}' transport started`)}async stop(){if(this.transport)await this.transport.stop(),this.info(`Queue '${this.name}' transport stopped`)}getOptions(){return{...this.options}}updateOptions(L){this.options={...this.options,...L}}}var y;((k)=>k.dispatch=async(Y,F=[],H={})=>{let U=Q(Y);if(!U)throw Error(`Job ${Y.name} is not marked with @Job.register() decorator`);if(!U.queue)throw Error(`Job ${Y.name} is not registered with a queue`);if(!x.exists(U.queue))throw Error(`Queue ${U.queue} does not exist. Please create the queue before dispatching the job.`);let z=x.get(U.queue),X={dispatchId:`dispatch_${Date.now()}_${i(8)}`,priority:U.priority,maxRetries:U.maxRetries,retryDelay:U.retryDelay,overlapBehavior:U.overlapBehavior,overlapDelay:U.overlapDelay,...H};return z.dispatch(Y,U.name,F,X)})(y||={});export{v as WorkerStatus,K as TransportMode,f as RedisWorker,B as RedisTransport,x as Queue,A as JobStatus,G as JobOverlapBehavior,O as Job,y as Heracles,P as BaseWorker};
2
+ import"@elysiumjs/core";import"reflect-metadata";import{InteractsWithConsole as S,Service as h}from"@elysiumjs/core";import{assign as l,uid as W}from"radash";var E;(($)=>($.job=Symbol("elysium:job"),$.queue=Symbol("elysium:queue"),$.worker=Symbol("elysium:worker"),$.transport=Symbol("elysium:transport"),$.jobTransaction=Symbol("elysium:jobTransaction"),$.workerPool=Symbol("elysium:workerPool")))(E||={});var Q=(L)=>{if(!Reflect.hasMetadata(E.job,L))return null;return Reflect.getMetadata(E.job,L)};var G;((F)=>{F.ALLOW_OVERLAP="allow_overlap";F.NO_OVERLAP="no_overlap"})(G||={});class O extends S{#Y;#F;#L="pending";#H;#z=0;#U;static register(L={}){return function(Y){L=l({name:Y.name,queue:"default",overlapBehavior:"allow_overlap",overlapDelay:0,priority:0},L);let F=`elysium.heracles.job.${L.name}`;h.instance(F,Y);let H={name:F,queue:L.queue,overlapBehavior:L.overlapBehavior,overlapDelay:L.overlapDelay,priority:L.priority,maxRetries:L.maxRetries,retryDelay:L.retryDelay};Reflect.defineMetadata(E.job,H,Y)}}static generateJobId(...L){return`job_${Date.now()}_${W(8)}`}static description="An Heracles job.";id;dispatchId;createdAt;get startedAt(){return this.#Y}get completedAt(){return this.#F}get status(){return this.#L}get lastError(){return this.#H}get retries(){return this.#z}get queueName(){return this.#U}set queueName(L){this.#U=L}incrementRetries(){this.#z++}constructor(L,Y){super();this.id=L??this.constructor.generateJobId(),this.dispatchId=Y??`dispatch_${Date.now()}_${W(8)}`,this.createdAt=new Date}async run(){if(this.#L==="cancelled")return;try{this.#L="running",this.#Y=new Date;let L=this.#U?` in queue '${this.#U}'`:"";if(this.debug(`Job ${this.id}${L} started at ${this.#Y.toISOString()}`),await this.execute(),this.#L==="running")this.#L="completed",this.#F=new Date,this.debug(`Job ${this.id} completed successfully at ${this.#F.toISOString()} after ${this.retries} retries`)}catch(L){this.fail(L instanceof Error?L:Error(String(L)))}}fail(L){if(this.#H=L,this.#L!=="cancelled")this.#L="failed",this.#F=new Date,this.error(`Job ${this.id} failed at ${this.#F.toISOString()} after ${this.retries} retries: ${this.#H.message}`),this.trace(this.#H)}cancel(){if(this.#L==="completed"||this.#L==="failed")return!1;let L=this.#L==="running";if(this.#L="cancelled",L)this.#F=new Date;return this.info(`Job ${this.id} was cancelled${L?" while running":" before execution"}`),!0}}var C=(L,Y,F)=>{return class extends L{constructor(...H){super(...H);this.id=Y,this.dispatchId=F}}},A;((V)=>{V.PENDING="pending";V.RUNNING="running";V.COMPLETED="completed";V.FAILED="failed";V.CANCELLED="cancelled";V.SCHEDULED_FOR_RETRY="scheduled_for_retry"})(A||={});import{Redis as c,Service as j}from"@elysiumjs/core";var K;((F)=>{F.CONSUMER="consumer";F.PRODUCER="producer"})(K||={});import{InteractsWithConsole as J,Redis as T}from"@elysiumjs/core";import{uid as w}from"radash";class B extends J{commandClient=null;subscriberClient=null;publisherClient=null;options;mode;messageHandlers=[];consumerId;isRunning=!1;cleanupTimer;jobLocks=new Map;constructor(L,Y={}){super();this.mode=L,this.options={connection:Y.connection||"default",keyPrefix:Y.keyPrefix||"elysium:heracles",consumerGroup:Y.consumerGroup||"workers",consumerName:Y.consumerName||`worker-${w(8)}`,batchSize:Y.batchSize||10,statusTTL:Y.statusTTL||86400,cleanupCompletedJobs:Y.cleanupCompletedJobs!==!1,completedJobRetention:Y.completedJobRetention||3600,maxStreamSize:Y.maxStreamSize||1000},this.consumerId=this.mode==="consumer"?this.options.consumerName:`producer-${w(8)}`}async initializeClients(){this.commandClient=await T.getConnection(this.options.connection).duplicate(),this.subscriberClient=await T.getConnection(this.options.connection).duplicate(),this.publisherClient=await T.getConnection(this.options.connection).duplicate(),this.setupConnectionHandlers()}setupConnectionHandlers(){this.commandClient.onconnect=()=>{this.debug("Command client connected")},this.commandClient.onclose=(L)=>{this.error(`Command client disconnected: ${L}`)},this.subscriberClient.onconnect=()=>{this.debug("Subscriber client connected")},this.subscriberClient.onclose=(L)=>{this.error(`Subscriber client disconnected: ${L}`)},this.publisherClient.onconnect=()=>{this.debug("Publisher client connected")},this.publisherClient.onclose=(L)=>{this.error(`Publisher client disconnected: ${L}`)}}async start(){if(this.isRunning)return;try{if(await this.initializeClients(),await Promise.all([this.commandClient.connect(),this.subscriberClient.connect(),this.publisherClient.connect()]),await this.commandClient.ping(),await this.publisherClient.ping(),this.info(`RedisTransport started in ${this.mode} mode with ID: ${this.consumerId}`),this.mode==="consumer"){if(await this.subscribeToChannels(),this.options.cleanupCompletedJobs)this.scheduleCleanup()}this.isRunning=!0}catch(L){throw this.error(`Failed to start RedisTransport: ${L}`),Error(`Redis connection failed: ${L}`)}}async stop(){if(this.isRunning=!1,this.cleanupTimer)clearInterval(this.cleanupTimer),this.cleanupTimer=void 0;if(this.mode==="consumer")await this.subscriberClient.unsubscribe();this.commandClient.close(),this.subscriberClient.close(),this.publisherClient.close(),this.info("RedisTransport stopped")}async subscribeToChannels(){let L=await this.getRegisteredQueues();for(let Y of L)await this.subscriberClient.subscribe(`${this.options.keyPrefix}:queue:${Y}:new`,async(F,H)=>{this.debug(`New job notification for queue ${Y}: ${F}`),await this.processJobFromStream(Y,F)}),await this.subscriberClient.subscribe(`${this.options.keyPrefix}:queue:${Y}:control`,async(F,H)=>{try{let U=JSON.parse(F);await this.handleControlMessage(U)}catch(U){this.error(`Error parsing control message: ${U}`)}}),await this.subscriberClient.subscribe(`${this.options.keyPrefix}:lock:${Y}:released`,(F,H)=>{this.debug(`Lock released for job ${F} in queue ${Y}`),this.notifyLockRelease(F,Y)});await this.subscriberClient.subscribe(`${this.options.keyPrefix}:worker:coordination`,async(Y,F)=>{try{let H=JSON.parse(Y);await this.handleWorkerCoordination(H)}catch(H){this.error(`Error handling worker coordination: ${H}`)}})}async processJobFromStream(L,Y){let F=this.getStreamKey(L);try{let H=await this.commandClient.send("XRANGE",[F,Y,Y]);if(H&&H.length>0){let[U,z]=H[0],V=this.deserializeMessage(z,L);for(let X of this.messageHandlers)try{await X(V)}catch(Z){this.error(`Error in message handler: ${Z}`)}await this.commandClient.send("XACK",[F,this.options.consumerGroup,Y])}}catch(H){this.error(`Error processing job from stream: ${H}`)}}async handleControlMessage(L){switch(L.type){case"pause":this.info(`Queue ${L.queue} paused`);break;case"resume":this.info(`Queue ${L.queue} resumed`);break;case"drain":this.info(`Queue ${L.queue} draining`);break;default:this.debug(`Unknown control message type: ${L.type}`)}}async handleWorkerCoordination(L){switch(L.type){case"worker:joined":this.debug(`Worker ${L.workerId} joined`);break;case"worker:left":this.debug(`Worker ${L.workerId} left`);break;case"rebalance":this.debug("Rebalancing work across workers");break}}async send(L){try{let Y=this.extractQueueName(L),F=this.getStreamKey(Y);if(L.type==="job:process"){let H=this.serializeMessage(L),U=await this.commandClient.send("XADD",[F,"*",...H]);await this.indexJob({jobId:L.jobId,dispatchId:L.dispatchId,queue:Y,status:"pending",priority:L.options?.priority||10,timestamp:Date.now()}),await this.storeJobStatus({jobId:L.jobId,dispatchId:L.dispatchId,queue:Y,status:"pending",retries:0,createdAt:new Date().toISOString(),updatedAt:new Date().toISOString(),messageId:U}),await this.publisherClient.publish(`${this.options.keyPrefix}:queue:${Y}:new`,U),this.debug(`Job ${L.jobId} dispatched to queue ${Y} with ID ${U}`)}else if(L.type==="job:status"||L.type==="job:result")await this.updateJobStatus(L.jobId,L.dispatchId,Y,{status:L.status,error:L.error,retries:L.retries,completedAt:L.completedAt,updatedAt:new Date().toISOString()}),await this.updateJobIndex(L.jobId,L.dispatchId,Y,L.status),await this.publisherClient.publish(`${this.options.keyPrefix}:job:${L.jobId}:status`,JSON.stringify({status:L.status,error:L.error,timestamp:Date.now()}));else if(L.type==="job:cancel")await this.publisherClient.publish(`${this.options.keyPrefix}:queue:${Y}:control`,JSON.stringify({type:"cancel",jobId:L.jobId,dispatchId:L.dispatchId}));else{let H=this.serializeMessage(L);await this.commandClient.send("XADD",[F,"*",...H])}}catch(Y){throw this.error(`Failed to send message: ${Y}`),Error(`Failed to send message: ${Y}`)}}onMessage(L){this.messageHandlers.push(L)}async getJobStatus(L,Y,F){try{let H=this.getJobStatusKey(L,Y,F),U=await this.commandClient.hmget(H,["jobId","dispatchId","queue","status","error","retries","createdAt","startedAt","completedAt","updatedAt","messageId"]);if(U&&U[0])return{jobId:U[0],dispatchId:U[1],queue:U[2]||F,status:U[3]||"unknown",error:U[4],retries:parseInt(U[5]||"0",10),createdAt:U[6]||new Date().toISOString(),startedAt:U[7],completedAt:U[8],updatedAt:U[9],messageId:U[10]};let z=await this.getJobFromIndex(L,Y,F);if(z)return{jobId:L,dispatchId:Y,queue:F,status:z.status,retries:0,createdAt:new Date(z.timestamp).toISOString()};return{jobId:L,dispatchId:Y,queue:F,status:"unknown",retries:0,createdAt:new Date().toISOString()}}catch(H){return this.warning(`Failed to get job status: ${H}`),{jobId:L,dispatchId:Y,queue:F,status:"unknown",retries:0,createdAt:new Date().toISOString()}}}async updateJobStatus(L,Y,F,H){try{let U=this.getJobStatusKey(L,Y,F),z=[];if(H.status)z.push("status",H.status);if(H.error!==void 0)z.push("error",H.error||"");if(H.retries!==void 0)z.push("retries",H.retries.toString());if(H.startedAt)z.push("startedAt",H.startedAt);if(H.completedAt)z.push("completedAt",H.completedAt);if(H.updatedAt)z.push("updatedAt",H.updatedAt);if(z.length>0){if(await this.commandClient.hmset(U,z),await this.commandClient.expire(U,this.options.statusTTL),H.status)await this.updateJobIndex(L,Y,F,H.status);await this.publisherClient.publish(`${this.options.keyPrefix}:job:${L}:status`,JSON.stringify({...H,timestamp:Date.now()})),this.debug(`Updated job ${L} status: ${H.status||"fields updated"}`)}}catch(U){throw this.error(`Failed to update job status: ${U}`),U}}async registerWorker(L,Y){try{for(let H of Y){let U=this.getStreamKey(H);await this.ensureConsumerGroupExists(U)}let F=this.getWorkerKey(L);await this.commandClient.hmset(F,["id",L,"status","active","lastSeen",new Date().toISOString(),"queues",JSON.stringify(Y)]),await this.commandClient.expire(F,60),await this.publisherClient.publish(`${this.options.keyPrefix}:worker:coordination`,JSON.stringify({type:"worker:joined",workerId:L,queues:Y,timestamp:Date.now()})),this.debug(`Registered worker ${L} for queues: ${Y.join(", ")}`)}catch(F){throw this.error(`Failed to register worker: ${F}`),F}}async unregisterWorker(L){try{let Y=this.getWorkerKey(L);await this.commandClient.del(Y),await this.publisherClient.publish(`${this.options.keyPrefix}:worker:coordination`,JSON.stringify({type:"worker:left",workerId:L,timestamp:Date.now()})),this.debug(`Unregistered worker ${L}`)}catch(Y){throw this.error(`Failed to unregister worker: ${Y}`),Y}}async indexJob(L){let Y=L.timestamp,F=`${L.jobId}:${L.dispatchId}`;await Promise.all([this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:queue:${L.queue}`,Y.toString(),F]),this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:status:${L.status}`,Y.toString(),F]),this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:priority`,L.priority.toString(),F])])}async updateJobIndex(L,Y,F,H){let U=`${L}:${Y}`,z=Date.now(),X=["pending","running","completed","failed","cancelled","scheduled_for_retry"].filter((_)=>_!==H).map((_)=>this.commandClient.send("ZREM",[`${this.options.keyPrefix}:idx:status:${_}`,U])),Z=this.commandClient.send("ZADD",[`${this.options.keyPrefix}:idx:status:${H}`,z.toString(),U]);await Promise.all([...X,Z])}async getJobFromIndex(L,Y,F){let H=`${L}:${Y}`,U=await this.commandClient.zscore(`${this.options.keyPrefix}:idx:queue:${F}`,H);if(U){let z=["pending","running","completed","failed","cancelled","scheduled_for_retry"];for(let V of z)if(await this.commandClient.zscore(`${this.options.keyPrefix}:idx:status:${V}`,H))return{jobId:L,dispatchId:Y,queue:F,status:V,priority:10,timestamp:U}}return null}async storeJobStatus(L){let Y=this.getJobStatusKey(L.jobId,L.dispatchId,L.queue),F=["jobId",L.jobId,"dispatchId",L.dispatchId,"queue",L.queue,"status",L.status,"retries",L.retries.toString(),"createdAt",L.createdAt,"updatedAt",L.updatedAt||Date.now().toString()];if(L.error)F.push("error",L.error);if(L.startedAt)F.push("startedAt",L.startedAt);if(L.completedAt)F.push("completedAt",L.completedAt);if(L.messageId)F.push("messageId",L.messageId);await this.commandClient.hmset(Y,F),await this.commandClient.expire(Y,this.options.statusTTL)}async ensureConsumerGroupExists(L){try{await this.commandClient.send("XGROUP",["CREATE",L,this.options.consumerGroup,"0","MKSTREAM"]),this.debug(`Created consumer group ${this.options.consumerGroup} for ${L}`)}catch(Y){if(!Y.message?.includes("BUSYGROUP"))this.error(`Error creating consumer group: ${Y}`)}}async getRegisteredQueues(){try{let L=`${this.options.keyPrefix}:stream:*`,Y=await this.commandClient.keys(L);if(!Array.isArray(Y)||Y.length===0)return["default"];return Y.map((F)=>F.replace(`${this.options.keyPrefix}:stream:`,"")).filter(Boolean)}catch(L){return this.error(`Failed to get registered queues: ${L}`),["default"]}}scheduleCleanup(){this.cleanupTimer=setInterval(async()=>{try{await this.performCleanup()}catch(Y){this.error(`Error during cleanup: ${Y}`)}},300000),this.debug("Scheduled cleanup every 300 seconds")}async performCleanup(){this.debug("Starting cleanup operation");let L=Date.now()-this.options.completedJobRetention*1000,Y=await this.getRegisteredQueues(),F=["completed","failed","cancelled"];for(let z of F){let V=`${this.options.keyPrefix}:idx:status:${z}`;await this.commandClient.send("ZREMRANGEBYSCORE",[V,"-inf",L.toString()])}for(let z of Y){let V=this.getStreamKey(z),X=await this.commandClient.send("XLEN",[V]);if(X>this.options.maxStreamSize)await this.commandClient.send("XTRIM",[V,"MAXLEN","~",this.options.maxStreamSize.toString()]),this.debug(`Trimmed stream ${V} from ${X} to ~${this.options.maxStreamSize}`)}let H=`${this.options.keyPrefix}:status:*`,U=await this.commandClient.keys(H);for(let z of U){let V=await this.commandClient.ttl(z);if(V===-1||V===-2)await this.commandClient.del(z)}this.debug("Cleanup operation completed")}extractQueueName(L){if("queue"in L&&typeof L.queue==="string")return L.queue;return"default"}serializeMessage(L){let Y=["type",L.type,"timestamp",Date.now().toString()];if("queue"in L&&L.queue)Y.push("queue",L.queue);switch(L.type){case"job:process":if(Y.push("job",L.job,"args",JSON.stringify(L.args||[]),"jobId",L.jobId,"dispatchId",L.dispatchId),L.options)Y.push("options",JSON.stringify(L.options));break;case"job:cancel":case"job:status":case"job:result":if(Y.push("jobId",L.jobId),Y.push("dispatchId",L.dispatchId),L.type!=="job:cancel"){if(L.status)Y.push("status",L.status);if(L.error)Y.push("error",L.error);if(L.retries!==void 0)Y.push("retries",L.retries.toString());if("completedAt"in L&&L.completedAt)Y.push("completedAt",L.completedAt)}break;case"worker:register":if(Y.push("workerId",L.workerId),L.queues)Y.push("queues",JSON.stringify(L.queues));break;case"worker:unregister":Y.push("workerId",L.workerId);break}return Y}deserializeMessage(L,Y){let F={};for(let z=0;z<L.length;z+=2)F[L[z]]=L[z+1];let H=F.type,U={type:H,queue:F.queue||Y};switch(H){case"job:process":return{...U,type:"job:process",job:F.job,args:JSON.parse(F.args||"[]"),jobId:F.jobId,dispatchId:F.dispatchId,options:F.options?JSON.parse(F.options):void 0};case"job:cancel":return{...U,type:"job:cancel",jobId:F.jobId,dispatchId:F.dispatchId};case"job:status":return{...U,type:"job:status",jobId:F.jobId,dispatchId:F.dispatchId,status:F.status,error:F.error,retries:F.retries?parseInt(F.retries,10):void 0};case"job:result":return{...U,type:"job:result",jobId:F.jobId,dispatchId:F.dispatchId,status:F.status,error:F.error,completedAt:F.completedAt||new Date().toISOString()};default:return U}}notifyLockRelease(L,Y){this.jobLocks.delete(L);for(let F of this.messageHandlers)try{F({type:"job:update",jobId:L,dispatchId:"",queue:Y,status:"lock_released",updates:{}})}catch(H){this.error(`Error notifying lock release: ${H}`)}}async isJobLocked(L,Y){let F=this.jobLocks.get(L);if(F&&F.lockedUntil>Date.now())return!0;let H=`${this.options.keyPrefix}:lock:${Y}:${L}`;return await this.commandClient.exists(H)}async acquireJobLock(L,Y,F=60000){let H=`${this.options.keyPrefix}:lock:${Y}:${L}`,U=`${this.consumerId}:${Date.now()}`;if(await this.commandClient.set(H,U,"PX",F.toString(),"NX")==="OK")return this.jobLocks.set(L,{lockedUntil:Date.now()+F,queue:Y}),this.debug(`Acquired lock for job ${L} in queue ${Y}`),!0;return!1}async releaseJobLock(L,Y){let F=`${this.options.keyPrefix}:lock:${Y}:${L}`;await this.commandClient.del(F),this.jobLocks.delete(L),await this.publisherClient.publish(`${this.options.keyPrefix}:lock:${Y}:released`,L),this.debug(`Released lock for job ${L} in queue ${Y}`)}getStreamKey(L){return`${this.options.keyPrefix}:stream:${L||"default"}`}getJobStatusKey(L,Y,F){return`${this.options.keyPrefix}:status:${F}:${L}:${Y}`}getWorkerKey(L){return`${this.options.keyPrefix}:worker:${L}`}}import{InteractsWithConsole as I}from"@elysiumjs/core";import{sort as M,uid as N}from"radash";var v;((X)=>{X.IDLE="idle";X.ACTIVE="active";X.PAUSED="paused";X.DRAINING="draining";X.STOPPING="stopping";X.STOPPED="stopped";X.DISCONNECTED="disconnected"})(v||={});class P extends I{#Y;transport=null;queues=new Map;defaultQueueName="default";defaultTransport;workerStatus="idle";startedAt=new Date;scheduledJobInterval=1000;scheduledJobTimer;constructor(L){super();this.#Y=L??N(24),this.createQueue({name:this.defaultQueueName})}get id(){return this.#Y}get status(){return this.workerStatus}async createQueue(L){if(this.queues.has(L.name))return this.debug(`Queue with name '${L.name}' already exists`),this;let Y={options:{...L,concurrency:L.concurrency||1,maxRetries:L.maxRetries||0,retryDelay:L.retryDelay||1000,pauseOnError:L.pauseOnError||!1},jobs:[],activeJobs:new Map,retryingJobs:new Map,paused:!1,draining:!1};return this.queues.set(L.name,Y),this.debug(`Created queue '${L.name}'`),this}async addJob(L,Y=this.defaultQueueName,F={}){let H=this.queues.get(Y);if(!H)throw this.error(`Queue '${Y}' does not exist`),Error(`Queue '${Y}' does not exist`);if(H.draining)throw this.warning(`Queue '${Y}' is draining, job ${L.id} will not be added`),Error(`Queue '${Y}' is draining`);L.queueName=Y;let U={job:L,retries:L.retries,queueName:Y,enqueuedAt:new Date,scheduledFor:F.scheduledFor,priority:F.priority??10,maxRetries:F.maxRetries,retryDelay:F.retryDelay,overlapBehavior:F.overlapBehavior,overlapDelay:F.overlapDelay};if(H.jobs.push(U),H.jobs=M(M(M(H.jobs,(z)=>z.enqueuedAt.getTime()),(z)=>z.scheduledFor?z.scheduledFor.getTime():0),(z)=>z.priority??10),this.debug(`Added job ${L.id} to queue '${Y}'`),this.workerStatus==="idle")this.workerStatus="active";if(!H.paused&&H.activeJobs.size<H.options.concurrency)try{this.processQueueJobs(Y)}catch(z){this.error(`Error processing next job in queue '${Y}': ${z}`)}return this}async pause(L){if(L){let Y=this.queues.get(L);if(Y)Y.paused=!0,this.info(`Paused queue '${L}'`);else this.warning(`Cannot pause non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.pause(Y);if(Array.from(this.queues.values()).every((Y)=>Y.paused)&&Array.from(this.queues.values()).every((Y)=>Y.activeJobs.size===0))this.workerStatus="paused",this.info(`Worker ${this.id} is now paused`)}return this}async resume(L){if(L){let Y=this.queues.get(L);if(Y){if(Y.paused=!1,this.info(`Resumed queue '${L}'`),this.workerStatus==="paused")this.workerStatus="active";await this.processQueueJobs(L)}else this.warning(`Cannot resume non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.resume(Y);if(Array.from(this.queues.values()).some((Y)=>Y.jobs.length>0||Y.activeJobs.size>0))this.workerStatus="active",this.info(`Worker ${this.id} is now active`)}return this}async drain(L){if(L){let Y=this.queues.get(L);if(Y)Y.draining=!0,this.info(`Draining queue '${L}'`);else this.warning(`Cannot drain non-existent queue '${L}'`)}else{for(let Y of this.queues.keys())await this.drain(Y);this.workerStatus="draining",this.info(`Worker ${this.id} is now draining`)}return this}async clear(L){if(L){let Y=this.queues.get(L);if(Y){let F=Y.jobs.length;for(let H of Y.jobs)H.job.cancel();Y.jobs=[],this.info(`Cleared ${F} jobs from queue '${L}'`)}else this.warning(`Cannot clear non-existent queue '${L}'`)}else{for(let[Y,F]of this.queues.entries()){let H=F.jobs.length;for(let U of F.jobs)U.job.cancel();F.jobs=[],this.info(`Cleared ${H} jobs from queue '${Y}'`)}if(Array.from(this.queues.values()).every((Y)=>Y.jobs.length===0&&Y.activeJobs.size===0))this.workerStatus="idle"}return this}async size(L=this.defaultQueueName){let Y=this.queues.get(L);if(!Y)throw Error(`Queue '${L}' does not exist`);return Y.jobs.length+Y.activeJobs.size}async totalSize(){let L=0;for(let Y of this.queues.values())L+=Y.jobs.length+Y.activeJobs.size;return L}async getStats(){let L={};for(let[Y,F]of this.queues.entries())L[Y]={waiting:F.jobs.length,retrying:F.retryingJobs.size,processing:F.activeJobs.size,paused:F.paused,draining:F.draining};return L}getInfo(){let L=0;for(let U of this.queues.values())L+=U.activeJobs.size;let Y=Array.from(this.queues.keys()),F=this.defaultTransport?this.defaultTransport.constructor.name:"None",H=async(U,z)=>{try{return await this.addJob(U,z),!0}catch(V){return!1}};return{id:this.id,status:this.workerStatus,queues:Y,activeJobs:L,transport:F,startedAt:this.startedAt,addJob:H,cancelJob:this.cancelJob.bind(this),cancelAllJobs:this.cancelAllJobs.bind(this)}}async cancelJob(L,Y){if(Y)return this.cancelJobInQueue(L,Y);else{for(let F of this.queues.keys())if(await this.cancelJobInQueue(L,F))return!0;return!1}}async cancelAllJobs(L){let Y=0;if(L){let F=this.queues.get(L);if(F){Y=F.jobs.length;for(let H of F.jobs)H.job.cancel();for(let[H,U]of F.activeJobs.entries())if(U.job.cancel())Y++;this.info(`Canceled ${Y} jobs in queue '${L}'`)}}else for(let[F,H]of this.queues.entries()){let U=H.jobs.length;for(let z of H.jobs)z.job.cancel();for(let[z,V]of H.activeJobs.entries())if(V.job.cancel())U++;Y+=U,this.info(`Canceled ${U} jobs in queue '${F}'`)}if(Y>0&&Array.from(this.queues.values()).every((F)=>F.jobs.length===0&&F.activeJobs.size===0))this.workerStatus="idle";return Y}async getJob(L,Y){if(Y){let F=this.queues.get(Y);if(F){let H=F.jobs.find((z)=>z.job.id===L);if(H)return H.job;let U=F.activeJobs.get(L);if(U)return U.job}}else for(let F of this.queues.values()){let H=F.jobs.find((z)=>z.job.id===L);if(H)return H.job;let U=F.activeJobs.get(L);if(U)return U.job}return null}async start(){if(this.workerStatus="active",this.startedAt=new Date,this.defaultTransport)await this.defaultTransport.start();for(let L of this.queues.keys())await this.processQueueJobs(L);this.startScheduledJobTimer(),this.info(`Worker ${this.id} started`)}async stop(L=!1){if(this.workerStatus="stopping",this.stopScheduledJobTimer(),L){await this.cancelAllJobs();for(let Y of this.queues.keys())await this.clear(Y)}else{for(let F of this.queues.keys())await this.drain(F);let Y=()=>{return Array.from(this.queues.values()).every((F)=>F.activeJobs.size===0)};if(!Y()){this.info(`Worker ${this.id} waiting for ${Array.from(this.queues.values()).reduce((F,H)=>F+H.activeJobs.size,0)} jobs to complete`);while(!Y())await new Promise((F)=>setTimeout(F,100))}}if(this.defaultTransport)await this.defaultTransport.stop();this.workerStatus="stopped",this.info(`Worker ${this.id} stopped`)}async setConcurrency(L,Y){let F=this.queues.get(L);if(!F)throw this.error(`Cannot set concurrency: Queue '${L}' does not exist`),Error(`Queue '${L}' does not exist`);if(Y<1)this.warning(`Invalid concurrency value ${Y}, using 1 instead`),Y=1;return F.options.concurrency=Y,this.info(`Set concurrency for queue '${L}' to ${Y}`),this.processQueueJobs(L)}startScheduledJobTimer(){if(this.scheduledJobTimer)clearInterval(this.scheduledJobTimer);this.scheduledJobTimer=setInterval(()=>{this.checkScheduledJobs()},this.scheduledJobInterval)}stopScheduledJobTimer(){if(this.scheduledJobTimer)clearInterval(this.scheduledJobTimer),this.scheduledJobTimer=void 0}checkScheduledJobs(){let L=new Date;for(let[Y,F]of this.queues.entries()){if(F.paused||F.draining)continue;let H=F.jobs.filter((U)=>U.scheduledFor&&U.scheduledFor<=L);for(let U of H)U.scheduledFor=void 0;if(H.length>0&&F.activeJobs.size<F.options.concurrency)this.processQueueJobs(Y)}}async processQueueJobs(L){let Y=this.queues.get(L);if(!Y||Y.paused)return;let F=[];while(F.length<Y.options.concurrency)F.push(this.processNextJob(L));await Promise.all(F);let H=Array.from(this.queues.values());if(H.some((U)=>U.activeJobs.size>0))this.workerStatus="active";else if(H.every((U)=>U.jobs.length===0&&U.activeJobs.size===0))this.workerStatus="idle";if(H.some((U)=>U.jobs.length>0))await Bun.sleep(10),setImmediate(()=>this.processQueueJobs(L))}async sendJobStatusUpdate(L,Y,F,H,U){try{await this.transport.updateJobStatus(L,Y,F,{dispatchId:Y,status:H,error:U,retries:0,startedAt:H==="running"?new Date().toISOString():void 0,completedAt:H==="completed"||H==="failed"||H==="cancelled"?new Date().toISOString():void 0,updatedAt:Date.now().toString()})}catch(z){this.error(`Error sending job status update: ${z}`)}}async cancelJobInQueue(L,Y){let F=this.queues.get(Y);if(!F)return!1;let H=F.jobs.findIndex((U)=>U.job.id===L);if(H>=0){if(F.jobs[H].job.cancel())return F.jobs.splice(H,1),this.info(`Canceled job ${L} in queue '${Y}'`),!0}if(F.activeJobs.has(L)){if(F.activeJobs.get(L).job.cancel())return this.info(`Canceled running job ${L} in queue '${Y}'`),!0}return!1}}class f extends P{isRunning=!1;registeredQueues=new Set;pendingBatches=new Map;batchTimer;metrics={jobsProcessed:0,jobsFailed:0,jobsRetried:0,averageProcessingTime:0};metricsClient;metricsTimer;maxBatchSize;batchInterval;constructor(L="default",Y={}){super(Y.id);if(this.maxBatchSize=Y.maxBatchSize||10,this.batchInterval=Y.batchInterval||100,this.transport=new B("consumer",{connection:L,consumerName:`worker-${this.id}`,batchSize:this.maxBatchSize,...Y}),this.transport.onMessage(this.handleTransportMessage.bind(this)),Y.enableMetrics)this.initializeMetrics(L);this.debug(`RedisWorker ${this.id} created with batch size: ${this.maxBatchSize}`)}async initializeMetrics(L){this.metricsClient=await c.getConnection(L).duplicate()}async reportMetrics(){if(!this.metricsClient||!this.metricsClient.connected)return;let L=`worker:${this.id}:metrics`;try{await this.metricsClient.hmset(L,["jobsProcessed",this.metrics.jobsProcessed.toString(),"jobsFailed",this.metrics.jobsFailed.toString(),"jobsRetried",this.metrics.jobsRetried.toString(),"averageProcessingTime",this.metrics.averageProcessingTime.toString(),"lastProcessedAt",(this.metrics.lastProcessedAt||new Date).toISOString(),"timestamp",Date.now().toString()]),await this.metricsClient.expire(L,3600)}catch(Y){this.error(`Failed to report metrics: ${Y}`)}}async start(){if(this.isRunning)return;await super.start(),this.isRunning=!0;for(let L of this.queues.keys())this.registeredQueues.add(L);if(await this.transport.start(),this.metricsClient)this.metricsTimer=setInterval(()=>{this.reportMetrics()},1e4),await this.metricsClient.connect();await this.transport.registerWorker(this.id,Array.from(this.registeredQueues)),this.startBatchProcessing(),this.info(`Redis worker ${this.id} started for queues: ${Array.from(this.registeredQueues).join(", ")}`)}async stop(L=!1){if(this.isRunning=!1,this.stopBatchProcessing(),!L)await this.processAllPendingBatches();if(await super.stop(L),await this.transport.unregisterWorker(this.id),await this.transport.stop(),this.metricsClient)this.metricsClient.close(),clearInterval(this.metricsTimer),this.metricsClient=void 0;this.info(`Redis worker ${this.id} stopped`)}startBatchProcessing(){this.batchTimer=setInterval(()=>{this.processBatches()},this.batchInterval)}stopBatchProcessing(){if(this.batchTimer)clearInterval(this.batchTimer),this.batchTimer=void 0}async processAllPendingBatches(){for(let[L,Y]of this.pendingBatches)if(Y.jobs.length>0)await this.processBatch(Y);this.pendingBatches.clear()}async processBatches(){let L=Date.now();for(let[Y,F]of this.pendingBatches){if(!this.queues.get(Y))continue;if(F.jobs.length>=this.maxBatchSize||F.jobs.length>0&&L-F.timestamp>this.batchInterval)this.pendingBatches.delete(Y),await this.processBatch(F)}}async processBatch(L){let Y=this.queues.get(L.queue);if(!Y||Y.paused)return;let F=Date.now(),H=new Map,U=[];for(let Z of L.jobs)if(Q(Z.job.constructor)?.overlapBehavior==="no_overlap"){let R=Z.job.id;if(!H.has(R))H.set(R,[]);H.get(R).push(Z)}else U.push(Z);let z=U.map((Z)=>this.processJobInternal(Z,L.queue)),V=[];for(let[Z,_]of H)V.push(this.processOverlapJobs(_,L.queue));await Promise.all([...z,...V]);let X=Date.now()-F;this.updateMetrics(L.jobs.length,X),this.debug(`Processed batch of ${L.jobs.length} jobs from queue '${L.queue}' in ${X}ms`)}async processOverlapJobs(L,Y){for(let F of L)if(await this.transport.acquireJobLock(F.job.id,Y))try{await this.processJobInternal(F,Y)}finally{let U=Q(F.job.constructor);if(U?.overlapDelay&&U.overlapDelay>0)setTimeout(()=>{this.transport.releaseJobLock(F.job.id,Y)},U.overlapDelay);else await this.transport.releaseJobLock(F.job.id,Y)}else{let U=this.queues.get(Y);if(U)U.jobs.push(F)}}async processJobInternal(L,Y){let F=this.queues.get(Y);if(!F)return;F.activeJobs.set(L.job.id,L),await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,Y,"running");try{if(await L.job.run(),L.job.status==="completed")this.metrics.jobsProcessed++,await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,Y,"completed");else if(L.job.status==="failed")await this.handleJobFailure(L,F,Y);else if(L.job.status==="cancelled")await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,Y,"cancelled")}catch(H){this.error(`Error processing job ${L.job.id}: ${H}`),L.job.fail(H instanceof Error?H:Error(String(H))),await this.handleJobFailure(L,F,Y)}finally{F.activeJobs.delete(L.job.id),this.metrics.lastProcessedAt=new Date}}async handleJobFailure(L,Y,F){let H=L.maxRetries!==void 0?L.maxRetries:Y.options.maxRetries;if(L.retries<H){L.retries++,L.job.incrementRetries(),this.metrics.jobsRetried++;let U=L.retryDelay!==void 0?L.retryDelay:Y.options.retryDelay;await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,F,"scheduled_for_retry",L.job.lastError?.message),Y.retryingJobs.set(L.job.id,L),setTimeout(()=>{if(L.job.status!=="cancelled")this.addJobToBatch(L,F);Y.retryingJobs.delete(L.job.id)},U)}else if(this.metrics.jobsFailed++,await this.sendJobStatusUpdate(L.job.id,L.job.dispatchId,F,"failed",L.job.lastError?.message),Y.options.pauseOnError)await this.pause(F)}addJobToBatch(L,Y){if(!this.pendingBatches.has(Y))this.pendingBatches.set(Y,{queue:Y,jobs:[],timestamp:Date.now()});let F=this.pendingBatches.get(Y);if(F.jobs.push(L),F.jobs.length>=this.maxBatchSize)this.pendingBatches.delete(Y),this.processBatch(F)}async processNextJob(L){let Y=this.queues.get(L);if(!Y||Y.paused||Y.activeJobs.size>=Y.options.concurrency)return;let F=new Date,H=[];for(let U=0;U<Y.jobs.length&&H.length<this.maxBatchSize;U++){let z=Y.jobs[U];if(z.scheduledFor&&z.scheduledFor>F)continue;if(z.job.status==="cancelled"){Y.jobs.splice(U,1),U--;continue}H.push(z)}for(let U of H){let z=Y.jobs.indexOf(U);if(z>=0)Y.jobs.splice(z,1)}if(H.length>0){let U={queue:L,jobs:H,timestamp:Date.now()};if(H.length===1)await this.processBatch(U);else this.pendingBatches.set(L,U)}}async handleTransportMessage(L){try{switch(L.type){case"job:process":{let{job:Y,args:F=[],queue:H="default",options:U={},jobId:z,dispatchId:V}=L;await this.handleProcessJob(z,V,Y,F,H,U);break}case"job:cancel":{let{jobId:Y,queue:F="default"}=L;if(await this.cancelJob(Y,F))await this.sendJobStatusUpdate(Y,L.dispatchId,F,"cancelled");break}case"job:cancelAll":{let{queue:Y="default"}=L;await this.cancelAllJobs(Y);break}case"job:update":{if(L.status==="lock_released"){let Y=L.queue;await this.processQueueJobs(Y)}break}}}catch(Y){this.error(`Error handling transport message: ${Y}`)}}async handleProcessJob(L,Y,F,H,U,z){try{let V=j.get(F);if(!V)throw Error(`Job class not found: ${F}`);let X=new(C(V,L,Y))(...H||[]);X.queueName=U;let Z={};if(z.scheduledFor)Z.scheduledFor=typeof z.scheduledFor==="string"?new Date(z.scheduledFor):z.scheduledFor;if(z.priority!==void 0)Z.priority=Number(z.priority);if(z.maxRetries!==void 0)Z.maxRetries=Number(z.maxRetries);if(z.retryDelay!==void 0)Z.retryDelay=Number(z.retryDelay);let _=Z.scheduledFor&&Z.scheduledFor>new Date?"scheduled_for_retry":"pending";await this.sendJobStatusUpdate(X.id,X.dispatchId,U,_),await this.addJob(X,U,Z)}catch(V){this.error(`Error processing job ${F}: ${V.message}`),await this.sendJobStatusUpdate(L,Y,U,"failed",V.message)}}updateMetrics(L,Y){let F=this.metrics.averageProcessingTime,H=this.metrics.jobsProcessed+this.metrics.jobsFailed;this.metrics.averageProcessingTime=(F*H+Y)/(H+L)}async createQueue(L){let Y=await super.createQueue(L);if(this.isRunning&&!this.registeredQueues.has(L.name))this.registeredQueues.add(L.name),await this.transport.registerWorker(this.id,Array.from(this.registeredQueues));return Y}}import{uid as i}from"radash";import{Event as D,InteractsWithConsole as p}from"@elysiumjs/core";import{map as g,uid as n}from"radash";class x extends p{static queues=new Map;static exists(L){return this.queues.has(L)}static get(L,Y){if(!this.queues.has(L)){let F=new x({name:L,...Y});this.queues.set(L,F)}return this.queues.get(L)}static getAll(){return new Map(this.queues)}name;transport;transportClass;transportOptions;options;jobs=[];constructor(L){super();this.name=L.name,this.transportClass=L.transport,this.transportOptions=L.transportOptions,this.options={concurrency:L.concurrency??1,maxRetries:L.maxRetries??0,retryDelay:L.retryDelay??1000,pauseOnError:L.pauseOnError??!1,...L},this.transport=new this.transportClass("producer",this.transportOptions),this.transport.onMessage(this.handleTransportMessage.bind(this))}on(L,Y){D.on(`elysium:heracles:queue:${this.name}:${L}`,Y)}once(L,Y){D.once(`elysium:heracles:queue:${this.name}:${L}`,Y)}emit(L,Y,F=null){D.emit(`elysium:heracles:queue:${this.name}:${L}`,Y,F)}async handleTransportMessage(L){switch(L.type){case"job:status":case"job:result":if(L.queue===this.name){if(L.type==="job:result")this.info(`Job ${L.jobId} completed with status: ${L.status}`);let Y=await this.getJobStatus({jobId:L.jobId,dispatchId:L.dispatchId}),F=Y?.createdAt??new Date().toISOString();this.updateJobStatus(L.jobId,L.dispatchId,{jobId:L.jobId,dispatchId:L.dispatchId,queue:L.queue,status:L.status,error:L.error,retries:L.retries??Y?.retries??0,createdAt:F,startedAt:L.startedAt??Y?.startedAt,completedAt:L.completedAt??Y?.completedAt,updatedAt:L.updatedAt??new Date().toISOString()})}break}this.emit(L.type,L,this)}updateJobStatus(L,Y,F){if(F.status==="completed"||F.status==="failed"||F.status==="cancelled")this.debug(`Job ${L} (${Y}) reached terminal state: ${F.status}`)}async dispatch(L,Y,F=[],H={}){let U=new Date,z=U.toISOString(),V=H.jobId??L.generateJobId(...F),X=H.dispatchId??`dispatch_${Date.now()}_${n(8)}`,Z={jobId:V,dispatchId:X,queue:this.name,status:H.scheduledFor&&H.scheduledFor>U?"scheduled_for_retry":"pending",retries:0,createdAt:z,updatedAt:z};this.updateJobStatus(V,X,Z),this.debug(`Dispatching job ${V} (dispatch: ${X}, name: ${Y}) to queue '${this.name}'`);try{let _={type:"job:process",job:Y,jobId:V,dispatchId:X,args:F,queue:this.name,options:{scheduledFor:H.scheduledFor?.toISOString(),priority:H.priority,maxRetries:H.maxRetries,retryDelay:H.retryDelay,overlapBehavior:H.overlapBehavior,overlapDelay:H.overlapDelay}};return await this.transport.send(_),{jobId:Z.jobId,dispatchId:X}}catch(_){throw this.error(`Failed to dispatch job ${Y} to queue '${this.name}': ${_.message}`),this.updateJobStatus(V,X,{jobId:Z.jobId,dispatchId:X,queue:this.name,status:"failed",error:_.message,retries:0,createdAt:Z.createdAt,completedAt:new Date().toISOString(),updatedAt:new Date().toISOString()}),_}}async cancelJob(L){this.debug(`Canceling job ${L.jobId} (${L.dispatchId}) in queue '${this.name}'`);try{let Y={type:"job:cancel",jobId:L.jobId,dispatchId:L.dispatchId,queue:this.name};return await this.transport.send(Y),!0}catch(Y){return this.error(`Failed to cancel job ${L.jobId}: ${Y}`),!1}}async cancelAllJobs(){this.debug(`Canceling all jobs in queue '${this.name}'`);try{let L={type:"job:cancelAll",queue:this.name};await this.transport.send(L)}catch(L){this.error(`Failed to cancel all jobs: ${L}`)}return 0}async getJobStatus(L){try{let Y=await this.transport.getJobStatus(L.jobId,L.dispatchId,this.name);if(Y)return this.debug(`Retrieved job ${L.jobId} (${L.dispatchId}) status from transport: ${Y.status}`),Y}catch(Y){this.warning(`Failed to get job status from transport: ${Y}`)}return null}getAllJobs(){return Array.from(this.jobs)}async getJobsByStatus(L){let Y=[];for(let[F,H]of await g(this.jobs,async(U)=>[U,await this.getJobStatus(U)]))if(H?.status===L)Y.push(F);return Y}async start(){if(this.transport)await this.transport.start(),this.info(`Queue '${this.name}' transport started`)}async stop(){if(this.transport)await this.transport.stop(),this.info(`Queue '${this.name}' transport stopped`)}getOptions(){return{...this.options}}updateOptions(L){this.options={...this.options,...L}}}var y;((k)=>k.dispatch=async(Y,F=[],H={})=>{let U=Q(Y);if(!U)throw Error(`Job ${Y.name} is not marked with @Job.register() decorator`);if(!U.queue)throw Error(`Job ${Y.name} is not registered with a queue`);if(!x.exists(U.queue))throw Error(`Queue ${U.queue} does not exist. Please create the queue before dispatching the job.`);let z=x.get(U.queue),X={dispatchId:`dispatch_${Date.now()}_${i(8)}`,priority:U.priority,maxRetries:U.maxRetries,retryDelay:U.retryDelay,overlapBehavior:U.overlapBehavior,overlapDelay:U.overlapDelay,...H};return z.dispatch(Y,U.name,F,X)})(y||={});export{v as WorkerStatus,K as TransportMode,f as RedisWorker,B as RedisTransport,x as Queue,A as JobStatus,G as JobOverlapBehavior,O as Job,y as Heracles,P as BaseWorker};
package/package.json CHANGED
@@ -1,46 +1,48 @@
1
1
  {
2
- "name": "@elysiumjs/heracles",
3
- "version": "0.6.0",
4
- "description": "Background job processing system for Elysium.js apps.",
5
- "module": "dist/index.js",
6
- "type": "module",
7
- "types": "dist/index.d.ts",
8
- "sideEffects": ["./src/index.ts"],
9
- "scripts": {
10
- "test": "bun test",
11
- "dev": "bun --watch src/index.ts",
12
- "format": "prettier --write 'src/**/*.{ts,tsx}'",
13
- "clean": "rimraf dist",
14
- "build:js": "bun build ./src/index.ts --outdir ./dist --target bun --packages external --root ./src --minify",
15
- "build:ts": "tsc",
16
- "build": "bun run clean && bun run build:js && bun run build:ts",
17
- "typecheck": "tsc --noEmit",
18
- "prepublishOnly": "bun run build",
19
- "example:optimized": "bun run examples/optimized_example.ts",
20
- "example:worker": "bun run examples/redis_worker_example.ts",
21
- "example:comprehensive": "bun run examples/heracles_comprehensive.ts"
22
- },
23
- "keywords": [
24
- "elysium",
25
- "job",
26
- "queue",
27
- "worker",
28
- "scheduler",
29
- "background",
30
- "processing",
31
- "heracles"
32
- ],
33
- "author": "Workbud Technologies Inc.",
34
- "license": "Apache-2.0",
35
- "dependencies": {
36
- "commander": "^14.0.1",
37
- "radash": "catalog:",
38
- "reflect-metadata": "catalog:"
39
- },
40
- "devDependencies": {
41
- "@elysiumjs/core": "workspace:*"
42
- },
43
- "peerDependencies": {
44
- "@elysiumjs/core": "workspace:^"
45
- }
2
+ "name": "@elysiumjs/heracles",
3
+ "version": "0.6.1",
4
+ "description": "Background job processing system for Elysium.js apps.",
5
+ "module": "dist/index.js",
6
+ "type": "module",
7
+ "types": "dist/index.d.ts",
8
+ "sideEffects": [
9
+ "./src/index.ts"
10
+ ],
11
+ "scripts": {
12
+ "test": "bun test",
13
+ "dev": "bun --watch src/index.ts",
14
+ "format": "prettier --write 'src/**/*.{ts,tsx}'",
15
+ "clean": "rimraf dist",
16
+ "build:js": "bun build ./src/index.ts --outdir ./dist --target bun --packages external --root ./src --minify",
17
+ "build:ts": "tsc",
18
+ "build": "bun run clean && bun run build:js && bun run build:ts",
19
+ "typecheck": "tsc --noEmit",
20
+ "prepublishOnly": "bun run build",
21
+ "example:optimized": "bun run examples/optimized_example.ts",
22
+ "example:worker": "bun run examples/redis_worker_example.ts",
23
+ "example:comprehensive": "bun run examples/heracles_comprehensive.ts"
24
+ },
25
+ "keywords": [
26
+ "elysium",
27
+ "job",
28
+ "queue",
29
+ "worker",
30
+ "scheduler",
31
+ "background",
32
+ "processing",
33
+ "heracles"
34
+ ],
35
+ "author": "Workbud Technologies Inc.",
36
+ "license": "Apache-2.0",
37
+ "dependencies": {
38
+ "commander": "^14.0.1",
39
+ "radash": "^12.1.1",
40
+ "reflect-metadata": "^0.2.2"
41
+ },
42
+ "devDependencies": {
43
+ "@elysiumjs/core": "0.6.1"
44
+ },
45
+ "peerDependencies": {
46
+ "@elysiumjs/core": "^0.6.1"
47
+ }
46
48
  }
package/.turbo/turbo-build.log DELETED
@@ -1,8 +0,0 @@
1
- $ bun run clean && bun run build:js && bun run build:ts
2
- $ rimraf dist
3
- $ bun build ./src/index.ts --outdir ./dist --target bun --packages external --root ./src --minify
4
- Bundled 14 modules in 11ms
5
-
6
- index.js 38.44 KB (entry point)
7
-
8
- $ tsc