@stacksjs/queue 0.75.484 → 0.75.487

package/dist/discovery.d.ts

@@ -2,7 +2,7 @@ import type { JobOptions } from '@stacksjs/types';
 /**
  * Discover jobs from the app/Jobs directory
  */
-export declare function discoverJobs(jobsPath?: string): Promise<DiscoveredJob[]>;
+export declare function discoverJobs(jobsPath?: string, options?: { strict?: boolean }): Promise<DiscoveredJob[]>;
 /**
  * Get a job by name from the registry
  */

package/dist/discovery.js

@@ -1 +1 @@
-import{log}from"@stacksjs/logging";import*as p from"@stacksjs/path";class JobRegistry{jobs=new Map;initialized=!1;register(job){this.jobs.set(job.name,job);log.debug(`Registered job: ${job.name} (${job.type})`)}get(name){return this.jobs.get(name)}all(){return Array.from(this.jobs.values())}byQueue(queue){return this.all().filter((job)=>job.config.queue===queue)}scheduled(){return this.all().filter((job)=>job.config.enabled!==!1&&(job.config.rate||job.config.schedule))}has(name){return this.jobs.has(name)}clear(){this.jobs.clear();this.initialized=!1}setInitialized(value){this.initialized=value}isInitialized(){return this.initialized}}export const jobRegistry=new JobRegistry;export async function discoverJobs(jobsPath){const basePath=jobsPath||p.userJobsPath(),discovered=[];try{const glob=new Bun.Glob("**/*.{ts,js}"),scanOptions={cwd:basePath,onlyFiles:!0,absolute:!0};for await(const file of glob.scan(scanOptions)){if(file.includes(".test.")||file.includes(".spec.")||file.endsWith("index.ts")||file.endsWith("index.js"))continue;try{const job=await loadJob(file);if(job){discovered.push(job);jobRegistry.register(job)}}catch(error){log.warn(`Failed to load job from ${file}: ${error.message}`)}}jobRegistry.setInitialized(!0);log.info(`Discovered ${discovered.length} jobs from ${basePath}`);return discovered}catch(error){log.error(`Failed to discover jobs: ${error.message}`);return[]}}async function loadJob(filePath){try{const module=await import(filePath),fileName=filePath.split("/").pop()?.replace(/\.(ts|js)$/,"")||"UnknownJob";if(module.default&&typeof module.default==="function"){const JobClass=module.default;if(typeof JobClass.handle==="function"||typeof JobClass.prototype?.handle==="function"){const config=JobClass.config||{};return{name:config.name||fileName,path:filePath,config:normalizeConfig(config,fileName),type:"class",module:JobClass}}}if(module.default&&typeof module.default==="object"){const jobConfig=module.default;if(typeof jobConfig.handle==="function"||typeof jobConfig.action==="string")return{name:jobConfig.name||fileName,path:filePath,config:normalizeConfig(jobConfig,fileName),type:"function",module:jobConfig}}return null}catch(error){log.debug(`Could not load job from ${filePath}: ${error.message}`);return null}}function normalizeConfig(config,fileName){return{name:config.name||fileName,description:config.description,queue:config.queue||"default",enabled:config.enabled,tries:config.tries??config.retries??3,backoff:config.backoff??config.retryAfter,rate:config.rate,schedule:config.schedule,timeout:config.timeout??config.timeOut,withoutOverlapping:config.withoutOverlapping,retryAfter:config.retryAfter,backoffConfig:config.backoffConfig}}export function getJob(name){return jobRegistry.get(name)}export function getAllJobs(){return jobRegistry.all()}export function getScheduledJobs(){return jobRegistry.scheduled()}export async function executeJob(name,payload){const job=jobRegistry.get(name);if(!job)throw Error(`Job "${name}" not found. Did you run discoverJobs()?`);try{if(job.type==="class"){if(typeof job.module.handle==="function")return await job.module.handle(payload);return await new job.module().handle(payload)}else{if(typeof job.module.handle==="function")return await job.module.handle(payload);throw Error(`Job "${name}" does not have a handle method`)}}catch(error){log.error(`Failed to execute job "${name}": ${error.message}`);throw error}}export function toJobOptions(job){const config=job.config;return{name:config.name,queue:config.queue,tries:config.tries,backoff:config.backoff,timeout:config.timeout,backoffConfig:config.backoffConfig,rate:config.rate}}
+import{existsSync}from"node:fs";import{log}from"@stacksjs/logging";import*as p from"@stacksjs/path";class JobRegistry{jobs=new Map;initialized=!1;register(job){this.jobs.set(job.name,job);log.debug(`Registered job: ${job.name} (${job.type})`)}get(name){return this.jobs.get(name)}all(){return Array.from(this.jobs.values())}byQueue(queue){return this.all().filter((job)=>job.config.queue===queue)}scheduled(){return this.all().filter((job)=>job.config.enabled!==!1&&(job.config.rate||job.config.schedule))}has(name){return this.jobs.has(name)}clear(){this.jobs.clear();this.initialized=!1}setInitialized(value){this.initialized=value}isInitialized(){return this.initialized}}export const jobRegistry=new JobRegistry;export async function discoverJobs(jobsPath,options={}){const basePath=jobsPath||p.userJobsPath(),discovered=[];if(!existsSync(basePath))return discovered;try{const glob=new Bun.Glob("**/*.{ts,js}"),scanOptions={cwd:basePath,onlyFiles:!0,absolute:!0};for await(const file of glob.scan(scanOptions)){if(file.endsWith(".d.ts")||file.includes(".test.")||file.includes(".spec.")||file.endsWith("index.ts")||file.endsWith("index.js"))continue;try{const job=await loadJob(file,options.strict);if(job){discovered.push(job);jobRegistry.register(job)}}catch(error){if(options.strict)throw error;log.warn(`Failed to load job from ${file}: ${error.message}`)}}jobRegistry.setInitialized(!0);log.info(`Discovered ${discovered.length} jobs from ${basePath}`);return discovered}catch(error){if(options.strict)throw error;log.error(`Failed to discover jobs: ${error.message}`);return[]}}async function loadJob(filePath,strict=!1){try{const module=await import(filePath),fileName=filePath.split("/").pop()?.replace(/\.(ts|js)$/,"")||"UnknownJob";if(module.default&&typeof module.default==="function"){const JobClass=module.default;if(typeof JobClass.handle==="function"||typeof JobClass.prototype?.handle==="function"){const config=JobClass.config||{};return{name:config.name||fileName,path:filePath,config:normalizeConfig(config,fileName),type:"class",module:JobClass}}}if(module.default&&typeof module.default==="object"){const jobConfig=module.default;if(typeof jobConfig.handle==="function"||typeof jobConfig.action==="string")return{name:jobConfig.name||fileName,path:filePath,config:normalizeConfig(jobConfig,fileName),type:"function",module:jobConfig}}if(strict)throw TypeError(`No executable job export in ${filePath}`);return null}catch(error){if(strict)throw error;log.debug(`Could not load job from ${filePath}: ${error.message}`);return null}}function normalizeConfig(config,fileName){return{name:config.name||fileName,description:config.description,queue:config.queue||"default",enabled:config.enabled,tries:config.tries??config.retries??3,backoff:config.backoff??config.retryAfter,rate:config.rate,schedule:config.schedule,timeout:config.timeout??config.timeOut,withoutOverlapping:config.withoutOverlapping,retryAfter:config.retryAfter,backoffConfig:config.backoffConfig}}export function getJob(name){return jobRegistry.get(name)}export function getAllJobs(){return jobRegistry.all()}export function getScheduledJobs(){return jobRegistry.scheduled()}export async function executeJob(name,payload){const job=jobRegistry.get(name);if(!job)throw Error(`Job "${name}" not found. Did you run discoverJobs()?`);try{if(job.type==="class"){if(typeof job.module.handle==="function")return await job.module.handle(payload);return await new job.module().handle(payload)}else{if(typeof job.module.handle==="function")return await job.module.handle(payload);throw Error(`Job "${name}" does not have a handle method`)}}catch(error){log.error(`Failed to execute job "${name}": ${error.message}`);throw error}}export function toJobOptions(job){const config=job.config;return{name:config.name,queue:config.queue,tries:config.tries,backoff:config.backoff,timeout:config.timeout,backoffConfig:config.backoffConfig,rate:config.rate}}

package/dist/index.d.ts

@@ -7,6 +7,7 @@ export type { DeadLetterReason, DeadLetterRecord, ListDeadLetterFilter } from '.
 export type { PoisonConfig } from './poison';
 export type { CircuitBreakerConfig } from './circuit-breaker';
 export type { JobEnvelope, JobEnvelopeOptions, ParsedEnvelope } from './envelope';
+export type { WorkerSnapshot } from './worker-monitor';
 // =============================================================================
 // Redis queue driver
 // =============================================================================
@@ -217,3 +218,4 @@ export {
   stopProcessor,
 } from './worker';
 export { resolveQueueDriver } from './driver';
+export { aggregateWorkerMetrics, readWorkerSnapshots } from './worker-monitor';

package/dist/index.js

@@ -1 +1 @@
-export*from"./bun-queue";export{Job}from"./action";export{getActionRunner,setActionRunner}from"./action-runner";export{Jobs,job,jobBatch,runJob}from"./job";export{claimDispatchKey,hasDispatchedKey,recordDispatchedKey,releaseDispatchKey}from"./idempotency";export{listDeadLetterJobs,moveToDeadLetter,purgeDeadLetterJobs,retryDeadLetterJob}from"./dead-letter";export{hashPayload,isQuarantined,listQuarantined,quarantineJob,recordFailureForPoison,unquarantineJob}from"./poison";export{isCircuitOpen,listCircuitState,pauseQueue,recordCircuitFailure,recordCircuitSuccess,resumeQueue}from"./circuit-breaker";export{assertEnvelopeSerializable,clearEnvelopeWarnings,createEnvelope,JOB_ENVELOPE_VERSION,parseEnvelope,serializeEnvelope}from"./envelope";export{setJobProgress,getJobProgress,cancelJob,isJobCancelled,clearJobState}from"./job-progress";export{discoverJobs,executeJob,getAllJobs,getJob,getScheduledJobs,jobRegistry,toJobOptions}from"./discovery";export{getRegisteredJobs,getSchedulerStatus,isSchedulerRunning,startScheduler,stopScheduler,triggerJob}from"./scheduler";export{emitQueueEvent,getGlobalMetrics,getQueueEvents,getWorkerTracker,onQueueEvent,OnQueueEvent,QueueEvents,QueueMetrics,withEvents}from"./events";export{checkQueueHealth,createHealthCheckHandler,isQueueHealthy,queuedJobState}from"./health";export{configureFailedJobNotifications,FailedJobNotifier,getFailedJobNotifier,notifyJobFailed}from"./notifications";export{createQueueTester,expectJobToFail,fake,getFakeQueue,isFaked,QueueTester,restore,runJob as runTestJob}from"./testing";export{Batch,DispatchedBatch,PendingBatch,getBatchCallbacks,isBatchCancelled,recordBatchJobCompletion,recordBatchJobFailure}from"./batch";export{executeFailedJobs,getActiveJobCount,isWorkerRunning,retryFailedJob,startProcessor,stopProcessor}from"./worker";export async function getRedisQueue(){const{RedisQueue}=await import("./drivers/redis");return RedisQueue}export{resolveQueueDriver}from"./driver";
+export*from"./bun-queue";export{Job}from"./action";export{getActionRunner,setActionRunner}from"./action-runner";export{Jobs,job,jobBatch,runJob}from"./job";export{claimDispatchKey,hasDispatchedKey,recordDispatchedKey,releaseDispatchKey}from"./idempotency";export{listDeadLetterJobs,moveToDeadLetter,purgeDeadLetterJobs,retryDeadLetterJob}from"./dead-letter";export{hashPayload,isQuarantined,listQuarantined,quarantineJob,recordFailureForPoison,unquarantineJob}from"./poison";export{isCircuitOpen,listCircuitState,pauseQueue,recordCircuitFailure,recordCircuitSuccess,resumeQueue}from"./circuit-breaker";export{assertEnvelopeSerializable,clearEnvelopeWarnings,createEnvelope,JOB_ENVELOPE_VERSION,parseEnvelope,serializeEnvelope}from"./envelope";export{setJobProgress,getJobProgress,cancelJob,isJobCancelled,clearJobState}from"./job-progress";export{discoverJobs,executeJob,getAllJobs,getJob,getScheduledJobs,jobRegistry,toJobOptions}from"./discovery";export{getRegisteredJobs,getSchedulerStatus,isSchedulerRunning,startScheduler,stopScheduler,triggerJob}from"./scheduler";export{emitQueueEvent,getGlobalMetrics,getQueueEvents,getWorkerTracker,onQueueEvent,OnQueueEvent,QueueEvents,QueueMetrics,withEvents}from"./events";export{checkQueueHealth,createHealthCheckHandler,isQueueHealthy,queuedJobState}from"./health";export{configureFailedJobNotifications,FailedJobNotifier,getFailedJobNotifier,notifyJobFailed}from"./notifications";export{createQueueTester,expectJobToFail,fake,getFakeQueue,isFaked,QueueTester,restore,runJob as runTestJob}from"./testing";export{Batch,DispatchedBatch,PendingBatch,getBatchCallbacks,isBatchCancelled,recordBatchJobCompletion,recordBatchJobFailure}from"./batch";export{executeFailedJobs,getActiveJobCount,isWorkerRunning,retryFailedJob,startProcessor,stopProcessor}from"./worker";export async function getRedisQueue(){const{RedisQueue}=await import("./drivers/redis");return RedisQueue}export{resolveQueueDriver}from"./driver";export{aggregateWorkerMetrics,readWorkerSnapshots}from"./worker-monitor";

package/dist/worker-monitor.d.ts

@@ -0,0 +1,28 @@
+import type { TrackedWorker } from './events';
+/** One upsert per worker heartbeat, never a read/modify/write shared counter. */
+export declare function persistWorkerSnapshot(snapshot: WorkerSnapshot): Promise<void>;
+export declare function pruneWorkerSnapshots(now?: unknown): Promise<void>;
+/** A stale or stopped worker never appears to be processing work. */
+export declare function workerSnapshotState(snapshot: WorkerSnapshot, now?: unknown): WorkerSnapshot;
+export declare function readWorkerSnapshots(now?: unknown): Promise<WorkerSnapshot[]>;
+export declare function aggregateWorkerMetrics(snapshots: WorkerSnapshot[]): {
+  counts: {
+  completed: ReturnType<WorkerSnapshot[].reduce>;
+  failed: ReturnType<WorkerSnapshot[].reduce>
+};
+  throughputPerMinute: unknown;
+  averageDuration: boolean
+};
+export declare const WORKER_STALE_MS: unknown;
+export declare const WORKER_RETENTION_MS: number;
+export declare interface WorkerSnapshot {
+  worker: TrackedWorker
+  heartbeatAt: number
+  activeJobs: number
+  metrics: {
+    completed: number
+    failed: number
+    throughputPerMinute: number
+    averageDuration: number
+  }
+}

package/dist/worker-monitor.js

@@ -0,0 +1 @@
+import{db,getDatabaseDialect,sqlHelpers}from"@stacksjs/database/runtime";export const WORKER_STALE_MS=60000,WORKER_RETENTION_MS=604800000;export async function persistWorkerSnapshot(snapshot){const dialect=sqlHelpers(getDatabaseDialect()),values=[snapshot.worker.id,JSON.stringify(snapshot),snapshot.heartbeatAt],columns=["worker_id","snapshot","heartbeat_at"],quote=(name)=>dialect.isMysql?`\`${name}\``:`"${name}"`,update=["snapshot","heartbeat_at"].map((column)=>dialect.isMysql?`${quote(column)} = VALUES(${quote(column)})`:`${quote(column)} = excluded.${quote(column)}`).join(", ");await db.unsafe(`INSERT INTO queue_workers (${columns.map(quote).join(", ")}) VALUES (${values.map((_,i)=>dialect.param(i+1)).join(", ")}) ${dialect.isMysql?"ON DUPLICATE KEY UPDATE":"ON CONFLICT (worker_id) DO UPDATE SET"} ${update}`,values).execute()}export async function pruneWorkerSnapshots(now=Date.now()){const dialect=sqlHelpers(getDatabaseDialect());await db.unsafe(`DELETE FROM queue_workers WHERE heartbeat_at < ${dialect.param(1)}`,[now-WORKER_RETENTION_MS]).execute()}export function workerSnapshotState(snapshot,now=Date.now()){if(snapshot.worker.status==="stopped"||now-snapshot.heartbeatAt>WORKER_STALE_MS)return{...snapshot,worker:{...snapshot.worker,status:"stopped"},activeJobs:0,metrics:{...snapshot.metrics,throughputPerMinute:0,averageDuration:0}};return snapshot}export async function readWorkerSnapshots(now=Date.now()){const dialect=sqlHelpers(getDatabaseDialect());return(await db.unsafe(`SELECT snapshot FROM queue_workers WHERE heartbeat_at >= ${dialect.param(1)} ORDER BY heartbeat_at DESC`,[now-WORKER_RETENTION_MS]).execute()).map((row)=>workerSnapshotState(JSON.parse(row.snapshot),now))}export function aggregateWorkerMetrics(snapshots){const throughputPerMinute=snapshots.reduce((sum,value)=>sum+value.metrics.throughputPerMinute,0);return{counts:{completed:snapshots.reduce((sum,value)=>sum+value.metrics.completed,0),failed:snapshots.reduce((sum,value)=>sum+value.metrics.failed,0)},throughputPerMinute,averageDuration:throughputPerMinute>0?snapshots.reduce((sum,value)=>sum+value.metrics.averageDuration*value.metrics.throughputPerMinute,0)/throughputPerMinute:0}}

package/dist/worker.js

@@ -1 +1 @@
-import{batchMembershipId}from"./batch-payload";import{resolveQueueDriver}from"./driver";import{err,ok}from"@stacksjs/error-handling";import{log}from"@stacksjs/logging";import process from"node:process";import{parseEnvelope}from"./envelope";import{updatedRowCount}from"./utils";let workerCrashHandlersInstalled=!1;function installWorkerCrashHandlers(){if(workerCrashHandlersInstalled)return;workerCrashHandlersInstalled=!0;process.on("unhandledRejection",(reason,_promise)=>{log.error(`Unhandled Rejection: ${reason}`)});process.on("uncaughtException",(error)=>{log.error(`Uncaught Exception: ${error.message}`)})}let activeJobCount=0,workerRunning=!1,workerId="";const inFlightJobs=new Set,inFlightDatabaseBatches=new Set,reserveErrorLoggedAt=new Map;function trackInFlight(promise,pending=inFlightJobs){pending.add(promise);promise.then(()=>pending.delete(promise),()=>pending.delete(promise));return promise}function getReservationTtlSec(){const raw=process.env.STACKS_QUEUE_RESERVATION_TTL_SEC,n=raw===void 0?Number.NaN:Number.parseInt(raw,10);return Number.isFinite(n)&&n>0?n:3600}function getRetryJitterRatio(){const raw=process.env.STACKS_QUEUE_RETRY_JITTER;if(raw===void 0)return 0.2;const n=Number(raw);if(!Number.isFinite(n)||n<0)return 0.2;return Math.min(n,1)}export function applyRetryJitter(delaySeconds,ratio,random=Math.random){if(!(delaySeconds>0)||ratio<=0)return delaySeconds;return Math.round(delaySeconds+delaySeconds*ratio*random())}function getSweepIntervalMs(){const raw=process.env.STACKS_QUEUE_SWEEP_INTERVAL_SEC,n=raw===void 0?Number.NaN:Number.parseInt(raw,10);return(Number.isFinite(n)&&n>0?n:60)*1000}async function sweepStaleReservations(){const ttlSec=getReservationTtlSec(),cutoff=Math.floor(Date.now()/1000)-ttlSec,now=Math.floor(Date.now()/1000);try{const{db}=await import("@stacksjs/database/runtime"),result=await db.updateTable("jobs").set({reserved_at:null,available_at:now}).where("reserved_at","<=",cutoff).executeTakeFirst(),requeued=updatedRowCount(result);if(requeued>0)log.warn(`[queue] Requeued ${requeued} job(s) whose reservation exceeded the ${ttlSec}s TTL - likely victims of a worker crash. Set STACKS_QUEUE_RESERVATION_TTL_SEC to tune.`);return requeued}catch(error){log.error("[queue] Reservation sweep failed",{reason:error instanceof Error?error.message:String(error)});return 0}}function isNonRetryableError(e){if(!e||typeof e!=="object")return!1;const status=e.status??e.statusCode;if(typeof status==="number"&&status>=400&&status<500)return!0;const name=e.name;if(typeof name==="string"&&(name==="ValidationError"||name==="ModelNotFoundError"))return!0;return!1}function getQueueDriver(){return resolveQueueDriver()}export async function startProcessor(queueName,options={}){try{log.info("Starting queue processor...");installWorkerCrashHandlers();workerRunning=!0;workerId=`worker-${process.pid}-${Date.now()}`;const concurrency=options.concurrency||1,queueDriver=getQueueDriver(),{getWorkerTracker,getGlobalMetrics}=await import("./events");getGlobalMetrics();getWorkerTracker().register(workerId,queueName||"default");if(queueDriver==="redis"){log.info("Using Redis queue driver (bun-queue)");await processJobsFromRedis(queueName||"default",concurrency);return ok(void 0)}const named=namedWorkerQueues(queueName);let queues;if(named.length>0)queues=named;else{queues=await getAllQueues();if(queues.length===0)queues=["default"]}log.info(`Processing queues: ${queues.join(", ")}`);await processJobsFromDatabase(queues,concurrency,{followNewQueues:named.length===0});return ok(void 0)}catch(error){workerRunning=!1;return err(error)}}async function getAllQueues(){try{const{db}=await import("@stacksjs/database/runtime"),queues=(await db.selectFrom("jobs").select("queue").distinct().execute()).map((r)=>r.queue).filter((q)=>Boolean(q));return queues.length>0?queues:["default"]}catch{return["default"]}}export function namedWorkerQueues(queueName){if(typeof queueName!=="string")return[];return[...new Set(queueName.split(",").map((name)=>name.trim()).filter(Boolean))]}async function processJobsFromDatabase(initialQueues,concurrency,options={followNewQueues:!0}){log.info("Listening for jobs...");let queues=initialQueues,lastQueueRefresh=Date.now();const queueRefreshInterval=1e4;let lastSweep=Date.now();const sweepIntervalMs=getSweepIntervalMs();await sweepStaleReservations();while(workerRunning)try{let processedJobs=!1;const now=Date.now();if(options.followNewQueues&&now-lastQueueRefresh>queueRefreshInterval){try{const refreshedQueues=await getAllQueues();if(refreshedQueues.length>0)queues=refreshedQueues}catch{}lastQueueRefresh=now}if(now-lastSweep>sweepIntervalMs){await sweepStaleReservations();lastSweep=now}for(const queueName of queues){if(!workerRunning)break;try{const{isCircuitOpen}=await import("./circuit-breaker");if(await isCircuitOpen(queueName))continue}catch{}if(!workerRunning)break;const processed=await trackInFlight(processDatabaseBatch(queueName,concurrency),inFlightDatabaseBatches);processedJobs||=processed}if(workerRunning&&!processedJobs)await sleep(1000)}catch{await sleep(3000)}}async function processDatabaseBatch(queueName,concurrency){let jobs=[];try{jobs=await fetchPendingJobs(queueName,concurrency)}catch(error){const now=Date.now(),last=reserveErrorLoggedAt.get(queueName)??0;if(now-last>60000){reserveErrorLoggedAt.set(queueName,now);log.error(`[queue] Could not reserve jobs on "${queueName}" - retrying each cycle:`,error)}return!1}await Promise.all(jobs.map(async(job)=>{try{log.info(`Processing job ${job.id} from queue "${queueName}"`);await trackInFlight(processJob(job))}catch{log.error(`Unexpected error processing job ${job.id}`)}}));return jobs.length>0}async function fetchPendingJobs(queueName,limit){const now=Math.floor(Date.now()/1000),{db}=await import("@stacksjs/database/runtime"),claimed=[];for(let i=0;i<limit&&workerRunning;i++){const job=await db.selectFrom("jobs").where("queue","=",queueName).whereNull("reserved_at").where("available_at","<=",now).orderBy("id","asc").limit(1).selectAll().executeTakeFirst();if(!job||!workerRunning)break;const result=await db.updateTable("jobs").set({reserved_at:now,attempts:Number(job.attempts||0)+1}).where("id","=",job.id).whereNull("reserved_at").executeTakeFirst();if(updatedRowCount(result)>0)claimed.push(job)}return claimed}async function processJob(job){const jobId=job.id,queueName=job.queue||"default";activeJobCount++;const startTime=Date.now(),{emitQueueEvent,getWorkerTracker}=await import("./events"),tracker=getWorkerTracker();tracker.markActive(workerId);let parsedJobName;try{parsedJobName=JSON.parse(job.payload||"{}").jobName}catch{}await emitQueueEvent("job:processing",{jobId:String(jobId),queueName,jobName:parsedJobName});try{const parsedForBatch=JSON.parse(job.payload||"{}"),batchId=batchMembershipId(parsedForBatch.payload);if(batchId){const{isBatchCancelled}=await import("./batch");if(await isBatchCancelled(batchId)){log.info(`[Queue] Skipping job ${jobId} - batch ${batchId} has been cancelled`);await deleteJob(jobId);activeJobCount--;tracker.markIdle(workerId);return}}}catch{}let jobError=null;const declared=parsedJobName?await(await import("./job")).jobDefaults(parsedJobName):{};try{const payload=JSON.parse(job.payload||"{}"),timeoutSec=readJobTimeoutSec(payload)??declared.timeout;if(timeoutSec===void 0)await executeJobPayload(payload);else await raceWithTimeout(executeJobPayload(payload),timeoutSec*1000,`Job ${jobId} exceeded ${timeoutSec}s timeout`)}catch(e){jobError=e instanceof Error?e:Error(String(e))}if(!jobError)try{await deleteJob(jobId);log.info(`[Queue] Job ${jobId} completed`);tracker.recordCompletion(workerId);try{const{recordCircuitSuccess}=await import("./circuit-breaker");await recordCircuitSuccess(job.queue??"default")}catch{}await emitQueueEvent("job:completed",{jobId:String(jobId),queueName,duration:Date.now()-startTime});try{const payload=JSON.parse(job.payload||"{}"),batchId=batchMembershipId(payload.payload);if(batchId){const{recordBatchJobCompletion}=await import("./batch");await recordBatchJobCompletion(batchId)}}catch{}}catch{log.info(`[Queue] Failed to delete completed job ${jobId}`)}else{const errorMessage=jobError.message;log.info(`[Queue] Job ${jobId} failed: ${errorMessage}`);tracker.recordFailure(workerId);await emitQueueEvent("job:failed",{jobId:String(jobId),queueName,error:jobError,duration:Date.now()-startTime,attemptsMade:(job.attempts||0)+1});let maxAttempts=1,parsedPayload={};try{parsedPayload=JSON.parse(job.payload||"{}");maxAttempts=parsedPayload.options?.tries||declared.tries||1}catch{}const currentAttempts=(job.attempts||0)+1;if(currentAttempts>=maxAttempts){log.info(`[Queue] Job ${jobId} exceeded max attempts (${currentAttempts}/${maxAttempts})`);let persisted=!1;if(parsedPayload?._retriedFromFailed===!0)try{const{moveToDeadLetter}=await import("./dead-letter");persisted=await moveToDeadLetter({queue:job.queue,payload:job.payload,exception:jobError.stack||jobError.message},"repeat-failure",2);if(persisted)log.info(`[Queue] Job ${jobId} re-failed after retry - moved to dead_letter_jobs`)}catch{persisted=!1}if(!persisted){log.info(`[Queue] Moving job ${jobId} to failed_jobs`);try{persisted=await moveToFailedJobs(job,jobError,{attempts:currentAttempts,maxAttempts,durationMs:Date.now()-startTime})}catch{persisted=!1}}if(persisted)try{await deleteJob(jobId)}catch{log.info(`[Queue] Failed to delete failed job ${jobId}`)}else log.error(`[Queue] Job ${jobId} exhausted its retries but could NOT be persisted to failed_jobs - leaving it in the queue to avoid data loss (the reservation sweep will retry it). Check that the failed_jobs table exists and is writable.`);try{const{recordFailureForPoison}=await import("./poison"),{recordCircuitFailure}=await import("./circuit-breaker"),jobName=parsedPayload?.jobName??"unknown";await recordFailureForPoison(jobName,parsedPayload?.payload);await recordCircuitFailure(job.queue??"default")}catch{}try{const batchId=batchMembershipId(parsedPayload.payload);if(batchId){const{recordBatchJobFailure}=await import("./batch");await recordBatchJobFailure(batchId,String(jobId),jobError)}}catch{}}else{const backoffDelays=parsedPayload.options?.backoff??declared.backoff;let retryDelay=30;if(Array.isArray(backoffDelays)&&backoffDelays.length>0){const backoffIndex=Math.min(currentAttempts-1,backoffDelays.length-1);retryDelay=backoffDelays[backoffIndex]}else if(typeof backoffDelays==="number"&&backoffDelays>0)retryDelay=backoffDelays;retryDelay=Number(retryDelay);if(!Number.isFinite(retryDelay)||retryDelay<0)retryDelay=30;retryDelay=applyRetryJitter(retryDelay,getRetryJitterRatio());log.info(`[Queue] Job ${jobId} will be retried in ${retryDelay}s (attempt ${currentAttempts}/${maxAttempts})`);try{await releaseJob(jobId,retryDelay);log.info(`[Queue] Job ${jobId} released for retry`)}catch{log.info(`[Queue] Failed to release job ${jobId} for retry`)}}}activeJobCount--;tracker.markIdle(workerId)}async function deleteJob(jobId){const{db}=await import("@stacksjs/database/runtime");await db.deleteFrom("jobs").where("id","=",jobId).execute()}async function releaseJob(jobId,delaySeconds=30){const retryAt=Math.floor(Date.now()/1000)+delaySeconds;log.debug(`Releasing job ${jobId} for retry at ${retryAt}`);try{const{db}=await import("@stacksjs/database/runtime");await db.updateTable("jobs").set({reserved_at:null,available_at:retryAt}).where("id","=",jobId).execute();log.debug(`Job ${jobId} released successfully`)}catch{log.error(`Failed to release job ${jobId}`)}}async function moveToFailedJobs(job,error,metrics){try{const failedAt=new Date().toISOString().slice(0,19).replace("T"," "),uuid=crypto.randomUUID(),exception=error.stack||error.message,{db}=await import("@stacksjs/database/runtime");await db.insertInto("failed_jobs").values({uuid,connection:"database",queue:job.queue,payload:job.payload,exception,attempts:metrics.attempts,max_attempts:metrics.maxAttempts,duration_ms:metrics.durationMs,failed_at:failedAt}).execute();return!0}catch(insertError){log.error("Failed to log failed job:",insertError);return!1}}function readJobTimeoutSec(payload){if(!payload||typeof payload!=="object")return;const t=payload.options?.timeout;if(typeof t!=="number"||!Number.isFinite(t)||t<=0)return;return t}async function raceWithTimeout(task,timeoutMs,message){let timer;const timeout=new Promise((_,reject)=>{timer=setTimeout(()=>reject(Error(message)),timeoutMs)});try{return await Promise.race([task,timeout])}finally{if(timer!==void 0)clearTimeout(timer)}}async function executeJobPayload(payload){const parsed=parseEnvelope(payload);if(!parsed.ok)throw Error(`[queue] Cannot deserialize job envelope: ${parsed.reason}`+(parsed.detail?` (${parsed.detail})`:""));const{runJob}=await import("./job");await runJob(parsed.envelope.jobName,{payload:parsed.envelope.payload,context:parsed.envelope.context,traceId:parsed.envelope.traceId})}async function processJobsFromRedis(queueName,concurrency){const{RedisQueue}=await import("./drivers/redis"),{queue:queueConfig}=await import("@stacksjs/config"),redisConfig=queueConfig?.connections?.redis;if(!redisConfig)throw Error("Redis queue connection is not configured. Check config/queue.ts");const queue=new RedisQueue(queueName,redisConfig),{emitQueueEvent,getWorkerTracker}=await import("./events"),tracker=getWorkerTracker(),handleRedisJob=async(bunJob)=>{activeJobCount++;tracker.markActive(workerId);const startTime=Date.now(),parsed=parseEnvelope(bunJob.data);if(!parsed.ok){activeJobCount--;tracker.markIdle(workerId);log.error(`[Queue] Skipping Redis job ${bunJob.id} - unparseable envelope: ${parsed.reason}${parsed.detail?` (${parsed.detail})`:""}`);return}const data=parsed.envelope,batchId=batchMembershipId(data.payload);if(batchId)try{const{isBatchCancelled}=await import("./batch");if(await isBatchCancelled(batchId)){log.info(`[Queue] Skipping Redis job ${bunJob.id} - batch ${batchId} has been cancelled`);activeJobCount--;tracker.markIdle(workerId);return}}catch{}await emitQueueEvent("job:processing",{jobId:String(bunJob.id),queueName});try{const{runJob}=await import("./job");await runJob(data.jobName,{payload:data.payload,context:data.context,traceId:data.traceId});tracker.recordCompletion(workerId);await emitQueueEvent("job:completed",{jobId:String(bunJob.id),queueName,duration:Date.now()-startTime});if(batchId)try{const{recordBatchJobCompletion}=await import("./batch");await recordBatchJobCompletion(batchId)}catch{}log.info(`[Queue] Redis job ${bunJob.id} completed`)}catch(e){tracker.recordFailure(workerId);await emitQueueEvent("job:failed",{jobId:String(bunJob.id),queueName,error:e instanceof Error?e:Error(String(e)),duration:Date.now()-startTime});if(batchId)try{const{recordBatchJobFailure}=await import("./batch");await recordBatchJobFailure(batchId,String(bunJob.id),e instanceof Error?e:Error(String(e)))}catch{}log.error(`[Queue] Redis job ${bunJob.id} failed: ${e}`);if(isNonRetryableError(e)){log.info(`[Queue] Redis job ${bunJob.id} hit a non-retryable error - skipping retry`);return}throw e}finally{activeJobCount--;tracker.markIdle(workerId)}};queue.process(concurrency,(bunJob)=>trackInFlight(handleRedisJob(bunJob)));log.info(`Listening for Redis jobs on queue "${queueName}" with concurrency ${concurrency}...`);while(workerRunning)await sleep(1000);await queue.close()}export async function stopProcessor(options={}){const graceMs=options.graceMs??1e4;workerRunning=!1;const pending=new Set([...inFlightDatabaseBatches,...inFlightJobs]);if(pending.size>0&&graceMs>0){log.info(`[queue] Draining database reservations and in-flight jobs (grace ${graceMs}ms)`);const drain=Promise.allSettled([...pending]),timeout=new Promise((resolve)=>setTimeout(()=>resolve("timeout"),graceMs));if(await Promise.race([drain.then(()=>"drained"),timeout])==="timeout"&&(inFlightJobs.size>0||inFlightDatabaseBatches.size>0))log.warn("[queue] Drain timed out with reservations or jobs still active. Their reservations will be reclaimed by the next worker's sweep (Q-2).")}if(workerId){const{getWorkerTracker}=await import("./events");getWorkerTracker().unregister(workerId)}if(getQueueDriver()==="redis"){const{closeSharedRedisQueues}=await import("./drivers/redis");await closeSharedRedisQueues()}log.info("Queue processor stopped")}export async function executeFailedJobs(){const{db}=await import("@stacksjs/database/runtime"),failedJobs=await db.selectFrom("failed_jobs").selectAll().execute();for(const failedJob of failedJobs)await retryFailedJob(Number(failedJob.id))}export async function retryFailedJob(id){const now=Math.floor(Date.now()/1000),createdAt=new Date().toISOString().slice(0,19).replace("T"," "),{db}=await import("@stacksjs/database/runtime"),failedJob=(await db.selectFrom("failed_jobs").where("id","=",id).selectAll().execute())[0];if(!failedJob)throw Error(`Failed job ${id} not found`);const storedPayload=typeof failedJob.payload==="string"?failedJob.payload:"";let payloadToRequeue=storedPayload;try{const env=JSON.parse(storedPayload||"{}");env._retriedFromFailed=!0;payloadToRequeue=JSON.stringify(env)}catch{}await db.insertInto("jobs").values({queue:failedJob.queue,payload:payloadToRequeue,attempts:0,reserved_at:null,available_at:now,created_at:createdAt}).execute();await db.deleteFrom("failed_jobs").where("id","=",id).execute();log.info(`Failed job ${id} has been re-queued`)}export function getActiveJobCount(){return activeJobCount}export function isWorkerRunning(){return workerRunning}function sleep(ms){return new Promise((resolve)=>setTimeout(resolve,ms))}
+import{persistWorkerSnapshot,pruneWorkerSnapshots}from"./worker-monitor";import{batchMembershipId}from"./batch-payload";import{resolveQueueDriver}from"./driver";import{err,ok}from"@stacksjs/error-handling";import{log}from"@stacksjs/logging";import process from"node:process";import{parseEnvelope}from"./envelope";import{updatedRowCount}from"./utils";let workerCrashHandlersInstalled=!1;function installWorkerCrashHandlers(){if(workerCrashHandlersInstalled)return;workerCrashHandlersInstalled=!0;process.on("unhandledRejection",(reason,_promise)=>{log.error(`Unhandled Rejection: ${reason}`)});process.on("uncaughtException",(error)=>{log.error(`Uncaught Exception: ${error.message}`)})}let activeJobCount=0,workerRunning=!1,workerId="";const inFlightJobs=new Set,inFlightDatabaseBatches=new Set,reserveErrorLoggedAt=new Map;function trackInFlight(promise,pending=inFlightJobs){pending.add(promise);promise.then(()=>pending.delete(promise),()=>pending.delete(promise));return promise}function getReservationTtlSec(){const raw=process.env.STACKS_QUEUE_RESERVATION_TTL_SEC,n=raw===void 0?Number.NaN:Number.parseInt(raw,10);return Number.isFinite(n)&&n>0?n:3600}function getRetryJitterRatio(){const raw=process.env.STACKS_QUEUE_RETRY_JITTER;if(raw===void 0)return 0.2;const n=Number(raw);if(!Number.isFinite(n)||n<0)return 0.2;return Math.min(n,1)}export function applyRetryJitter(delaySeconds,ratio,random=Math.random){if(!(delaySeconds>0)||ratio<=0)return delaySeconds;return Math.round(delaySeconds+delaySeconds*ratio*random())}function getSweepIntervalMs(){const raw=process.env.STACKS_QUEUE_SWEEP_INTERVAL_SEC,n=raw===void 0?Number.NaN:Number.parseInt(raw,10);return(Number.isFinite(n)&&n>0?n:60)*1000}async function sweepStaleReservations(){const ttlSec=getReservationTtlSec(),cutoff=Math.floor(Date.now()/1000)-ttlSec,now=Math.floor(Date.now()/1000);try{const{db}=await import("@stacksjs/database/runtime"),result=await db.updateTable("jobs").set({reserved_at:null,available_at:now}).where("reserved_at","<=",cutoff).executeTakeFirst(),requeued=updatedRowCount(result);if(requeued>0)log.warn(`[queue] Requeued ${requeued} job(s) whose reservation exceeded the ${ttlSec}s TTL - likely victims of a worker crash. Set STACKS_QUEUE_RESERVATION_TTL_SEC to tune.`);return requeued}catch(error){log.error("[queue] Reservation sweep failed",{reason:error instanceof Error?error.message:String(error)});return 0}}function isNonRetryableError(e){if(!e||typeof e!=="object")return!1;const status=e.status??e.statusCode;if(typeof status==="number"&&status>=400&&status<500)return!0;const name=e.name;if(typeof name==="string"&&(name==="ValidationError"||name==="ModelNotFoundError"))return!0;return!1}function getQueueDriver(){return resolveQueueDriver()}async function monitorWorker(id){const{getWorkerTracker,getGlobalMetrics}=await import("./events");let writing,warned=!1;const persist=()=>{if(writing)return writing;const worker=getWorkerTracker().getAll().find((worker)=>worker.id===id);if(!worker)return Promise.resolve();const metrics=getGlobalMetrics().getMetrics();writing=persistWorkerSnapshot({worker:{...worker},heartbeatAt:Date.now(),activeJobs:getActiveJobCount(),metrics:{completed:worker.processedCount,failed:worker.failedCount,throughputPerMinute:metrics.throughputPerMinute,averageDuration:metrics.averageDuration}}).catch((error)=>{if(warned)return;warned=!0;log.warn(`Queue worker telemetry unavailable. Run migrations for QueueWorker: ${error instanceof Error?error.message:String(error)}`)}).finally(()=>{writing=void 0});return writing};pruneWorkerSnapshots().catch(()=>{});persist();const timer=setInterval(()=>{persist()},1e4);timer.unref();return async()=>{clearInterval(timer);await writing;await persist()}}export async function startProcessor(queueName,options={}){let stopMonitor,monitoredId;try{log.info("Starting queue processor...");installWorkerCrashHandlers();workerRunning=!0;workerId=`worker-${process.pid}-${Date.now()}`;const concurrency=options.concurrency||1,queueDriver=getQueueDriver(),{getWorkerTracker,getGlobalMetrics}=await import("./events");getGlobalMetrics();getWorkerTracker().register(workerId,queueName||"*");monitoredId=workerId;stopMonitor=await monitorWorker(monitoredId);if(queueDriver==="redis"){log.info("Using Redis queue driver (bun-queue)");await processJobsFromRedis(queueName||"default",concurrency);return ok(void 0)}const named=namedWorkerQueues(queueName);let queues;if(named.length>0)queues=named;else{queues=await getAllQueues();if(queues.length===0)queues=["default"]}log.info(`Processing queues: ${queues.join(", ")}`);await processJobsFromDatabase(queues,concurrency,{followNewQueues:named.length===0});return ok(void 0)}catch(error){workerRunning=!1;return err(error)}finally{const{getWorkerTracker}=await import("./events");if(monitoredId)getWorkerTracker().unregister(monitoredId);await stopMonitor?.()}}async function getAllQueues(){try{const{db}=await import("@stacksjs/database/runtime"),queues=(await db.selectFrom("jobs").select("queue").distinct().execute()).map((r)=>r.queue).filter((q)=>Boolean(q));return queues.length>0?queues:["default"]}catch{return["default"]}}export function namedWorkerQueues(queueName){if(typeof queueName!=="string")return[];return[...new Set(queueName.split(",").map((name)=>name.trim()).filter(Boolean))]}async function processJobsFromDatabase(initialQueues,concurrency,options={followNewQueues:!0}){log.info("Listening for jobs...");let queues=initialQueues,lastQueueRefresh=Date.now();const queueRefreshInterval=1e4;let lastSweep=Date.now();const sweepIntervalMs=getSweepIntervalMs();await sweepStaleReservations();while(workerRunning)try{let processedJobs=!1;const now=Date.now();if(options.followNewQueues&&now-lastQueueRefresh>queueRefreshInterval){try{const refreshedQueues=await getAllQueues();if(refreshedQueues.length>0)queues=refreshedQueues}catch{}lastQueueRefresh=now}if(now-lastSweep>sweepIntervalMs){await sweepStaleReservations();lastSweep=now}for(const queueName of queues){if(!workerRunning)break;try{const{isCircuitOpen}=await import("./circuit-breaker");if(await isCircuitOpen(queueName))continue}catch{}if(!workerRunning)break;const processed=await trackInFlight(processDatabaseBatch(queueName,concurrency),inFlightDatabaseBatches);processedJobs||=processed}if(workerRunning&&!processedJobs)await sleep(1000)}catch{await sleep(3000)}}async function processDatabaseBatch(queueName,concurrency){let jobs=[];try{jobs=await fetchPendingJobs(queueName,concurrency)}catch(error){const now=Date.now(),last=reserveErrorLoggedAt.get(queueName)??0;if(now-last>60000){reserveErrorLoggedAt.set(queueName,now);log.error(`[queue] Could not reserve jobs on "${queueName}" - retrying each cycle:`,error)}return!1}await Promise.all(jobs.map(async(job)=>{try{log.info(`Processing job ${job.id} from queue "${queueName}"`);await trackInFlight(processJob(job))}catch{log.error(`Unexpected error processing job ${job.id}`)}}));return jobs.length>0}async function fetchPendingJobs(queueName,limit){const now=Math.floor(Date.now()/1000),{db}=await import("@stacksjs/database/runtime"),claimed=[];for(let i=0;i<limit&&workerRunning;i++){const job=await db.selectFrom("jobs").where("queue","=",queueName).whereNull("reserved_at").where("available_at","<=",now).orderBy("id","asc").limit(1).selectAll().executeTakeFirst();if(!job||!workerRunning)break;const result=await db.updateTable("jobs").set({reserved_at:now,attempts:Number(job.attempts||0)+1}).where("id","=",job.id).whereNull("reserved_at").executeTakeFirst();if(updatedRowCount(result)>0)claimed.push(job)}return claimed}async function processJob(job){const jobId=job.id,queueName=job.queue||"default";activeJobCount++;const startTime=Date.now(),{emitQueueEvent,getWorkerTracker}=await import("./events"),tracker=getWorkerTracker();tracker.markActive(workerId);let parsedJobName;try{parsedJobName=JSON.parse(job.payload||"{}").jobName}catch{}await emitQueueEvent("job:processing",{jobId:String(jobId),queueName,jobName:parsedJobName});try{const parsedForBatch=JSON.parse(job.payload||"{}"),batchId=batchMembershipId(parsedForBatch.payload);if(batchId){const{isBatchCancelled}=await import("./batch");if(await isBatchCancelled(batchId)){log.info(`[Queue] Skipping job ${jobId} - batch ${batchId} has been cancelled`);await deleteJob(jobId);activeJobCount--;tracker.markIdle(workerId);return}}}catch{}let jobError=null;const declared=parsedJobName?await(await import("./job")).jobDefaults(parsedJobName):{};try{const payload=JSON.parse(job.payload||"{}"),timeoutSec=readJobTimeoutSec(payload)??declared.timeout;if(timeoutSec===void 0)await executeJobPayload(payload);else await raceWithTimeout(executeJobPayload(payload),timeoutSec*1000,`Job ${jobId} exceeded ${timeoutSec}s timeout`)}catch(e){jobError=e instanceof Error?e:Error(String(e))}if(!jobError)try{await deleteJob(jobId);log.info(`[Queue] Job ${jobId} completed`);tracker.recordCompletion(workerId);try{const{recordCircuitSuccess}=await import("./circuit-breaker");await recordCircuitSuccess(job.queue??"default")}catch{}await emitQueueEvent("job:completed",{jobId:String(jobId),queueName,duration:Date.now()-startTime});try{const payload=JSON.parse(job.payload||"{}"),batchId=batchMembershipId(payload.payload);if(batchId){const{recordBatchJobCompletion}=await import("./batch");await recordBatchJobCompletion(batchId)}}catch{}}catch{log.info(`[Queue] Failed to delete completed job ${jobId}`)}else{const errorMessage=jobError.message;log.info(`[Queue] Job ${jobId} failed: ${errorMessage}`);tracker.recordFailure(workerId);await emitQueueEvent("job:failed",{jobId:String(jobId),queueName,error:jobError,duration:Date.now()-startTime,attemptsMade:(job.attempts||0)+1});let maxAttempts=1,parsedPayload={};try{parsedPayload=JSON.parse(job.payload||"{}");maxAttempts=parsedPayload.options?.tries||declared.tries||1}catch{}const currentAttempts=(job.attempts||0)+1;if(currentAttempts>=maxAttempts){log.info(`[Queue] Job ${jobId} exceeded max attempts (${currentAttempts}/${maxAttempts})`);let persisted=!1;if(parsedPayload?._retriedFromFailed===!0)try{const{moveToDeadLetter}=await import("./dead-letter");persisted=await moveToDeadLetter({queue:job.queue,payload:job.payload,exception:jobError.stack||jobError.message},"repeat-failure",2);if(persisted)log.info(`[Queue] Job ${jobId} re-failed after retry - moved to dead_letter_jobs`)}catch{persisted=!1}if(!persisted){log.info(`[Queue] Moving job ${jobId} to failed_jobs`);try{persisted=await moveToFailedJobs(job,jobError,{attempts:currentAttempts,maxAttempts,durationMs:Date.now()-startTime})}catch{persisted=!1}}if(persisted)try{await deleteJob(jobId)}catch{log.info(`[Queue] Failed to delete failed job ${jobId}`)}else log.error(`[Queue] Job ${jobId} exhausted its retries but could NOT be persisted to failed_jobs - leaving it in the queue to avoid data loss (the reservation sweep will retry it). Check that the failed_jobs table exists and is writable.`);try{const{recordFailureForPoison}=await import("./poison"),{recordCircuitFailure}=await import("./circuit-breaker"),jobName=parsedPayload?.jobName??"unknown";await recordFailureForPoison(jobName,parsedPayload?.payload);await recordCircuitFailure(job.queue??"default")}catch{}try{const batchId=batchMembershipId(parsedPayload.payload);if(batchId){const{recordBatchJobFailure}=await import("./batch");await recordBatchJobFailure(batchId,String(jobId),jobError)}}catch{}}else{const backoffDelays=parsedPayload.options?.backoff??declared.backoff;let retryDelay=30;if(Array.isArray(backoffDelays)&&backoffDelays.length>0){const backoffIndex=Math.min(currentAttempts-1,backoffDelays.length-1);retryDelay=backoffDelays[backoffIndex]}else if(typeof backoffDelays==="number"&&backoffDelays>0)retryDelay=backoffDelays;retryDelay=Number(retryDelay);if(!Number.isFinite(retryDelay)||retryDelay<0)retryDelay=30;retryDelay=applyRetryJitter(retryDelay,getRetryJitterRatio());log.info(`[Queue] Job ${jobId} will be retried in ${retryDelay}s (attempt ${currentAttempts}/${maxAttempts})`);try{await releaseJob(jobId,retryDelay);log.info(`[Queue] Job ${jobId} released for retry`)}catch{log.info(`[Queue] Failed to release job ${jobId} for retry`)}}}activeJobCount--;tracker.markIdle(workerId)}async function deleteJob(jobId){const{db}=await import("@stacksjs/database/runtime");await db.deleteFrom("jobs").where("id","=",jobId).execute()}async function releaseJob(jobId,delaySeconds=30){const retryAt=Math.floor(Date.now()/1000)+delaySeconds;log.debug(`Releasing job ${jobId} for retry at ${retryAt}`);try{const{db}=await import("@stacksjs/database/runtime");await db.updateTable("jobs").set({reserved_at:null,available_at:retryAt}).where("id","=",jobId).execute();log.debug(`Job ${jobId} released successfully`)}catch{log.error(`Failed to release job ${jobId}`)}}async function moveToFailedJobs(job,error,metrics){try{const failedAt=new Date().toISOString().slice(0,19).replace("T"," "),uuid=crypto.randomUUID(),exception=error.stack||error.message,{db}=await import("@stacksjs/database/runtime");await db.insertInto("failed_jobs").values({uuid,connection:"database",queue:job.queue,payload:job.payload,exception,attempts:metrics.attempts,max_attempts:metrics.maxAttempts,duration_ms:metrics.durationMs,failed_at:failedAt}).execute();return!0}catch(insertError){log.error("Failed to log failed job:",insertError);return!1}}function readJobTimeoutSec(payload){if(!payload||typeof payload!=="object")return;const t=payload.options?.timeout;if(typeof t!=="number"||!Number.isFinite(t)||t<=0)return;return t}async function raceWithTimeout(task,timeoutMs,message){let timer;const timeout=new Promise((_,reject)=>{timer=setTimeout(()=>reject(Error(message)),timeoutMs)});try{return await Promise.race([task,timeout])}finally{if(timer!==void 0)clearTimeout(timer)}}async function executeJobPayload(payload){const parsed=parseEnvelope(payload);if(!parsed.ok)throw Error(`[queue] Cannot deserialize job envelope: ${parsed.reason}`+(parsed.detail?` (${parsed.detail})`:""));const{runJob}=await import("./job");await runJob(parsed.envelope.jobName,{payload:parsed.envelope.payload,context:parsed.envelope.context,traceId:parsed.envelope.traceId})}async function processJobsFromRedis(queueName,concurrency){const{RedisQueue}=await import("./drivers/redis"),{queue:queueConfig}=await import("@stacksjs/config"),redisConfig=queueConfig?.connections?.redis;if(!redisConfig)throw Error("Redis queue connection is not configured. Check config/queue.ts");const queue=new RedisQueue(queueName,redisConfig),{emitQueueEvent,getWorkerTracker}=await import("./events"),tracker=getWorkerTracker(),handleRedisJob=async(bunJob)=>{activeJobCount++;tracker.markActive(workerId);const startTime=Date.now(),parsed=parseEnvelope(bunJob.data);if(!parsed.ok){activeJobCount--;tracker.markIdle(workerId);log.error(`[Queue] Skipping Redis job ${bunJob.id} - unparseable envelope: ${parsed.reason}${parsed.detail?` (${parsed.detail})`:""}`);return}const data=parsed.envelope,batchId=batchMembershipId(data.payload);if(batchId)try{const{isBatchCancelled}=await import("./batch");if(await isBatchCancelled(batchId)){log.info(`[Queue] Skipping Redis job ${bunJob.id} - batch ${batchId} has been cancelled`);activeJobCount--;tracker.markIdle(workerId);return}}catch{}await emitQueueEvent("job:processing",{jobId:String(bunJob.id),queueName});try{const{runJob}=await import("./job");await runJob(data.jobName,{payload:data.payload,context:data.context,traceId:data.traceId});tracker.recordCompletion(workerId);await emitQueueEvent("job:completed",{jobId:String(bunJob.id),queueName,duration:Date.now()-startTime});if(batchId)try{const{recordBatchJobCompletion}=await import("./batch");await recordBatchJobCompletion(batchId)}catch{}log.info(`[Queue] Redis job ${bunJob.id} completed`)}catch(e){tracker.recordFailure(workerId);await emitQueueEvent("job:failed",{jobId:String(bunJob.id),queueName,error:e instanceof Error?e:Error(String(e)),duration:Date.now()-startTime});if(batchId)try{const{recordBatchJobFailure}=await import("./batch");await recordBatchJobFailure(batchId,String(bunJob.id),e instanceof Error?e:Error(String(e)))}catch{}log.error(`[Queue] Redis job ${bunJob.id} failed: ${e}`);if(isNonRetryableError(e)){log.info(`[Queue] Redis job ${bunJob.id} hit a non-retryable error - skipping retry`);return}throw e}finally{activeJobCount--;tracker.markIdle(workerId)}};queue.process(concurrency,(bunJob)=>trackInFlight(handleRedisJob(bunJob)));log.info(`Listening for Redis jobs on queue "${queueName}" with concurrency ${concurrency}...`);while(workerRunning)await sleep(1000);await queue.close()}export async function stopProcessor(options={}){const graceMs=options.graceMs??1e4;workerRunning=!1;const pending=new Set([...inFlightDatabaseBatches,...inFlightJobs]);if(pending.size>0&&graceMs>0){log.info(`[queue] Draining database reservations and in-flight jobs (grace ${graceMs}ms)`);const drain=Promise.allSettled([...pending]),timeout=new Promise((resolve)=>setTimeout(()=>resolve("timeout"),graceMs));if(await Promise.race([drain.then(()=>"drained"),timeout])==="timeout"&&(inFlightJobs.size>0||inFlightDatabaseBatches.size>0))log.warn("[queue] Drain timed out with reservations or jobs still active. Their reservations will be reclaimed by the next worker's sweep (Q-2).")}if(workerId){const{getWorkerTracker}=await import("./events");getWorkerTracker().unregister(workerId)}if(getQueueDriver()==="redis"){const{closeSharedRedisQueues}=await import("./drivers/redis");await closeSharedRedisQueues()}log.info("Queue processor stopped")}export async function executeFailedJobs(){const{db}=await import("@stacksjs/database/runtime"),failedJobs=await db.selectFrom("failed_jobs").selectAll().execute();for(const failedJob of failedJobs)await retryFailedJob(Number(failedJob.id))}export async function retryFailedJob(id){const now=Math.floor(Date.now()/1000),createdAt=new Date().toISOString().slice(0,19).replace("T"," "),{db}=await import("@stacksjs/database/runtime"),failedJob=(await db.selectFrom("failed_jobs").where("id","=",id).selectAll().execute())[0];if(!failedJob)throw Error(`Failed job ${id} not found`);const storedPayload=typeof failedJob.payload==="string"?failedJob.payload:"";let payloadToRequeue=storedPayload;try{const env=JSON.parse(storedPayload||"{}");env._retriedFromFailed=!0;payloadToRequeue=JSON.stringify(env)}catch{}await db.insertInto("jobs").values({queue:failedJob.queue,payload:payloadToRequeue,attempts:0,reserved_at:null,available_at:now,created_at:createdAt}).execute();await db.deleteFrom("failed_jobs").where("id","=",id).execute();log.info(`Failed job ${id} has been re-queued`)}export function getActiveJobCount(){return activeJobCount}export function isWorkerRunning(){return workerRunning}function sleep(ms){return new Promise((resolve)=>setTimeout(resolve,ms))}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@stacksjs/queue",
   "type": "module",
   "sideEffects": false,
-  "version": "0.75.484",
+  "version": "0.75.487",
   "description": "The Stacks Queue system powered by bun-queue.",
   "author": "Chris Breuer",
   "contributors": [
@@ -63,23 +63,23 @@
     "prepublishOnly": "bun run build"
   },
   "dependencies": {
-    "@stacksjs/action-runner": "0.75.484",
+    "@stacksjs/action-runner": "0.75.487",
     "@stacksjs/bun-queue": "^0.1.8",
-    "@stacksjs/cache": "0.75.484",
-    "@stacksjs/config": "0.75.484",
-    "@stacksjs/cron": "0.75.484",
-    "@stacksjs/database": "0.75.484",
-    "@stacksjs/env": "0.75.484",
-    "@stacksjs/error-handling": "0.75.484",
-    "@stacksjs/logging": "0.75.484",
-    "@stacksjs/path": "0.75.484"
+    "@stacksjs/cache": "0.75.487",
+    "@stacksjs/config": "0.75.487",
+    "@stacksjs/cron": "0.75.487",
+    "@stacksjs/database": "0.75.487",
+    "@stacksjs/env": "0.75.487",
+    "@stacksjs/error-handling": "0.75.487",
+    "@stacksjs/logging": "0.75.487",
+    "@stacksjs/path": "0.75.487"
   },
   "devDependencies": {
     "better-dx": "^0.2.24"
   },
   "peerDependencies": {
-    "@stacksjs/email": "0.75.484",
-    "@stacksjs/router": "0.75.484"
+    "@stacksjs/email": "0.75.487",
+    "@stacksjs/router": "0.75.487"
   },
   "peerDependenciesMeta": {
     "@stacksjs/email": {