@workglow/job-queue 0.2.23 → 0.2.24

package/dist/browser.js

@@ -303,6 +303,7 @@ class JobQueueClient {
       status: JobStatus2.PENDING
     };
     const id = await this.storage.add(job);
+    this.server?.handleJobAdded(id);
     return this.createJobHandle(id);
   }
   async submitBatch(inputs, options) {
@@ -342,29 +343,52 @@ class JobQueueClient {
   async waitFor(jobId) {
     if (!jobId)
       throw new JobNotFoundError("Cannot wait for undefined job");
+    const { promise, resolve, reject } = Promise.withResolvers();
+    promise.catch(() => {});
+    const promises = this.activeJobPromises.get(jobId) || [];
+    promises.push({ resolve, reject });
+    this.activeJobPromises.set(jobId, promises);
     const job = await this.getJob(jobId);
-    if (!job)
+    if (!job) {
+      this.removePromise(jobId, resolve, reject);
       throw new JobNotFoundError(`Job ${jobId} not found`);
+    }
     if (job.status === JobStatus2.COMPLETED) {
+      this.removePromise(jobId, resolve, reject);
       return job.output;
     }
     if (job.status === JobStatus2.DISABLED) {
+      this.removePromise(jobId, resolve, reject);
       throw new JobDisabledError(`Job ${jobId} was disabled`);
     }
     if (job.status === JobStatus2.FAILED) {
+      this.removePromise(jobId, resolve, reject);
       throw this.buildErrorFromJob(job);
     }
-    const { promise, resolve, reject } = Promise.withResolvers();
-    promise.catch(() => {});
-    const promises = this.activeJobPromises.get(jobId) || [];
-    promises.push({ resolve, reject });
-    this.activeJobPromises.set(jobId, promises);
     return promise;
   }
+  removePromise(jobId, resolve, reject) {
+    const list = this.activeJobPromises.get(jobId);
+    if (!list)
+      return;
+    const idx = list.findIndex((p) => p.resolve === resolve && p.reject === reject);
+    if (idx !== -1)
+      list.splice(idx, 1);
+    if (list.length === 0)
+      this.activeJobPromises.delete(jobId);
+  }
   async abort(jobId) {
     if (!jobId)
       throw new JobNotFoundError("Cannot abort undefined job");
-    await this.storage.abort(jobId);
+    const firedLocally = this.server?.abortJob(jobId) ?? false;
+    if (!firedLocally) {
+      try {
+        await this.storage.abort(jobId);
+      } finally {
+        this.events.emit("job_aborting", this.queueName, jobId);
+      }
+      return;
+    }
     this.events.emit("job_aborting", this.queueName, jobId);
   }
   async abortJobRun(jobRunId) {
@@ -531,7 +555,7 @@ class JobQueueClient {
 
 // src/job/JobQueueServer.ts
 import { JobStatus as JobStatus4 } from "@workglow/storage";
-import { EventEmitter as EventEmitter3 } from "@workglow/util";
+import { EventEmitter as EventEmitter3, getLogger as getLogger2 } from "@workglow/util";
 
 // src/limiter/NullLimiter.ts
 import { createServiceToken } from "@workglow/util";
@@ -567,10 +591,14 @@ class JobQueueWorker {
   jobClass;
   limiter;
   pollIntervalMs;
+  stopTimeoutMs;
   events = new EventEmitter2;
   running = false;
+  inFlight = new Map;
   wakeResolve = null;
   wakeTimer = null;
+  wakePending = false;
+  loopPromise = null;
   activeJobAbortControllers = new Map;
   processingTimes = new Map;
   constructor(jobClass, options) {
@@ -580,6 +608,7 @@ class JobQueueWorker {
     this.jobClass = jobClass;
     this.limiter = options.limiter ?? new NullLimiter;
     this.pollIntervalMs = options.pollIntervalMs ?? 100;
+    this.stopTimeoutMs = options.stopTimeoutMs ?? 30000;
   }
   async start() {
     if (this.running) {
@@ -587,17 +616,27 @@ class JobQueueWorker {
     }
     this.running = true;
     this.events.emit("worker_start");
-    this.processJobs();
+    this.loopPromise = this.processJobs();
     return this;
   }
+  requestAbort(jobId) {
+    const controller = this.activeJobAbortControllers.get(jobId);
+    if (controller && !controller.signal.aborted) {
+      controller.abort();
+      return true;
+    }
+    return false;
+  }
   notify() {
+    this.wakePending = true;
     if (this.wakeResolve) {
       if (this.wakeTimer) {
         clearTimeout(this.wakeTimer);
         this.wakeTimer = null;
       }
-      this.wakeResolve();
+      const resolve = this.wakeResolve;
       this.wakeResolve = null;
+      resolve();
     }
   }
   async stop() {
@@ -606,15 +645,24 @@ class JobQueueWorker {
     }
     this.running = false;
     this.notify();
-    const size = await this.storage.size(JobStatus3.PROCESSING);
-    const sleepTime = Math.max(100, size * 2);
-    await sleep(sleepTime);
-    for (const controller of this.activeJobAbortControllers.values()) {
-      if (!controller.signal.aborted) {
-        controller.abort();
+    const loopPromise = this.loopPromise;
+    this.loopPromise = null;
+    if (loopPromise) {
+      await loopPromise;
+    }
+    if (this.stopTimeoutMs > 0 && this.inFlight.size > 0) {
+      const drain = Promise.allSettled([...this.inFlight.values()]);
+      await Promise.race([drain, sleep(this.stopTimeoutMs)]);
+    }
+    if (this.inFlight.size > 0) {
+      for (const controller of this.activeJobAbortControllers.values()) {
+        if (!controller.signal.aborted) {
+          controller.abort();
+        }
       }
+      const abortDrain = Promise.allSettled([...this.inFlight.values()]);
+      await Promise.race([abortDrain, sleep(1000)]);
     }
-    await sleep(sleepTime);
     this.events.emit("worker_stop");
     return this;
   }
@@ -662,6 +710,10 @@ class JobQueueWorker {
         if (canProceed) {
           const job = await this.next();
           if (job) {
+            if (!this.running) {
+              await this.releaseClaimedJob(job);
+              return;
+            }
             this.processSingleJob(job);
             continue;
           }
@@ -690,18 +742,27 @@ class JobQueueWorker {
     return this.pollIntervalMs;
   }
   waitForWakeOrTimeout(timeoutMs) {
+    if (this.wakePending) {
+      this.wakePending = false;
+      return Promise.resolve();
+    }
     return new Promise((resolve) => {
       this.wakeTimer = setTimeout(() => {
         this.wakeTimer = null;
         this.wakeResolve = null;
+        this.wakePending = false;
         resolve();
       }, timeoutMs);
       this.wakeResolve = () => {
+        this.wakePending = false;
         resolve();
       };
     });
   }
   async checkForAbortingJobs() {
+    if (this.activeJobAbortControllers.size === 0) {
+      return;
+    }
     const abortingJobs = await this.storage.peek(JobStatus3.ABORTING);
     for (const jobData of abortingJobs) {
       const controller = this.activeJobAbortControllers.get(jobData.id);
@@ -714,6 +775,8 @@ class JobQueueWorker {
     if (!job || !job.id) {
       throw new JobNotFoundError("Invalid job provided for processing");
     }
+    const { promise: inFlightPromise, resolve: resolveInFlight } = Promise.withResolvers();
+    this.inFlight.set(job.id, inFlightPromise);
     const startTime = Date.now();
     const telemetry = getTelemetryProvider();
     const span = telemetry.isEnabled ? telemetry.startSpan("workglow.job.process", {
@@ -764,7 +827,12 @@ class JobQueueWorker {
       span?.setAttributes({ "workglow.job.error": spanErrorMessage });
     } finally {
       span?.end();
-      await this.limiter.recordJobCompletion();
+      try {
+        await this.limiter.recordJobCompletion();
+      } finally {
+        this.inFlight.delete(job.id);
+        resolveInFlight();
+      }
     }
   }
   async executeJob(job, signal) {
@@ -777,7 +845,6 @@ class JobQueueWorker {
   }
   async updateProgress(jobId, progress, message = "", details = null) {
     progress = Math.max(0, Math.min(100, progress));
-    await this.storage.saveProgress(jobId, progress, message, details);
     this.events.emit("job_progress", jobId, progress, message, details);
   }
   async completeJob(job, output) {
@@ -830,6 +897,13 @@ class JobQueueWorker {
       this.cleanupJob(job.id);
     }
   }
+  async releaseClaimedJob(job) {
+    try {
+      await this.storage.release(job.id);
+    } catch (err) {
+      getLogger().error("releaseClaimedJob errored:", { error: err });
+    }
+  }
   async rescheduleJob(job, retryDate) {
     try {
       job.status = JobStatus3.PENDING;
@@ -848,6 +922,9 @@ class JobQueueWorker {
   createAbortController(jobId) {
     if (!jobId)
       throw new JobNotFoundError("Cannot create abort controller for undefined job");
+    if (!this.inFlight.has(jobId)) {
+      throw new Error(`createAbortController invariant violated: jobId ${String(jobId)} is not in inFlight. ` + `Abort controllers must only be created from within processSingleJob.`);
+    }
     if (this.activeJobAbortControllers.has(jobId)) {
       return this.activeJobAbortControllers.get(jobId);
     }
@@ -857,13 +934,18 @@ class JobQueueWorker {
     return abortController;
   }
   async handleAbort(jobId) {
+    if (this.inFlight.has(jobId)) {
+      return;
+    }
     const job = await this.getJob(jobId);
     if (!job) {
       getLogger().error("handleAbort: job not found", { jobId });
       return;
     }
-    const error = new AbortSignalJobError("Job Aborted");
-    await this.failJob(job, error);
+    if (job.status === JobStatus3.COMPLETED || job.status === JobStatus3.FAILED || job.status === JobStatus3.DISABLED) {
+      return;
+    }
+    await this.failJob(job, new AbortSignalJobError("Job Aborted"));
   }
   async getJob(id) {
     const job = await this.storage.get(id);
@@ -920,6 +1002,7 @@ class JobQueueServer {
   deleteAfterFailureMs;
   deleteAfterDisabledMs;
   cleanupIntervalMs;
+  stopTimeoutMs;
   events = new EventEmitter3;
   workers = [];
   clients = new Set;
@@ -946,6 +1029,7 @@ class JobQueueServer {
     this.deleteAfterFailureMs = options.deleteAfterFailureMs;
     this.deleteAfterDisabledMs = options.deleteAfterDisabledMs;
     this.cleanupIntervalMs = options.cleanupIntervalMs ?? 1e4;
+    this.stopTimeoutMs = options.stopTimeoutMs;
     this.initializeWorkers();
   }
   async start() {
@@ -961,11 +1045,21 @@ class JobQueueServer {
           this.notifyWorkers();
         }
       });
-    } catch {}
+    } catch (err) {
+      getLogger2().debug("subscribeToChanges unsupported on this storage", {
+        queueName: this.queueName,
+        error: err
+      });
+    }
     await Promise.all(this.workers.map((worker) => worker.start()));
-    this.startCleanupLoop();
+    if (this.hasRetentionTtls()) {
+      this.startCleanupLoop();
+    }
     return this;
   }
+  hasRetentionTtls() {
+    return this.deleteAfterCompletionMs !== undefined && this.deleteAfterCompletionMs > 0 || this.deleteAfterFailureMs !== undefined && this.deleteAfterFailureMs > 0 || this.deleteAfterDisabledMs !== undefined && this.deleteAfterDisabledMs > 0;
+  }
   async stop() {
     if (!this.running) {
       return this;
@@ -1024,6 +1118,17 @@ class JobQueueServer {
       worker.notify();
     }
   }
+  handleJobAdded(_jobId) {
+    this.notifyWorkers();
+  }
+  abortJob(jobId) {
+    for (const worker of this.workers) {
+      if (worker.requestAbort(jobId)) {
+        return true;
+      }
+    }
+    return false;
+  }
   on(event, listener) {
     this.events.on(event, listener);
   }
@@ -1041,7 +1146,8 @@ class JobQueueServer {
       storage: this.storage,
       queueName: this.queueName,
       limiter: this.limiter,
-      pollIntervalMs: this.pollIntervalMs
+      pollIntervalMs: this.pollIntervalMs,
+      stopTimeoutMs: this.stopTimeoutMs
     });
     worker.on("job_start", (jobId) => {
       this.stats = { ...this.stats, totalJobs: this.stats.totalJobs + 1 };
@@ -1490,4 +1596,4 @@ export {
   AbortSignalJobError
 };
 
-//# debugId=A8F0F2BEFB3C518064756E2164756E21
+//# debugId=DEA35BFCE014423364756E2164756E21