@workglow/job-queue 0.2.25 → 0.2.27

package/dist/bun.js

@@ -563,6 +563,12 @@ import { createServiceToken } from "@workglow/util";
 var NULL_JOB_LIMITER = createServiceToken("jobqueue.limiter.null");
 
 class NullLimiter {
+  scope = "process";
+  static SENTINEL = Symbol("NullLimiter.acquired");
+  async tryAcquire() {
+    return NullLimiter.SENTINEL;
+  }
+  async release(_token) {}
   async canProceed() {
     return true;
   }
@@ -585,6 +591,8 @@ import {
   SpanStatusCode,
   uuid4
 } from "@workglow/util";
+var MAX_LIMITER_WAKE_MS = 30000;
+
 class JobQueueWorker {
   queueName;
   workerId;
@@ -668,15 +676,16 @@ class JobQueueWorker {
     return this;
   }
   async processNext() {
-    const canProceed = await this.limiter.canProceed();
-    if (!canProceed) {
-      return false;
-    }
     const job = await this.next();
     if (!job) {
       return false;
     }
-    await this.processSingleJob(job);
+    const limiterToken = await this.limiter.tryAcquire();
+    if (limiterToken === null || limiterToken === undefined) {
+      await this.releaseClaimedJob(job);
+      return false;
+    }
+    await this.processSingleJob(job, limiterToken);
     return true;
   }
   isRunning() {
@@ -707,29 +716,46 @@ class JobQueueWorker {
     while (this.running) {
       try {
         await this.checkForAbortingJobs();
-        const canProceed = await this.limiter.canProceed();
-        if (canProceed) {
-          const job = await this.next();
-          if (job) {
-            if (!this.running) {
-              await this.releaseClaimedJob(job);
-              return;
-            }
-            this.processSingleJob(job);
-            continue;
-          }
-        }
-        if (canProceed) {
+        const job = await this.next();
+        if (!job) {
           const delay = await this.getIdleDelay();
           await this.waitForWakeOrTimeout(delay);
-        } else {
-          await this.waitForWakeOrTimeout(this.pollIntervalMs);
+          continue;
+        }
+        if (!this.running) {
+          await this.releaseClaimedJob(job);
+          return;
+        }
+        const limiterToken = await this.limiter.tryAcquire();
+        if (limiterToken === null || limiterToken === undefined) {
+          await this.releaseClaimedJob(job);
+          await this.waitForWakeOrTimeout(await this.getLimiterWakeDelay());
+          continue;
         }
+        if (!this.running) {
+          try {
+            await this.limiter.release(limiterToken);
+          } catch {}
+          await this.releaseClaimedJob(job);
+          return;
+        }
+        this.processSingleJob(job, limiterToken);
       } catch {
         await sleep(this.pollIntervalMs);
       }
     }
   }
+  async getLimiterWakeDelay() {
+    try {
+      const next = await this.limiter.getNextAvailableTime();
+      const delay = next.getTime() - Date.now();
+      if (delay <= 0)
+        return this.pollIntervalMs;
+      return Math.min(Math.max(delay, this.pollIntervalMs), MAX_LIMITER_WAKE_MS);
+    } catch {
+      return this.pollIntervalMs;
+    }
+  }
   async getIdleDelay() {
     try {
       const pending = await this.storage.peek(JobStatus3.PENDING, 1);
@@ -772,7 +798,7 @@ class JobQueueWorker {
       }
     }
   }
-  async processSingleJob(job) {
+  async processSingleJob(job, limiterToken) {
     if (!job || !job.id) {
       throw new JobNotFoundError("Invalid job provided for processing");
     }
@@ -789,9 +815,17 @@ class JobQueueWorker {
         "workglow.job.max_retries": job.maxRetries
       }
     }) : undefined;
+    let slotReleased = false;
     try {
-      await this.validateJobState(job);
-      await this.limiter.recordJobStart();
+      try {
+        await this.validateJobState(job);
+      } catch (validationErr) {
+        try {
+          await this.limiter.release(limiterToken);
+          slotReleased = true;
+        } catch {}
+        throw validationErr;
+      }
       const abortController = this.createAbortController(job.id);
       this.events.emit("job_start", job.id);
       const output = await this.executeJob(job, abortController.signal);
@@ -829,7 +863,9 @@ class JobQueueWorker {
     } finally {
       span?.end();
       try {
-        await this.limiter.recordJobCompletion();
+        if (!slotReleased) {
+          await this.limiter.recordJobCompletion();
+        }
       } finally {
         this.inFlight.delete(job.id);
         resolveInFlight();
@@ -1039,10 +1075,17 @@ class JobQueueServer {
     }
     this.running = true;
     this.events.emit("server_start", this.queueName);
+    if (this.limiter.scope === "process" && this.storage.scope === "cluster" && !(this.limiter instanceof NullLimiter)) {
+      getLogger2().warn("Process-scoped limiter on cluster-scoped queue storage \u2014 limit is enforced per-process, not cluster-wide. Use a cluster-scoped rate limiter storage (Postgres/Supabase) for global enforcement.", {
+        queueName: this.queueName,
+        limiterScope: this.limiter.scope,
+        storage: this.storage.constructor.name
+      });
+    }
     await this.fixupJobs();
     try {
       this.storageUnsubscribe = this.storage.subscribeToChanges((change) => {
-        if (change.type === "INSERT" || change.type === "UPDATE" && change.new?.status === JobStatus4.PENDING) {
+        if (change.type === "INSERT" || change.type === "RESYNC" || change.type === "UPDATE" && change.new?.status === JobStatus4.PENDING) {
           this.notifyWorkers();
         }
       });
@@ -1297,6 +1340,9 @@ class CompositeLimiter {
   constructor(limiters = []) {
     this.limiters = limiters;
   }
+  get scope() {
+    return this.limiters.every((l) => l.scope === "cluster") && this.limiters.length > 0 ? "cluster" : "process";
+  }
   addLimiter(limiter) {
     this.limiters.push(limiter);
   }
@@ -1308,6 +1354,27 @@ class CompositeLimiter {
     }
     return true;
   }
+  async tryAcquire() {
+    const tokens = [];
+    for (const limiter of this.limiters) {
+      const t = await limiter.tryAcquire();
+      if (t === null || t === undefined) {
+        for (let i = tokens.length - 1;i >= 0; i--) {
+          try {
+            await this.limiters[i].release(tokens[i]);
+          } catch {}
+        }
+        return null;
+      }
+      tokens.push(t);
+    }
+    return tokens;
+  }
+  async release(token) {
+    if (!Array.isArray(token))
+      return;
+    await Promise.all(this.limiters.map((l, i) => l.release(token[i]).catch(() => {})));
+  }
   async recordJobStart() {
     await Promise.all(this.limiters.map((limiter) => limiter.recordJobStart()));
   }
@@ -1339,6 +1406,7 @@ import { createServiceToken as createServiceToken2 } from "@workglow/util";
 var CONCURRENT_JOB_LIMITER = createServiceToken2("jobqueue.limiter.concurrent");
 
 class ConcurrencyLimiter {
+  scope = "process";
   currentRunningJobs = 0;
   maxConcurrentJobs;
   nextAllowedStartTime = new Date;
@@ -1348,6 +1416,19 @@ class ConcurrencyLimiter {
   async canProceed() {
     return this.currentRunningJobs < this.maxConcurrentJobs && Date.now() >= this.nextAllowedStartTime.getTime();
   }
+  static SENTINEL = Symbol("ConcurrencyLimiter.acquired");
+  async tryAcquire() {
+    if (this.currentRunningJobs >= this.maxConcurrentJobs || Date.now() < this.nextAllowedStartTime.getTime()) {
+      return null;
+    }
+    this.currentRunningJobs++;
+    return ConcurrencyLimiter.SENTINEL;
+  }
+  async release(token) {
+    if (token !== ConcurrencyLimiter.SENTINEL)
+      return;
+    this.currentRunningJobs = Math.max(0, this.currentRunningJobs - 1);
+  }
   async recordJobStart() {
     this.currentRunningJobs++;
   }
@@ -1371,13 +1452,32 @@ class ConcurrencyLimiter {
 // src/limiter/DelayLimiter.ts
 class DelayLimiter {
   delayInMilliseconds;
+  scope = "process";
   nextAvailableTime = new Date;
+  lastAcquireBaseline = 0;
   constructor(delayInMilliseconds = 50) {
     this.delayInMilliseconds = delayInMilliseconds;
   }
   async canProceed() {
     return Date.now() >= this.nextAvailableTime.getTime();
   }
+  async tryAcquire() {
+    const now = Date.now();
+    if (now < this.nextAvailableTime.getTime()) {
+      return null;
+    }
+    const previous = this.nextAvailableTime.getTime();
+    this.lastAcquireBaseline = previous;
+    this.nextAvailableTime = new Date(now + this.delayInMilliseconds);
+    return previous;
+  }
+  async release(token) {
+    if (typeof token !== "number")
+      return;
+    if (this.nextAvailableTime.getTime() === this.lastAcquireBaseline + this.delayInMilliseconds) {
+      this.nextAvailableTime = new Date(token);
+    }
+  }
   async recordJobStart() {
     this.nextAvailableTime = new Date(Date.now() + this.delayInMilliseconds);
   }
@@ -1400,12 +1500,14 @@ import { createServiceToken as createServiceToken3 } from "@workglow/util";
 var EVENLY_SPACED_JOB_RATE_LIMITER = createServiceToken3("jobqueue.limiter.rate.evenlyspaced");
 
 class EvenlySpacedRateLimiter {
+  scope = "process";
   maxExecutions;
   windowSizeMs;
   idealInterval;
   nextAvailableTime = Date.now();
   lastStartTime = 0;
   durations = [];
+  acquireChain = Promise.resolve();
   constructor({ maxExecutions, windowSizeInSeconds }) {
     if (maxExecutions <= 0) {
       throw new Error("maxExecutions must be > 0");
@@ -1421,6 +1523,43 @@ class EvenlySpacedRateLimiter {
     const now = Date.now();
     return now >= this.nextAvailableTime;
   }
+  async tryAcquire() {
+    const previous = this.acquireChain;
+    let release;
+    const next = new Promise((r) => {
+      release = r;
+    });
+    this.acquireChain = next;
+    try {
+      await previous;
+      const now = Date.now();
+      if (now < this.nextAvailableTime) {
+        return null;
+      }
+      const priorNextAvailable = this.nextAvailableTime;
+      this.lastStartTime = now;
+      if (this.durations.length === 0) {
+        this.nextAvailableTime = now + this.idealInterval;
+      } else {
+        const sum = this.durations.reduce((a, b) => a + b, 0);
+        const avgDuration = sum / this.durations.length;
+        const waitMs = Math.max(0, this.idealInterval - avgDuration);
+        this.nextAvailableTime = now + waitMs;
+      }
+      return { priorNextAvailable, advancedTo: this.nextAvailableTime };
+    } finally {
+      release(undefined);
+    }
+  }
+  async release(token) {
+    if (!token || typeof token !== "object" || typeof token.advancedTo !== "number") {
+      return;
+    }
+    const t = token;
+    if (this.nextAvailableTime === t.advancedTo) {
+      this.nextAvailableTime = t.priorNextAvailable;
+    }
+  }
   async recordJobStart() {
     const now = Date.now();
     this.lastStartTime = now;
@@ -1471,6 +1610,7 @@ class RateLimiter {
   initialBackoffDelay;
   backoffMultiplier;
   maxBackoffDelay;
+  localBackoffUntilMs = 0;
   constructor(storage, queueName, {
     maxExecutions,
     windowSizeInSeconds,
@@ -1502,6 +1642,27 @@ class RateLimiter {
     this.maxBackoffDelay = maxBackoffDelay;
     this.currentBackoffDelay = initialBackoffDelay;
   }
+  get scope() {
+    return this.storage.scope;
+  }
+  async tryAcquire() {
+    const token = await this.storage.tryReserveExecution(this.queueName, this.maxExecutions, this.windowSizeInMilliseconds);
+    if (token !== null && token !== undefined) {
+      this.currentBackoffDelay = this.initialBackoffDelay;
+      this.localBackoffUntilMs = 0;
+      return token;
+    }
+    this.localBackoffUntilMs = Date.now() + this.addJitter(this.currentBackoffDelay);
+    this.increaseBackoff();
+    return null;
+  }
+  async release(token) {
+    if (token === null || token === undefined)
+      return;
+    await this.storage.releaseExecution(this.queueName, token);
+    this.currentBackoffDelay = this.initialBackoffDelay;
+    this.localBackoffUntilMs = 0;
+  }
   addJitter(base) {
     return base + Math.random() * base;
   }
@@ -1547,17 +1708,18 @@ class RateLimiter {
   async recordJobCompletion() {}
   async getNextAvailableTime() {
     const oldestExecution = await this.storage.getOldestExecutionAtOffset(this.queueName, this.maxExecutions - 1);
-    let rateLimitedTime = new Date;
+    let latestMs = Date.now();
     if (oldestExecution) {
-      rateLimitedTime = new Date(oldestExecution);
-      rateLimitedTime.setSeconds(rateLimitedTime.getSeconds() + this.windowSizeInMilliseconds / 1000);
+      latestMs = Math.max(latestMs, new Date(oldestExecution).getTime() + this.windowSizeInMilliseconds);
     }
     const nextAvailableStr = await this.storage.getNextAvailableTime(this.queueName);
-    let nextAvailableTime = new Date;
     if (nextAvailableStr) {
-      nextAvailableTime = new Date(nextAvailableStr);
+      latestMs = Math.max(latestMs, new Date(nextAvailableStr).getTime());
+    }
+    if (this.localBackoffUntilMs > latestMs) {
+      latestMs = this.localBackoffUntilMs;
     }
-    return nextAvailableTime > rateLimitedTime ? nextAvailableTime : rateLimitedTime;
+    return new Date(latestMs);
   }
   async setNextAvailableTime(date) {
     await this.storage.setNextAvailableTime(this.queueName, date.toISOString());
@@ -1565,6 +1727,7 @@ class RateLimiter {
   async clear() {
     await this.storage.clear(this.queueName);
     this.currentBackoffDelay = this.initialBackoffDelay;
+    this.localBackoffUntilMs = 0;
   }
 }
 export {
@@ -1597,4 +1760,4 @@ export {
   AbortSignalJobError
 };
 
-//# debugId=306268EEB05F1A3964756E2164756E21
+//# debugId=81121BC83C83DF5F64756E2164756E21