@workglow/job-queue 0.2.26 → 0.2.28

package/dist/bun.js

@@ -1,6 +1,15 @@
 // @bun
-// src/job/Job.ts
-import { JobStatus } from "@workglow/storage";
+// src/queue-storage/IQueueStorage.ts
+import { createServiceToken } from "@workglow/util";
+var QUEUE_STORAGE = createServiceToken("jobqueue.storage");
+var JobStatus = {
+  PENDING: "PENDING",
+  PROCESSING: "PROCESSING",
+  COMPLETED: "COMPLETED",
+  ABORTING: "ABORTING",
+  FAILED: "FAILED",
+  DISABLED: "DISABLED"
+};
 
 // src/job/JobError.ts
 import { BaseError } from "@workglow/util";
@@ -176,7 +185,6 @@ ${fullMessage}`;
 }
 
 // src/job/JobQueueClient.ts
-import { JobStatus as JobStatus2 } from "@workglow/storage";
 import { EventEmitter } from "@workglow/util";
 
 // src/job/JobStorageConverters.ts
@@ -301,7 +309,7 @@ class JobQueueClient {
       run_after: options?.runAfter?.toISOString() ?? new Date().toISOString(),
       deadline_at: options?.deadlineAt?.toISOString() ?? null,
       completed_at: null,
-      status: JobStatus2.PENDING
+      status: JobStatus.PENDING
     };
     const id = await this.storage.add(job);
     this.server?.handleJobAdded(id);
@@ -354,15 +362,15 @@ class JobQueueClient {
       this.removePromise(jobId, resolve, reject);
       throw new JobNotFoundError(`Job ${jobId} not found`);
     }
-    if (job.status === JobStatus2.COMPLETED) {
+    if (job.status === JobStatus.COMPLETED) {
       this.removePromise(jobId, resolve, reject);
       return job.output;
     }
-    if (job.status === JobStatus2.DISABLED) {
+    if (job.status === JobStatus.DISABLED) {
       this.removePromise(jobId, resolve, reject);
       throw new JobDisabledError(`Job ${jobId} was disabled`);
     }
-    if (job.status === JobStatus2.FAILED) {
+    if (job.status === JobStatus.FAILED) {
       this.removePromise(jobId, resolve, reject);
       throw this.buildErrorFromJob(job);
     }
@@ -397,7 +405,7 @@ class JobQueueClient {
       throw new JobNotFoundError("Cannot abort job run with undefined jobRunId");
     const jobs = await this.getJobsByRunId(jobRunId);
     await Promise.allSettled(jobs.map((job) => {
-      if (job.status === JobStatus2.PROCESSING || job.status === JobStatus2.PENDING) {
+      if (job.status === JobStatus.PROCESSING || job.status === JobStatus.PENDING) {
         return this.abort(job.id);
       }
     }));
@@ -504,15 +512,15 @@ class JobQueueClient {
     if (change.type === "UPDATE" && change.new) {
       const newStatus = change.new.status;
       const oldStatus = change.old?.status;
-      if (newStatus === JobStatus2.PROCESSING && oldStatus === JobStatus2.PENDING) {
+      if (newStatus === JobStatus.PROCESSING && oldStatus === JobStatus.PENDING) {
         this.handleJobStart(jobId);
-      } else if (newStatus === JobStatus2.COMPLETED) {
+      } else if (newStatus === JobStatus.COMPLETED) {
         this.handleJobComplete(jobId, change.new.output);
-      } else if (newStatus === JobStatus2.FAILED) {
+      } else if (newStatus === JobStatus.FAILED) {
         this.handleJobError(jobId, change.new.error ?? "Job failed", change.new.error_code ?? undefined);
-      } else if (newStatus === JobStatus2.DISABLED) {
+      } else if (newStatus === JobStatus.DISABLED) {
         this.handleJobDisabled(jobId);
-      } else if (newStatus === JobStatus2.PENDING && oldStatus === JobStatus2.PROCESSING) {
+      } else if (newStatus === JobStatus.PENDING && oldStatus === JobStatus.PROCESSING) {
         const runAfter = change.new.run_after ? new Date(change.new.run_after) : new Date;
         this.handleJobRetry(jobId, runAfter);
       }
@@ -555,14 +563,19 @@ class JobQueueClient {
 }
 
 // src/job/JobQueueServer.ts
-import { JobStatus as JobStatus4 } from "@workglow/storage";
 import { EventEmitter as EventEmitter3, getLogger as getLogger2 } from "@workglow/util";
 
 // src/limiter/NullLimiter.ts
-import { createServiceToken } from "@workglow/util";
-var NULL_JOB_LIMITER = createServiceToken("jobqueue.limiter.null");
+import { createServiceToken as createServiceToken2 } from "@workglow/util";
+var NULL_JOB_LIMITER = createServiceToken2("jobqueue.limiter.null");
 
 class NullLimiter {
+  scope = "process";
+  static SENTINEL = Symbol("NullLimiter.acquired");
+  async tryAcquire() {
+    return NullLimiter.SENTINEL;
+  }
+  async release(_token) {}
   async canProceed() {
     return true;
   }
@@ -576,7 +589,6 @@ class NullLimiter {
 }
 
 // src/job/JobQueueWorker.ts
-import { JobStatus as JobStatus3 } from "@workglow/storage";
 import {
   EventEmitter as EventEmitter2,
   getLogger,
@@ -585,6 +597,8 @@ import {
   SpanStatusCode,
   uuid4
 } from "@workglow/util";
+var MAX_LIMITER_WAKE_MS = 30000;
+
 class JobQueueWorker {
   queueName;
   workerId;
@@ -668,15 +682,16 @@ class JobQueueWorker {
     return this;
   }
   async processNext() {
-    const canProceed = await this.limiter.canProceed();
-    if (!canProceed) {
-      return false;
-    }
     const job = await this.next();
     if (!job) {
       return false;
     }
-    await this.processSingleJob(job);
+    const limiterToken = await this.limiter.tryAcquire();
+    if (limiterToken === null || limiterToken === undefined) {
+      await this.releaseClaimedJob(job);
+      return false;
+    }
+    await this.processSingleJob(job, limiterToken);
     return true;
   }
   isRunning() {
@@ -707,32 +722,49 @@ class JobQueueWorker {
     while (this.running) {
       try {
         await this.checkForAbortingJobs();
-        const canProceed = await this.limiter.canProceed();
-        if (canProceed) {
-          const job = await this.next();
-          if (job) {
-            if (!this.running) {
-              await this.releaseClaimedJob(job);
-              return;
-            }
-            this.processSingleJob(job);
-            continue;
-          }
-        }
-        if (canProceed) {
+        const job = await this.next();
+        if (!job) {
           const delay = await this.getIdleDelay();
           await this.waitForWakeOrTimeout(delay);
-        } else {
-          await this.waitForWakeOrTimeout(this.pollIntervalMs);
+          continue;
+        }
+        if (!this.running) {
+          await this.releaseClaimedJob(job);
+          return;
+        }
+        const limiterToken = await this.limiter.tryAcquire();
+        if (limiterToken === null || limiterToken === undefined) {
+          await this.releaseClaimedJob(job);
+          await this.waitForWakeOrTimeout(await this.getLimiterWakeDelay());
+          continue;
+        }
+        if (!this.running) {
+          try {
+            await this.limiter.release(limiterToken);
+          } catch {}
+          await this.releaseClaimedJob(job);
+          return;
         }
+        this.processSingleJob(job, limiterToken);
       } catch {
         await sleep(this.pollIntervalMs);
       }
     }
   }
+  async getLimiterWakeDelay() {
+    try {
+      const next = await this.limiter.getNextAvailableTime();
+      const delay = next.getTime() - Date.now();
+      if (delay <= 0)
+        return this.pollIntervalMs;
+      return Math.min(Math.max(delay, this.pollIntervalMs), MAX_LIMITER_WAKE_MS);
+    } catch {
+      return this.pollIntervalMs;
+    }
+  }
   async getIdleDelay() {
     try {
-      const pending = await this.storage.peek(JobStatus3.PENDING, 1);
+      const pending = await this.storage.peek(JobStatus.PENDING, 1);
       if (pending.length > 0 && pending[0].run_after) {
         const delay = new Date(pending[0].run_after).getTime() - Date.now();
         if (delay > 0) {
@@ -764,7 +796,7 @@ class JobQueueWorker {
     if (this.activeJobAbortControllers.size === 0) {
       return;
     }
-    const abortingJobs = await this.storage.peek(JobStatus3.ABORTING);
+    const abortingJobs = await this.storage.peek(JobStatus.ABORTING);
     for (const jobData of abortingJobs) {
       const controller = this.activeJobAbortControllers.get(jobData.id);
       if (controller && !controller.signal.aborted) {
@@ -772,7 +804,7 @@ class JobQueueWorker {
       }
     }
   }
-  async processSingleJob(job) {
+  async processSingleJob(job, limiterToken) {
     if (!job || !job.id) {
       throw new JobNotFoundError("Invalid job provided for processing");
     }
@@ -789,9 +821,17 @@ class JobQueueWorker {
         "workglow.job.max_retries": job.maxRetries
       }
     }) : undefined;
+    let slotReleased = false;
     try {
-      await this.validateJobState(job);
-      await this.limiter.recordJobStart();
+      try {
+        await this.validateJobState(job);
+      } catch (validationErr) {
+        try {
+          await this.limiter.release(limiterToken);
+          slotReleased = true;
+        } catch {}
+        throw validationErr;
+      }
       const abortController = this.createAbortController(job.id);
       this.events.emit("job_start", job.id);
       const output = await this.executeJob(job, abortController.signal);
@@ -829,7 +869,9 @@ class JobQueueWorker {
     } finally {
       span?.end();
       try {
-        await this.limiter.recordJobCompletion();
+        if (!slotReleased) {
+          await this.limiter.recordJobCompletion();
+        }
       } finally {
         this.inFlight.delete(job.id);
         resolveInFlight();
@@ -850,7 +892,7 @@ class JobQueueWorker {
   }
   async completeJob(job, output) {
     try {
-      job.status = JobStatus3.COMPLETED;
+      job.status = JobStatus.COMPLETED;
       job.progress = 100;
       job.progressMessage = "";
       job.progressDetails = null;
@@ -868,7 +910,7 @@ class JobQueueWorker {
   }
   async failJob(job, error) {
     try {
-      job.status = JobStatus3.FAILED;
+      job.status = JobStatus.FAILED;
       job.progress = 100;
       job.completedAt = new Date;
       job.progressMessage = "";
@@ -885,7 +927,7 @@ class JobQueueWorker {
   }
   async disableJob(job) {
     try {
-      job.status = JobStatus3.DISABLED;
+      job.status = JobStatus.DISABLED;
       job.progress = 100;
       job.completedAt = new Date;
       job.progressMessage = "";
@@ -907,7 +949,7 @@ class JobQueueWorker {
   }
   async rescheduleJob(job, retryDate) {
     try {
-      job.status = JobStatus3.PENDING;
+      job.status = JobStatus.PENDING;
       const nextAvailableTime = await this.limiter.getNextAvailableTime();
       job.runAfter = retryDate instanceof Date ? retryDate : nextAvailableTime;
       job.progress = 0;
@@ -943,7 +985,7 @@ class JobQueueWorker {
       getLogger().error("handleAbort: job not found", { jobId });
       return;
     }
-    if (job.status === JobStatus3.COMPLETED || job.status === JobStatus3.FAILED || job.status === JobStatus3.DISABLED) {
+    if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED || job.status === JobStatus.DISABLED) {
       return;
     }
     await this.failJob(job, new AbortSignalJobError("Job Aborted"));
@@ -955,19 +997,19 @@ class JobQueueWorker {
     return this.storageToClass(job);
   }
   async validateJobState(job) {
-    if (job.status === JobStatus3.COMPLETED) {
+    if (job.status === JobStatus.COMPLETED) {
       throw new PermanentJobError(`Job ${job.id} is already completed`);
     }
-    if (job.status === JobStatus3.FAILED) {
+    if (job.status === JobStatus.FAILED) {
       throw new PermanentJobError(`Job ${job.id} has failed`);
     }
-    if (job.status === JobStatus3.ABORTING || this.activeJobAbortControllers.get(job.id)?.signal.aborted) {
+    if (job.status === JobStatus.ABORTING || this.activeJobAbortControllers.get(job.id)?.signal.aborted) {
       throw new AbortSignalJobError(`Job ${job.id} is being aborted`);
     }
     if (job.deadlineAt && job.deadlineAt < new Date) {
       throw new PermanentJobError(`Job ${job.id} has exceeded its deadline`);
     }
-    if (job.status === JobStatus3.DISABLED) {
+    if (job.status === JobStatus.DISABLED) {
       throw new JobDisabledError(`Job ${job.id} has been disabled`);
     }
   }
@@ -1039,10 +1081,17 @@ class JobQueueServer {
     }
     this.running = true;
     this.events.emit("server_start", this.queueName);
+    if (this.limiter.scope === "process" && this.storage.scope === "cluster" && !(this.limiter instanceof NullLimiter)) {
+      getLogger2().warn("Process-scoped limiter on cluster-scoped queue storage \u2014 limit is enforced per-process, not cluster-wide. Use a cluster-scoped rate limiter storage (Postgres/Supabase) for global enforcement.", {
+        queueName: this.queueName,
+        limiterScope: this.limiter.scope,
+        storage: this.storage.constructor.name
+      });
+    }
     await this.fixupJobs();
     try {
       this.storageUnsubscribe = this.storage.subscribeToChanges((change) => {
-        if (change.type === "INSERT" || change.type === "UPDATE" && change.new?.status === JobStatus4.PENDING) {
+        if (change.type === "INSERT" || change.type === "RESYNC" || change.type === "UPDATE" && change.new?.status === JobStatus.PENDING) {
           this.notifyWorkers();
         }
       });
@@ -1237,13 +1286,13 @@ class JobQueueServer {
   async cleanupJobs() {
     try {
       if (this.deleteAfterCompletionMs !== undefined && this.deleteAfterCompletionMs > 0) {
-        await this.storage.deleteJobsByStatusAndAge(JobStatus4.COMPLETED, this.deleteAfterCompletionMs);
+        await this.storage.deleteJobsByStatusAndAge(JobStatus.COMPLETED, this.deleteAfterCompletionMs);
       }
       if (this.deleteAfterFailureMs !== undefined && this.deleteAfterFailureMs > 0) {
-        await this.storage.deleteJobsByStatusAndAge(JobStatus4.FAILED, this.deleteAfterFailureMs);
+        await this.storage.deleteJobsByStatusAndAge(JobStatus.FAILED, this.deleteAfterFailureMs);
       }
       if (this.deleteAfterDisabledMs !== undefined && this.deleteAfterDisabledMs > 0) {
-        await this.storage.deleteJobsByStatusAndAge(JobStatus4.DISABLED, this.deleteAfterDisabledMs);
+        await this.storage.deleteJobsByStatusAndAge(JobStatus.DISABLED, this.deleteAfterDisabledMs);
       }
     } catch (error) {
       console.error("Error in cleanup:", error);
@@ -1251,8 +1300,8 @@ class JobQueueServer {
   }
   async fixupJobs() {
     try {
-      const stuckProcessingJobs = await this.storage.peek(JobStatus4.PROCESSING);
-      const stuckAbortingJobs = await this.storage.peek(JobStatus4.ABORTING);
+      const stuckProcessingJobs = await this.storage.peek(JobStatus.PROCESSING);
+      const stuckAbortingJobs = await this.storage.peek(JobStatus.ABORTING);
       const stuckJobs = [...stuckProcessingJobs, ...stuckAbortingJobs];
       const currentWorkerIds = new Set(this.getWorkerIds());
       for (const jobData of stuckJobs) {
@@ -1261,12 +1310,12 @@ class JobQueueServer {
         }
         const job = this.storageToClass(jobData);
         if (job.runAttempts >= job.maxRetries) {
-          job.status = JobStatus4.FAILED;
+          job.status = JobStatus.FAILED;
           job.error = "Max retries reached";
           job.errorCode = "MAX_RETRIES_REACHED";
           job.workerId = null;
         } else {
-          job.status = JobStatus4.PENDING;
+          job.status = JobStatus.PENDING;
           job.runAfter = job.lastRanAt || new Date;
           job.progress = 0;
           job.progressMessage = "";
@@ -1297,6 +1346,9 @@ class CompositeLimiter {
   constructor(limiters = []) {
     this.limiters = limiters;
   }
+  get scope() {
+    return this.limiters.every((l) => l.scope === "cluster") && this.limiters.length > 0 ? "cluster" : "process";
+  }
   addLimiter(limiter) {
     this.limiters.push(limiter);
   }
@@ -1308,6 +1360,27 @@ class CompositeLimiter {
     }
     return true;
   }
+  async tryAcquire() {
+    const tokens = [];
+    for (const limiter of this.limiters) {
+      const t = await limiter.tryAcquire();
+      if (t === null || t === undefined) {
+        for (let i = tokens.length - 1;i >= 0; i--) {
+          try {
+            await this.limiters[i].release(tokens[i]);
+          } catch {}
+        }
+        return null;
+      }
+      tokens.push(t);
+    }
+    return tokens;
+  }
+  async release(token) {
+    if (!Array.isArray(token))
+      return;
+    await Promise.all(this.limiters.map((l, i) => l.release(token[i]).catch(() => {})));
+  }
   async recordJobStart() {
     await Promise.all(this.limiters.map((limiter) => limiter.recordJobStart()));
   }
@@ -1335,10 +1408,11 @@ class CompositeLimiter {
 }
 
 // src/limiter/ConcurrencyLimiter.ts
-import { createServiceToken as createServiceToken2 } from "@workglow/util";
-var CONCURRENT_JOB_LIMITER = createServiceToken2("jobqueue.limiter.concurrent");
+import { createServiceToken as createServiceToken3 } from "@workglow/util";
+var CONCURRENT_JOB_LIMITER = createServiceToken3("jobqueue.limiter.concurrent");
 
 class ConcurrencyLimiter {
+  scope = "process";
   currentRunningJobs = 0;
   maxConcurrentJobs;
   nextAllowedStartTime = new Date;
@@ -1348,6 +1422,19 @@ class ConcurrencyLimiter {
   async canProceed() {
     return this.currentRunningJobs < this.maxConcurrentJobs && Date.now() >= this.nextAllowedStartTime.getTime();
   }
+  static SENTINEL = Symbol("ConcurrencyLimiter.acquired");
+  async tryAcquire() {
+    if (this.currentRunningJobs >= this.maxConcurrentJobs || Date.now() < this.nextAllowedStartTime.getTime()) {
+      return null;
+    }
+    this.currentRunningJobs++;
+    return ConcurrencyLimiter.SENTINEL;
+  }
+  async release(token) {
+    if (token !== ConcurrencyLimiter.SENTINEL)
+      return;
+    this.currentRunningJobs = Math.max(0, this.currentRunningJobs - 1);
+  }
   async recordJobStart() {
     this.currentRunningJobs++;
   }
@@ -1371,13 +1458,32 @@ class ConcurrencyLimiter {
 // src/limiter/DelayLimiter.ts
 class DelayLimiter {
   delayInMilliseconds;
+  scope = "process";
   nextAvailableTime = new Date;
+  lastAcquireBaseline = 0;
   constructor(delayInMilliseconds = 50) {
     this.delayInMilliseconds = delayInMilliseconds;
   }
   async canProceed() {
     return Date.now() >= this.nextAvailableTime.getTime();
   }
+  async tryAcquire() {
+    const now = Date.now();
+    if (now < this.nextAvailableTime.getTime()) {
+      return null;
+    }
+    const previous = this.nextAvailableTime.getTime();
+    this.lastAcquireBaseline = previous;
+    this.nextAvailableTime = new Date(now + this.delayInMilliseconds);
+    return previous;
+  }
+  async release(token) {
+    if (typeof token !== "number")
+      return;
+    if (this.nextAvailableTime.getTime() === this.lastAcquireBaseline + this.delayInMilliseconds) {
+      this.nextAvailableTime = new Date(token);
+    }
+  }
   async recordJobStart() {
     this.nextAvailableTime = new Date(Date.now() + this.delayInMilliseconds);
   }
@@ -1396,16 +1502,18 @@ class DelayLimiter {
 }
 
 // src/limiter/EvenlySpacedRateLimiter.ts
-import { createServiceToken as createServiceToken3 } from "@workglow/util";
-var EVENLY_SPACED_JOB_RATE_LIMITER = createServiceToken3("jobqueue.limiter.rate.evenlyspaced");
+import { createServiceToken as createServiceToken4 } from "@workglow/util";
+var EVENLY_SPACED_JOB_RATE_LIMITER = createServiceToken4("jobqueue.limiter.rate.evenlyspaced");
 
 class EvenlySpacedRateLimiter {
+  scope = "process";
   maxExecutions;
   windowSizeMs;
   idealInterval;
   nextAvailableTime = Date.now();
   lastStartTime = 0;
   durations = [];
+  acquireChain = Promise.resolve();
   constructor({ maxExecutions, windowSizeInSeconds }) {
     if (maxExecutions <= 0) {
       throw new Error("maxExecutions must be > 0");
@@ -1421,6 +1529,43 @@ class EvenlySpacedRateLimiter {
     const now = Date.now();
     return now >= this.nextAvailableTime;
   }
+  async tryAcquire() {
+    const previous = this.acquireChain;
+    let release;
+    const next = new Promise((r) => {
+      release = r;
+    });
+    this.acquireChain = next;
+    try {
+      await previous;
+      const now = Date.now();
+      if (now < this.nextAvailableTime) {
+        return null;
+      }
+      const priorNextAvailable = this.nextAvailableTime;
+      this.lastStartTime = now;
+      if (this.durations.length === 0) {
+        this.nextAvailableTime = now + this.idealInterval;
+      } else {
+        const sum = this.durations.reduce((a, b) => a + b, 0);
+        const avgDuration = sum / this.durations.length;
+        const waitMs = Math.max(0, this.idealInterval - avgDuration);
+        this.nextAvailableTime = now + waitMs;
+      }
+      return { priorNextAvailable, advancedTo: this.nextAvailableTime };
+    } finally {
+      release(undefined);
+    }
+  }
+  async release(token) {
+    if (!token || typeof token !== "object" || typeof token.advancedTo !== "number") {
+      return;
+    }
+    const t = token;
+    if (this.nextAvailableTime === t.advancedTo) {
+      this.nextAvailableTime = t.priorNextAvailable;
+    }
+  }
   async recordJobStart() {
     const now = Date.now();
     this.lastStartTime = now;
@@ -1458,8 +1603,8 @@ class EvenlySpacedRateLimiter {
 }
 
 // src/limiter/ILimiter.ts
-import { createServiceToken as createServiceToken4 } from "@workglow/util";
-var JOB_LIMITER = createServiceToken4("jobqueue.limiter");
+import { createServiceToken as createServiceToken5 } from "@workglow/util";
+var JOB_LIMITER = createServiceToken5("jobqueue.limiter");
 
 // src/limiter/RateLimiter.ts
 class RateLimiter {
@@ -1471,6 +1616,7 @@ class RateLimiter {
   initialBackoffDelay;
   backoffMultiplier;
   maxBackoffDelay;
+  localBackoffUntilMs = 0;
   constructor(storage, queueName, {
     maxExecutions,
     windowSizeInSeconds,
@@ -1502,6 +1648,27 @@ class RateLimiter {
     this.maxBackoffDelay = maxBackoffDelay;
     this.currentBackoffDelay = initialBackoffDelay;
   }
+  get scope() {
+    return this.storage.scope;
+  }
+  async tryAcquire() {
+    const token = await this.storage.tryReserveExecution(this.queueName, this.maxExecutions, this.windowSizeInMilliseconds);
+    if (token !== null && token !== undefined) {
+      this.currentBackoffDelay = this.initialBackoffDelay;
+      this.localBackoffUntilMs = 0;
+      return token;
+    }
+    this.localBackoffUntilMs = Date.now() + this.addJitter(this.currentBackoffDelay);
+    this.increaseBackoff();
+    return null;
+  }
+  async release(token) {
+    if (token === null || token === undefined)
+      return;
+    await this.storage.releaseExecution(this.queueName, token);
+    this.currentBackoffDelay = this.initialBackoffDelay;
+    this.localBackoffUntilMs = 0;
+  }
   addJitter(base) {
     return base + Math.random() * base;
   }
@@ -1547,17 +1714,18 @@ class RateLimiter {
   async recordJobCompletion() {}
   async getNextAvailableTime() {
     const oldestExecution = await this.storage.getOldestExecutionAtOffset(this.queueName, this.maxExecutions - 1);
-    let rateLimitedTime = new Date;
+    let latestMs = Date.now();
     if (oldestExecution) {
-      rateLimitedTime = new Date(oldestExecution);
-      rateLimitedTime.setSeconds(rateLimitedTime.getSeconds() + this.windowSizeInMilliseconds / 1000);
+      latestMs = Math.max(latestMs, new Date(oldestExecution).getTime() + this.windowSizeInMilliseconds);
     }
     const nextAvailableStr = await this.storage.getNextAvailableTime(this.queueName);
-    let nextAvailableTime = new Date;
     if (nextAvailableStr) {
-      nextAvailableTime = new Date(nextAvailableStr);
+      latestMs = Math.max(latestMs, new Date(nextAvailableStr).getTime());
+    }
+    if (this.localBackoffUntilMs > latestMs) {
+      latestMs = this.localBackoffUntilMs;
     }
-    return nextAvailableTime > rateLimitedTime ? nextAvailableTime : rateLimitedTime;
+    return new Date(latestMs);
   }
   async setNextAvailableTime(date) {
     await this.storage.setNextAvailableTime(this.queueName, date.toISOString());
@@ -1565,6 +1733,408 @@ class RateLimiter {
   async clear() {
     await this.storage.clear(this.queueName);
     this.currentBackoffDelay = this.initialBackoffDelay;
+    this.localBackoffUntilMs = 0;
+  }
+}
+
+// src/queue-storage/InMemoryQueueStorage.ts
+import {
+  createServiceToken as createServiceToken6,
+  EventEmitter as EventEmitter4,
+  getLogger as getLogger3,
+  makeFingerprint,
+  sleep as sleep2,
+  uuid4 as uuid42
+} from "@workglow/util";
+var IN_MEMORY_QUEUE_STORAGE = createServiceToken6("jobqueue.storage.inMemory");
+
+class InMemoryQueueStorage {
+  queueName;
+  scope = "process";
+  prefixValues;
+  events = new EventEmitter4;
+  constructor(queueName, options) {
+    this.queueName = queueName;
+    this.jobQueue = [];
+    this.prefixValues = options?.prefixValues ?? {};
+  }
+  jobQueue;
+  matchesPrefixes(job) {
+    for (const [key, value] of Object.entries(this.prefixValues)) {
+      if (job[key] !== value) {
+        return false;
+      }
+    }
+    return true;
+  }
+  pendingQueue() {
+    const now = new Date().toISOString();
+    return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === JobStatus.PENDING).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
+  }
+  async add(job) {
+    await sleep2(0);
+    const now = new Date().toISOString();
+    const jobWithPrefixes = job;
+    jobWithPrefixes.id = jobWithPrefixes.id ?? uuid42();
+    jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid42();
+    jobWithPrefixes.queue = this.queueName;
+    jobWithPrefixes.fingerprint = await makeFingerprint(jobWithPrefixes.input);
+    jobWithPrefixes.status = JobStatus.PENDING;
+    jobWithPrefixes.progress = 0;
+    jobWithPrefixes.progress_message = "";
+    jobWithPrefixes.progress_details = null;
+    jobWithPrefixes.created_at = now;
+    jobWithPrefixes.run_after = now;
+    for (const [key, value] of Object.entries(this.prefixValues)) {
+      jobWithPrefixes[key] = value;
+    }
+    this.jobQueue.push(jobWithPrefixes);
+    this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
+    return jobWithPrefixes.id;
+  }
+  async get(id) {
+    await sleep2(0);
+    const job = this.jobQueue.find((j) => j.id === id);
+    if (job && this.matchesPrefixes(job)) {
+      return job;
+    }
+    return;
+  }
+  async peek(status = JobStatus.PENDING, num = 100) {
+    await sleep2(0);
+    num = Number(num) || 100;
+    return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
+  }
+  async next(workerId) {
+    await sleep2(0);
+    const top = this.pendingQueue();
+    const job = top[0];
+    if (job) {
+      const oldJob = { ...job };
+      job.status = JobStatus.PROCESSING;
+      job.last_ran_at = new Date().toISOString();
+      job.worker_id = workerId;
+      this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
+      return job;
+    }
+  }
+  async size(status = JobStatus.PENDING) {
+    await sleep2(0);
+    return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
+  }
+  async saveProgress(id, progress, message, details) {
+    await sleep2(0);
+    const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
+    if (!job) {
+      const jobWithAnyPrefix = this.jobQueue.find((j) => j.id === id);
+      getLogger3().warn("Job not found for progress update", {
+        id,
+        reason: jobWithAnyPrefix ? "prefix_mismatch" : "missing",
+        existingStatus: jobWithAnyPrefix?.status,
+        queueName: this.queueName,
+        prefixValues: this.prefixValues
+      });
+      return;
+    }
+    if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED) {
+      getLogger3().warn("Job already completed or failed for progress update", {
+        id,
+        status: job.status,
+        completedAt: job.completed_at,
+        error: job.error
+      });
+      return;
+    }
+    const oldJob = { ...job };
+    job.progress = progress;
+    job.progress_message = message;
+    job.progress_details = details;
+    this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
+  }
+  async complete(job) {
+    await sleep2(0);
+    const jobWithPrefixes = job;
+    const index = this.jobQueue.findIndex((j) => j.id === job.id && this.matchesPrefixes(j));
+    if (index !== -1) {
+      const existing = this.jobQueue[index];
+      const currentAttempts = existing?.run_attempts ?? 0;
+      jobWithPrefixes.run_attempts = currentAttempts + 1;
+      for (const [key, value] of Object.entries(this.prefixValues)) {
+        jobWithPrefixes[key] = value;
+      }
+      this.jobQueue[index] = jobWithPrefixes;
+      this.events.emit("change", { type: "UPDATE", old: existing, new: jobWithPrefixes });
+    }
+  }
+  async release(id) {
+    await sleep2(0);
+    const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
+    if (job) {
+      const oldJob = { ...job };
+      job.status = JobStatus.PENDING;
+      job.worker_id = null;
+      job.progress = 0;
+      job.progress_message = "";
+      job.progress_details = null;
+      this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
+    }
+  }
+  async abort(id) {
+    await sleep2(0);
+    const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
+    if (job) {
+      const oldJob = { ...job };
+      job.status = JobStatus.ABORTING;
+      this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
+    }
+  }
+  async getByRunId(runId) {
+    await sleep2(0);
+    return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
+  }
+  async deleteAll() {
+    await sleep2(0);
+    const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
+    this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
+    for (const job of deletedJobs) {
+      this.events.emit("change", { type: "DELETE", old: job });
+    }
+  }
+  async outputForInput(input) {
+    await sleep2(0);
+    const fingerprint = await makeFingerprint(input);
+    return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === JobStatus.COMPLETED)?.output ?? null;
+  }
+  async delete(id) {
+    await sleep2(0);
+    const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
+    this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
+    if (deletedJob) {
+      this.events.emit("change", { type: "DELETE", old: deletedJob });
+    }
+  }
+  async deleteJobsByStatusAndAge(status, olderThanMs) {
+    await sleep2(0);
+    const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
+    const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
+    this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
+    for (const job of deletedJobs) {
+      this.events.emit("change", { type: "DELETE", old: job });
+    }
+  }
+  async setupDatabase() {}
+  matchesPrefixFilter(job, prefixFilter) {
+    if (prefixFilter && Object.keys(prefixFilter).length === 0) {
+      return true;
+    }
+    const filterValues = prefixFilter ?? this.prefixValues;
+    if (Object.keys(filterValues).length === 0) {
+      return true;
+    }
+    const jobWithPrefixes = job;
+    for (const [key, value] of Object.entries(filterValues)) {
+      if (jobWithPrefixes[key] !== value) {
+        return false;
+      }
+    }
+    return true;
+  }
+  subscribeToChanges(callback, options) {
+    const prefixFilter = options?.prefixFilter;
+    const filteredCallback = (change) => {
+      const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
+      const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
+      if (!newMatches && !oldMatches) {
+        return;
+      }
+      callback(change);
+    };
+    return this.events.subscribe("change", filteredCallback);
+  }
+}
+
+// src/queue-storage/TelemetryQueueStorage.ts
+import { traced } from "@workglow/util";
+
+class TelemetryQueueStorage {
+  storageName;
+  inner;
+  constructor(storageName, inner) {
+    this.storageName = storageName;
+    this.inner = inner;
+  }
+  get scope() {
+    return this.inner.scope;
+  }
+  add(job) {
+    return traced("workglow.storage.queue.add", this.storageName, () => this.inner.add(job));
+  }
+  get(id) {
+    return traced("workglow.storage.queue.get", this.storageName, () => this.inner.get(id));
+  }
+  next(workerId) {
+    return traced("workglow.storage.queue.next", this.storageName, () => this.inner.next(workerId));
+  }
+  peek(status, num) {
+    return traced("workglow.storage.queue.peek", this.storageName, () => this.inner.peek(status, num));
+  }
+  size(status) {
+    return traced("workglow.storage.queue.size", this.storageName, () => this.inner.size(status));
+  }
+  complete(job) {
+    return traced("workglow.storage.queue.complete", this.storageName, () => this.inner.complete(job));
+  }
+  release(id) {
+    return traced("workglow.storage.queue.release", this.storageName, () => this.inner.release(id));
+  }
+  deleteAll() {
+    return traced("workglow.storage.queue.deleteAll", this.storageName, () => this.inner.deleteAll());
+  }
+  outputForInput(input) {
+    return traced("workglow.storage.queue.outputForInput", this.storageName, () => this.inner.outputForInput(input));
+  }
+  abort(id) {
+    return traced("workglow.storage.queue.abort", this.storageName, () => this.inner.abort(id));
+  }
+  getByRunId(runId) {
+    return traced("workglow.storage.queue.getByRunId", this.storageName, () => this.inner.getByRunId(runId));
+  }
+  saveProgress(id, progress, message, details) {
+    return traced("workglow.storage.queue.saveProgress", this.storageName, () => this.inner.saveProgress(id, progress, message, details));
+  }
+  delete(id) {
+    return traced("workglow.storage.queue.delete", this.storageName, () => this.inner.delete(id));
+  }
+  deleteJobsByStatusAndAge(status, olderThanMs) {
+    return traced("workglow.storage.queue.deleteJobsByStatusAndAge", this.storageName, () => this.inner.deleteJobsByStatusAndAge(status, olderThanMs));
+  }
+  setupDatabase() {
+    return this.inner.setupDatabase();
+  }
+  subscribeToChanges(callback, options) {
+    return this.inner.subscribeToChanges(callback, options);
+  }
+}
+
+// src/rate-limiter-storage/IRateLimiterStorage.ts
+import { createServiceToken as createServiceToken7 } from "@workglow/util";
+var RATE_LIMITER_STORAGE = createServiceToken7("ratelimiter.storage");
+
+// src/rate-limiter-storage/InMemoryRateLimiterStorage.ts
+import { createServiceToken as createServiceToken8, sleep as sleep3, uuid4 as uuid43 } from "@workglow/util";
+var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage.inMemory");
+
+class InMemoryRateLimiterStorage {
+  scope = "process";
+  prefixValues;
+  executions = new Map;
+  nextAvailableTimes = new Map;
+  reserveChains = new Map;
+  constructor(options) {
+    this.prefixValues = options?.prefixValues ?? {};
+  }
+  makeKey(queueName) {
+    const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
+    return prefixPart ? `${prefixPart}|${queueName}` : queueName;
+  }
+  async setupDatabase() {}
+  async withKeyLock(key, fn) {
+    const previous = this.reserveChains.get(key) ?? Promise.resolve();
+    let release;
+    const next = new Promise((resolve) => {
+      release = resolve;
+    });
+    this.reserveChains.set(key, next);
+    try {
+      await previous;
+      return await fn();
+    } finally {
+      release(undefined);
+      if (this.reserveChains.get(key) === next) {
+        this.reserveChains.delete(key);
+      }
+    }
+  }
+  async tryReserveExecution(queueName, maxExecutions, windowMs) {
+    const key = this.makeKey(queueName);
+    return this.withKeyLock(key, () => {
+      const now = Date.now();
+      const windowStart = new Date(now - windowMs);
+      const executions = this.executions.get(key) ?? [];
+      const live = executions.filter((e) => e.executedAt > windowStart);
+      if (live.length >= maxExecutions) {
+        if (live.length !== executions.length) {
+          this.executions.set(key, live);
+        }
+        return null;
+      }
+      const next = this.nextAvailableTimes.get(key);
+      if (next && next.getTime() > now) {
+        return null;
+      }
+      const id = uuid43();
+      live.push({ id, queueName, executedAt: new Date(now) });
+      this.executions.set(key, live);
+      return id;
+    });
+  }
+  async releaseExecution(queueName, token) {
+    if (token === null || token === undefined)
+      return;
+    const key = this.makeKey(queueName);
+    await this.withKeyLock(key, () => {
+      const executions = this.executions.get(key);
+      if (!executions || executions.length === 0)
+        return;
+      const idx = executions.findIndex((e) => e.id === token);
+      if (idx === -1)
+        return;
+      executions.splice(idx, 1);
+      this.executions.set(key, executions);
+    });
+  }
+  async recordExecution(queueName) {
+    await sleep3(0);
+    const key = this.makeKey(queueName);
+    const executions = this.executions.get(key) ?? [];
+    executions.push({
+      id: uuid43(),
+      queueName,
+      executedAt: new Date
+    });
+    this.executions.set(key, executions);
+  }
+  async getExecutionCount(queueName, windowStartTime) {
+    await sleep3(0);
+    const key = this.makeKey(queueName);
+    const executions = this.executions.get(key) ?? [];
+    const windowStart = new Date(windowStartTime);
+    return executions.filter((e) => e.executedAt > windowStart).length;
+  }
+  async getOldestExecutionAtOffset(queueName, offset) {
+    await sleep3(0);
+    const key = this.makeKey(queueName);
+    const executions = this.executions.get(key) ?? [];
+    const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
+    const execution = sorted[offset];
+    return execution?.executedAt.toISOString();
+  }
+  async getNextAvailableTime(queueName) {
+    await sleep3(0);
+    const key = this.makeKey(queueName);
+    const time = this.nextAvailableTimes.get(key);
+    return time?.toISOString();
+  }
+  async setNextAvailableTime(queueName, nextAvailableAt) {
+    await sleep3(0);
+    const key = this.makeKey(queueName);
+    this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
+  }
+  async clear(queueName) {
+    await sleep3(0);
+    const key = this.makeKey(queueName);
+    this.executions.delete(key);
+    this.nextAvailableTimes.delete(key);
   }
 }
 export {
@@ -1573,8 +2143,11 @@ export {
   formatErrorChainForDiagnostics,
   classToStorage,
   applyPersistedDiagnosticsToStack,
+  TelemetryQueueStorage,
   RetryableJobError,
   RateLimiter,
+  RATE_LIMITER_STORAGE,
+  QUEUE_STORAGE,
   PermanentJobError,
   NullLimiter,
   NULL_JOB_LIMITER,
@@ -1588,6 +2161,10 @@ export {
   Job,
   JOB_LIMITER,
   JOB_ERROR_DIAGNOSTICS_MARKER,
+  InMemoryRateLimiterStorage,
+  InMemoryQueueStorage,
+  IN_MEMORY_RATE_LIMITER_STORAGE,
+  IN_MEMORY_QUEUE_STORAGE,
   EvenlySpacedRateLimiter,
   EVENLY_SPACED_JOB_RATE_LIMITER,
   DelayLimiter,
@@ -1597,4 +2174,4 @@ export {
   AbortSignalJobError
 };
 
-//# debugId=306268EEB05F1A3964756E2164756E21
+//# debugId=4ACCF006917528EA64756E2164756E21