@workglow/job-queue 0.2.27 → 0.2.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/browser.js +457 -43
  2. package/dist/browser.js.map +15 -10
  3. package/dist/bun.js +457 -43
  4. package/dist/bun.js.map +15 -10
  5. package/dist/common.d.ts +5 -0
  6. package/dist/common.d.ts.map +1 -1
  7. package/dist/job/Job.d.ts +1 -1
  8. package/dist/job/Job.d.ts.map +1 -1
  9. package/dist/job/JobQueueClient.d.ts +2 -1
  10. package/dist/job/JobQueueClient.d.ts.map +1 -1
  11. package/dist/job/JobQueueServer.d.ts +1 -1
  12. package/dist/job/JobQueueServer.d.ts.map +1 -1
  13. package/dist/job/JobQueueWorker.d.ts +1 -1
  14. package/dist/job/JobQueueWorker.d.ts.map +1 -1
  15. package/dist/job/JobStorageConverters.d.ts +1 -1
  16. package/dist/job/JobStorageConverters.d.ts.map +1 -1
  17. package/dist/limiter/CompositeLimiter.d.ts.map +1 -1
  18. package/dist/limiter/RateLimiter.d.ts +1 -1
  19. package/dist/limiter/RateLimiter.d.ts.map +1 -1
  20. package/dist/node.js +457 -43
  21. package/dist/node.js.map +15 -10
  22. package/dist/queue-storage/IQueueStorage.d.ts +229 -0
  23. package/dist/queue-storage/IQueueStorage.d.ts.map +1 -0
  24. package/dist/queue-storage/InMemoryQueueStorage.d.ts +149 -0
  25. package/dist/queue-storage/InMemoryQueueStorage.d.ts.map +1 -0
  26. package/dist/queue-storage/TelemetryQueueStorage.d.ts +33 -0
  27. package/dist/queue-storage/TelemetryQueueStorage.d.ts.map +1 -0
  28. package/dist/rate-limiter-storage/IRateLimiterStorage.d.ts +127 -0
  29. package/dist/rate-limiter-storage/IRateLimiterStorage.d.ts.map +1 -0
  30. package/dist/rate-limiter-storage/InMemoryRateLimiterStorage.d.ts +43 -0
  31. package/dist/rate-limiter-storage/InMemoryRateLimiterStorage.d.ts.map +1 -0
  32. package/package.json +3 -8
package/dist/bun.js CHANGED
@@ -1,6 +1,15 @@
1
1
  // @bun
2
- // src/job/Job.ts
3
- import { JobStatus } from "@workglow/storage";
2
+ // src/queue-storage/IQueueStorage.ts
3
+ import { createServiceToken } from "@workglow/util";
4
+ var QUEUE_STORAGE = createServiceToken("jobqueue.storage");
5
+ var JobStatus = {
6
+ PENDING: "PENDING",
7
+ PROCESSING: "PROCESSING",
8
+ COMPLETED: "COMPLETED",
9
+ ABORTING: "ABORTING",
10
+ FAILED: "FAILED",
11
+ DISABLED: "DISABLED"
12
+ };
4
13
 
5
14
  // src/job/JobError.ts
6
15
  import { BaseError } from "@workglow/util";
@@ -176,7 +185,6 @@ ${fullMessage}`;
176
185
  }
177
186
 
178
187
  // src/job/JobQueueClient.ts
179
- import { JobStatus as JobStatus2 } from "@workglow/storage";
180
188
  import { EventEmitter } from "@workglow/util";
181
189
 
182
190
  // src/job/JobStorageConverters.ts
@@ -301,7 +309,7 @@ class JobQueueClient {
301
309
  run_after: options?.runAfter?.toISOString() ?? new Date().toISOString(),
302
310
  deadline_at: options?.deadlineAt?.toISOString() ?? null,
303
311
  completed_at: null,
304
- status: JobStatus2.PENDING
312
+ status: JobStatus.PENDING
305
313
  };
306
314
  const id = await this.storage.add(job);
307
315
  this.server?.handleJobAdded(id);
@@ -354,15 +362,15 @@ class JobQueueClient {
354
362
  this.removePromise(jobId, resolve, reject);
355
363
  throw new JobNotFoundError(`Job ${jobId} not found`);
356
364
  }
357
- if (job.status === JobStatus2.COMPLETED) {
365
+ if (job.status === JobStatus.COMPLETED) {
358
366
  this.removePromise(jobId, resolve, reject);
359
367
  return job.output;
360
368
  }
361
- if (job.status === JobStatus2.DISABLED) {
369
+ if (job.status === JobStatus.DISABLED) {
362
370
  this.removePromise(jobId, resolve, reject);
363
371
  throw new JobDisabledError(`Job ${jobId} was disabled`);
364
372
  }
365
- if (job.status === JobStatus2.FAILED) {
373
+ if (job.status === JobStatus.FAILED) {
366
374
  this.removePromise(jobId, resolve, reject);
367
375
  throw this.buildErrorFromJob(job);
368
376
  }
@@ -397,7 +405,7 @@ class JobQueueClient {
397
405
  throw new JobNotFoundError("Cannot abort job run with undefined jobRunId");
398
406
  const jobs = await this.getJobsByRunId(jobRunId);
399
407
  await Promise.allSettled(jobs.map((job) => {
400
- if (job.status === JobStatus2.PROCESSING || job.status === JobStatus2.PENDING) {
408
+ if (job.status === JobStatus.PROCESSING || job.status === JobStatus.PENDING) {
401
409
  return this.abort(job.id);
402
410
  }
403
411
  }));
@@ -504,15 +512,15 @@ class JobQueueClient {
504
512
  if (change.type === "UPDATE" && change.new) {
505
513
  const newStatus = change.new.status;
506
514
  const oldStatus = change.old?.status;
507
- if (newStatus === JobStatus2.PROCESSING && oldStatus === JobStatus2.PENDING) {
515
+ if (newStatus === JobStatus.PROCESSING && oldStatus === JobStatus.PENDING) {
508
516
  this.handleJobStart(jobId);
509
- } else if (newStatus === JobStatus2.COMPLETED) {
517
+ } else if (newStatus === JobStatus.COMPLETED) {
510
518
  this.handleJobComplete(jobId, change.new.output);
511
- } else if (newStatus === JobStatus2.FAILED) {
519
+ } else if (newStatus === JobStatus.FAILED) {
512
520
  this.handleJobError(jobId, change.new.error ?? "Job failed", change.new.error_code ?? undefined);
513
- } else if (newStatus === JobStatus2.DISABLED) {
521
+ } else if (newStatus === JobStatus.DISABLED) {
514
522
  this.handleJobDisabled(jobId);
515
- } else if (newStatus === JobStatus2.PENDING && oldStatus === JobStatus2.PROCESSING) {
523
+ } else if (newStatus === JobStatus.PENDING && oldStatus === JobStatus.PROCESSING) {
516
524
  const runAfter = change.new.run_after ? new Date(change.new.run_after) : new Date;
517
525
  this.handleJobRetry(jobId, runAfter);
518
526
  }
@@ -555,12 +563,11 @@ class JobQueueClient {
555
563
  }
556
564
 
557
565
  // src/job/JobQueueServer.ts
558
- import { JobStatus as JobStatus4 } from "@workglow/storage";
559
566
  import { EventEmitter as EventEmitter3, getLogger as getLogger2 } from "@workglow/util";
560
567
 
561
568
  // src/limiter/NullLimiter.ts
562
- import { createServiceToken } from "@workglow/util";
563
- var NULL_JOB_LIMITER = createServiceToken("jobqueue.limiter.null");
569
+ import { createServiceToken as createServiceToken2 } from "@workglow/util";
570
+ var NULL_JOB_LIMITER = createServiceToken2("jobqueue.limiter.null");
564
571
 
565
572
  class NullLimiter {
566
573
  scope = "process";
@@ -582,7 +589,6 @@ class NullLimiter {
582
589
  }
583
590
 
584
591
  // src/job/JobQueueWorker.ts
585
- import { JobStatus as JobStatus3 } from "@workglow/storage";
586
592
  import {
587
593
  EventEmitter as EventEmitter2,
588
594
  getLogger,
@@ -758,7 +764,7 @@ class JobQueueWorker {
758
764
  }
759
765
  async getIdleDelay() {
760
766
  try {
761
- const pending = await this.storage.peek(JobStatus3.PENDING, 1);
767
+ const pending = await this.storage.peek(JobStatus.PENDING, 1);
762
768
  if (pending.length > 0 && pending[0].run_after) {
763
769
  const delay = new Date(pending[0].run_after).getTime() - Date.now();
764
770
  if (delay > 0) {
@@ -790,7 +796,7 @@ class JobQueueWorker {
790
796
  if (this.activeJobAbortControllers.size === 0) {
791
797
  return;
792
798
  }
793
- const abortingJobs = await this.storage.peek(JobStatus3.ABORTING);
799
+ const abortingJobs = await this.storage.peek(JobStatus.ABORTING);
794
800
  for (const jobData of abortingJobs) {
795
801
  const controller = this.activeJobAbortControllers.get(jobData.id);
796
802
  if (controller && !controller.signal.aborted) {
@@ -886,7 +892,7 @@ class JobQueueWorker {
886
892
  }
887
893
  async completeJob(job, output) {
888
894
  try {
889
- job.status = JobStatus3.COMPLETED;
895
+ job.status = JobStatus.COMPLETED;
890
896
  job.progress = 100;
891
897
  job.progressMessage = "";
892
898
  job.progressDetails = null;
@@ -904,7 +910,7 @@ class JobQueueWorker {
904
910
  }
905
911
  async failJob(job, error) {
906
912
  try {
907
- job.status = JobStatus3.FAILED;
913
+ job.status = JobStatus.FAILED;
908
914
  job.progress = 100;
909
915
  job.completedAt = new Date;
910
916
  job.progressMessage = "";
@@ -921,7 +927,7 @@ class JobQueueWorker {
921
927
  }
922
928
  async disableJob(job) {
923
929
  try {
924
- job.status = JobStatus3.DISABLED;
930
+ job.status = JobStatus.DISABLED;
925
931
  job.progress = 100;
926
932
  job.completedAt = new Date;
927
933
  job.progressMessage = "";
@@ -943,7 +949,7 @@ class JobQueueWorker {
943
949
  }
944
950
  async rescheduleJob(job, retryDate) {
945
951
  try {
946
- job.status = JobStatus3.PENDING;
952
+ job.status = JobStatus.PENDING;
947
953
  const nextAvailableTime = await this.limiter.getNextAvailableTime();
948
954
  job.runAfter = retryDate instanceof Date ? retryDate : nextAvailableTime;
949
955
  job.progress = 0;
@@ -979,7 +985,7 @@ class JobQueueWorker {
979
985
  getLogger().error("handleAbort: job not found", { jobId });
980
986
  return;
981
987
  }
982
- if (job.status === JobStatus3.COMPLETED || job.status === JobStatus3.FAILED || job.status === JobStatus3.DISABLED) {
988
+ if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED || job.status === JobStatus.DISABLED) {
983
989
  return;
984
990
  }
985
991
  await this.failJob(job, new AbortSignalJobError("Job Aborted"));
@@ -991,19 +997,19 @@ class JobQueueWorker {
991
997
  return this.storageToClass(job);
992
998
  }
993
999
  async validateJobState(job) {
994
- if (job.status === JobStatus3.COMPLETED) {
1000
+ if (job.status === JobStatus.COMPLETED) {
995
1001
  throw new PermanentJobError(`Job ${job.id} is already completed`);
996
1002
  }
997
- if (job.status === JobStatus3.FAILED) {
1003
+ if (job.status === JobStatus.FAILED) {
998
1004
  throw new PermanentJobError(`Job ${job.id} has failed`);
999
1005
  }
1000
- if (job.status === JobStatus3.ABORTING || this.activeJobAbortControllers.get(job.id)?.signal.aborted) {
1006
+ if (job.status === JobStatus.ABORTING || this.activeJobAbortControllers.get(job.id)?.signal.aborted) {
1001
1007
  throw new AbortSignalJobError(`Job ${job.id} is being aborted`);
1002
1008
  }
1003
1009
  if (job.deadlineAt && job.deadlineAt < new Date) {
1004
1010
  throw new PermanentJobError(`Job ${job.id} has exceeded its deadline`);
1005
1011
  }
1006
- if (job.status === JobStatus3.DISABLED) {
1012
+ if (job.status === JobStatus.DISABLED) {
1007
1013
  throw new JobDisabledError(`Job ${job.id} has been disabled`);
1008
1014
  }
1009
1015
  }
@@ -1085,7 +1091,7 @@ class JobQueueServer {
1085
1091
  await this.fixupJobs();
1086
1092
  try {
1087
1093
  this.storageUnsubscribe = this.storage.subscribeToChanges((change) => {
1088
- if (change.type === "INSERT" || change.type === "RESYNC" || change.type === "UPDATE" && change.new?.status === JobStatus4.PENDING) {
1094
+ if (change.type === "INSERT" || change.type === "RESYNC" || change.type === "UPDATE" && change.new?.status === JobStatus.PENDING) {
1089
1095
  this.notifyWorkers();
1090
1096
  }
1091
1097
  });
@@ -1280,13 +1286,13 @@ class JobQueueServer {
1280
1286
  async cleanupJobs() {
1281
1287
  try {
1282
1288
  if (this.deleteAfterCompletionMs !== undefined && this.deleteAfterCompletionMs > 0) {
1283
- await this.storage.deleteJobsByStatusAndAge(JobStatus4.COMPLETED, this.deleteAfterCompletionMs);
1289
+ await this.storage.deleteJobsByStatusAndAge(JobStatus.COMPLETED, this.deleteAfterCompletionMs);
1284
1290
  }
1285
1291
  if (this.deleteAfterFailureMs !== undefined && this.deleteAfterFailureMs > 0) {
1286
- await this.storage.deleteJobsByStatusAndAge(JobStatus4.FAILED, this.deleteAfterFailureMs);
1292
+ await this.storage.deleteJobsByStatusAndAge(JobStatus.FAILED, this.deleteAfterFailureMs);
1287
1293
  }
1288
1294
  if (this.deleteAfterDisabledMs !== undefined && this.deleteAfterDisabledMs > 0) {
1289
- await this.storage.deleteJobsByStatusAndAge(JobStatus4.DISABLED, this.deleteAfterDisabledMs);
1295
+ await this.storage.deleteJobsByStatusAndAge(JobStatus.DISABLED, this.deleteAfterDisabledMs);
1290
1296
  }
1291
1297
  } catch (error) {
1292
1298
  console.error("Error in cleanup:", error);
@@ -1294,8 +1300,8 @@ class JobQueueServer {
1294
1300
  }
1295
1301
  async fixupJobs() {
1296
1302
  try {
1297
- const stuckProcessingJobs = await this.storage.peek(JobStatus4.PROCESSING);
1298
- const stuckAbortingJobs = await this.storage.peek(JobStatus4.ABORTING);
1303
+ const stuckProcessingJobs = await this.storage.peek(JobStatus.PROCESSING);
1304
+ const stuckAbortingJobs = await this.storage.peek(JobStatus.ABORTING);
1299
1305
  const stuckJobs = [...stuckProcessingJobs, ...stuckAbortingJobs];
1300
1306
  const currentWorkerIds = new Set(this.getWorkerIds());
1301
1307
  for (const jobData of stuckJobs) {
@@ -1304,12 +1310,12 @@ class JobQueueServer {
1304
1310
  }
1305
1311
  const job = this.storageToClass(jobData);
1306
1312
  if (job.runAttempts >= job.maxRetries) {
1307
- job.status = JobStatus4.FAILED;
1313
+ job.status = JobStatus.FAILED;
1308
1314
  job.error = "Max retries reached";
1309
1315
  job.errorCode = "MAX_RETRIES_REACHED";
1310
1316
  job.workerId = null;
1311
1317
  } else {
1312
- job.status = JobStatus4.PENDING;
1318
+ job.status = JobStatus.PENDING;
1313
1319
  job.runAfter = job.lastRanAt || new Date;
1314
1320
  job.progress = 0;
1315
1321
  job.progressMessage = "";
@@ -1402,8 +1408,8 @@ class CompositeLimiter {
1402
1408
  }
1403
1409
 
1404
1410
  // src/limiter/ConcurrencyLimiter.ts
1405
- import { createServiceToken as createServiceToken2 } from "@workglow/util";
1406
- var CONCURRENT_JOB_LIMITER = createServiceToken2("jobqueue.limiter.concurrent");
1411
+ import { createServiceToken as createServiceToken3 } from "@workglow/util";
1412
+ var CONCURRENT_JOB_LIMITER = createServiceToken3("jobqueue.limiter.concurrent");
1407
1413
 
1408
1414
  class ConcurrencyLimiter {
1409
1415
  scope = "process";
@@ -1496,8 +1502,8 @@ class DelayLimiter {
1496
1502
  }
1497
1503
 
1498
1504
  // src/limiter/EvenlySpacedRateLimiter.ts
1499
- import { createServiceToken as createServiceToken3 } from "@workglow/util";
1500
- var EVENLY_SPACED_JOB_RATE_LIMITER = createServiceToken3("jobqueue.limiter.rate.evenlyspaced");
1505
+ import { createServiceToken as createServiceToken4 } from "@workglow/util";
1506
+ var EVENLY_SPACED_JOB_RATE_LIMITER = createServiceToken4("jobqueue.limiter.rate.evenlyspaced");
1501
1507
 
1502
1508
  class EvenlySpacedRateLimiter {
1503
1509
  scope = "process";
@@ -1597,8 +1603,8 @@ class EvenlySpacedRateLimiter {
1597
1603
  }
1598
1604
 
1599
1605
  // src/limiter/ILimiter.ts
1600
- import { createServiceToken as createServiceToken4 } from "@workglow/util";
1601
- var JOB_LIMITER = createServiceToken4("jobqueue.limiter");
1606
+ import { createServiceToken as createServiceToken5 } from "@workglow/util";
1607
+ var JOB_LIMITER = createServiceToken5("jobqueue.limiter");
1602
1608
 
1603
1609
  // src/limiter/RateLimiter.ts
1604
1610
  class RateLimiter {
@@ -1730,14 +1736,418 @@ class RateLimiter {
1730
1736
  this.localBackoffUntilMs = 0;
1731
1737
  }
1732
1738
  }
1739
+
1740
+ // src/queue-storage/InMemoryQueueStorage.ts
1741
+ import {
1742
+ createServiceToken as createServiceToken6,
1743
+ EventEmitter as EventEmitter4,
1744
+ getLogger as getLogger3,
1745
+ makeFingerprint,
1746
+ sleep as sleep2,
1747
+ uuid4 as uuid42
1748
+ } from "@workglow/util";
1749
+ var IN_MEMORY_QUEUE_STORAGE = createServiceToken6("jobqueue.storage.inMemory");
1750
+
1751
+ class InMemoryQueueStorage {
1752
+ queueName;
1753
+ scope = "process";
1754
+ prefixValues;
1755
+ events = new EventEmitter4;
1756
+ constructor(queueName, options) {
1757
+ this.queueName = queueName;
1758
+ this.jobQueue = [];
1759
+ this.prefixValues = options?.prefixValues ?? {};
1760
+ }
1761
+ jobQueue;
1762
+ matchesPrefixes(job) {
1763
+ for (const [key, value] of Object.entries(this.prefixValues)) {
1764
+ if (job[key] !== value) {
1765
+ return false;
1766
+ }
1767
+ }
1768
+ return true;
1769
+ }
1770
+ pendingQueue() {
1771
+ const now = new Date().toISOString();
1772
+ return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === JobStatus.PENDING).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
1773
+ }
1774
+ async add(job) {
1775
+ await sleep2(0);
1776
+ const now = new Date().toISOString();
1777
+ const jobWithPrefixes = job;
1778
+ jobWithPrefixes.id = jobWithPrefixes.id ?? uuid42();
1779
+ jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid42();
1780
+ jobWithPrefixes.queue = this.queueName;
1781
+ jobWithPrefixes.fingerprint = await makeFingerprint(jobWithPrefixes.input);
1782
+ jobWithPrefixes.status = JobStatus.PENDING;
1783
+ jobWithPrefixes.progress = 0;
1784
+ jobWithPrefixes.progress_message = "";
1785
+ jobWithPrefixes.progress_details = null;
1786
+ jobWithPrefixes.created_at = now;
1787
+ jobWithPrefixes.run_after = now;
1788
+ for (const [key, value] of Object.entries(this.prefixValues)) {
1789
+ jobWithPrefixes[key] = value;
1790
+ }
1791
+ this.jobQueue.push(jobWithPrefixes);
1792
+ this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
1793
+ return jobWithPrefixes.id;
1794
+ }
1795
+ async get(id) {
1796
+ await sleep2(0);
1797
+ const job = this.jobQueue.find((j) => j.id === id);
1798
+ if (job && this.matchesPrefixes(job)) {
1799
+ return job;
1800
+ }
1801
+ return;
1802
+ }
1803
+ async peek(status = JobStatus.PENDING, num = 100) {
1804
+ await sleep2(0);
1805
+ num = Number(num) || 100;
1806
+ return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
1807
+ }
1808
+ async next(workerId) {
1809
+ await sleep2(0);
1810
+ const top = this.pendingQueue();
1811
+ const job = top[0];
1812
+ if (job) {
1813
+ const oldJob = { ...job };
1814
+ job.status = JobStatus.PROCESSING;
1815
+ job.last_ran_at = new Date().toISOString();
1816
+ job.worker_id = workerId;
1817
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1818
+ return job;
1819
+ }
1820
+ }
1821
+ async size(status = JobStatus.PENDING) {
1822
+ await sleep2(0);
1823
+ return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
1824
+ }
1825
+ async saveProgress(id, progress, message, details) {
1826
+ await sleep2(0);
1827
+ const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
1828
+ if (!job) {
1829
+ const jobWithAnyPrefix = this.jobQueue.find((j) => j.id === id);
1830
+ getLogger3().warn("Job not found for progress update", {
1831
+ id,
1832
+ reason: jobWithAnyPrefix ? "prefix_mismatch" : "missing",
1833
+ existingStatus: jobWithAnyPrefix?.status,
1834
+ queueName: this.queueName,
1835
+ prefixValues: this.prefixValues
1836
+ });
1837
+ return;
1838
+ }
1839
+ if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED) {
1840
+ getLogger3().warn("Job already completed or failed for progress update", {
1841
+ id,
1842
+ status: job.status,
1843
+ completedAt: job.completed_at,
1844
+ error: job.error
1845
+ });
1846
+ return;
1847
+ }
1848
+ const oldJob = { ...job };
1849
+ job.progress = progress;
1850
+ job.progress_message = message;
1851
+ job.progress_details = details;
1852
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1853
+ }
1854
+ async complete(job) {
1855
+ await sleep2(0);
1856
+ const jobWithPrefixes = job;
1857
+ const index = this.jobQueue.findIndex((j) => j.id === job.id && this.matchesPrefixes(j));
1858
+ if (index !== -1) {
1859
+ const existing = this.jobQueue[index];
1860
+ const currentAttempts = existing?.run_attempts ?? 0;
1861
+ jobWithPrefixes.run_attempts = currentAttempts + 1;
1862
+ for (const [key, value] of Object.entries(this.prefixValues)) {
1863
+ jobWithPrefixes[key] = value;
1864
+ }
1865
+ this.jobQueue[index] = jobWithPrefixes;
1866
+ this.events.emit("change", { type: "UPDATE", old: existing, new: jobWithPrefixes });
1867
+ }
1868
+ }
1869
+ async release(id) {
1870
+ await sleep2(0);
1871
+ const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
1872
+ if (job) {
1873
+ const oldJob = { ...job };
1874
+ job.status = JobStatus.PENDING;
1875
+ job.worker_id = null;
1876
+ job.progress = 0;
1877
+ job.progress_message = "";
1878
+ job.progress_details = null;
1879
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1880
+ }
1881
+ }
1882
+ async abort(id) {
1883
+ await sleep2(0);
1884
+ const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
1885
+ if (job) {
1886
+ const oldJob = { ...job };
1887
+ job.status = JobStatus.ABORTING;
1888
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1889
+ }
1890
+ }
1891
+ async getByRunId(runId) {
1892
+ await sleep2(0);
1893
+ return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
1894
+ }
1895
+ async deleteAll() {
1896
+ await sleep2(0);
1897
+ const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
1898
+ this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
1899
+ for (const job of deletedJobs) {
1900
+ this.events.emit("change", { type: "DELETE", old: job });
1901
+ }
1902
+ }
1903
+ async outputForInput(input) {
1904
+ await sleep2(0);
1905
+ const fingerprint = await makeFingerprint(input);
1906
+ return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === JobStatus.COMPLETED)?.output ?? null;
1907
+ }
1908
+ async delete(id) {
1909
+ await sleep2(0);
1910
+ const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
1911
+ this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
1912
+ if (deletedJob) {
1913
+ this.events.emit("change", { type: "DELETE", old: deletedJob });
1914
+ }
1915
+ }
1916
+ async deleteJobsByStatusAndAge(status, olderThanMs) {
1917
+ await sleep2(0);
1918
+ const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
1919
+ const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
1920
+ this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
1921
+ for (const job of deletedJobs) {
1922
+ this.events.emit("change", { type: "DELETE", old: job });
1923
+ }
1924
+ }
1925
+ async setupDatabase() {}
1926
+ matchesPrefixFilter(job, prefixFilter) {
1927
+ if (prefixFilter && Object.keys(prefixFilter).length === 0) {
1928
+ return true;
1929
+ }
1930
+ const filterValues = prefixFilter ?? this.prefixValues;
1931
+ if (Object.keys(filterValues).length === 0) {
1932
+ return true;
1933
+ }
1934
+ const jobWithPrefixes = job;
1935
+ for (const [key, value] of Object.entries(filterValues)) {
1936
+ if (jobWithPrefixes[key] !== value) {
1937
+ return false;
1938
+ }
1939
+ }
1940
+ return true;
1941
+ }
1942
+ subscribeToChanges(callback, options) {
1943
+ const prefixFilter = options?.prefixFilter;
1944
+ const filteredCallback = (change) => {
1945
+ const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
1946
+ const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
1947
+ if (!newMatches && !oldMatches) {
1948
+ return;
1949
+ }
1950
+ callback(change);
1951
+ };
1952
+ return this.events.subscribe("change", filteredCallback);
1953
+ }
1954
+ }
1955
+
1956
+ // src/queue-storage/TelemetryQueueStorage.ts
1957
+ import { traced } from "@workglow/util";
1958
+
1959
+ class TelemetryQueueStorage {
1960
+ storageName;
1961
+ inner;
1962
+ constructor(storageName, inner) {
1963
+ this.storageName = storageName;
1964
+ this.inner = inner;
1965
+ }
1966
+ get scope() {
1967
+ return this.inner.scope;
1968
+ }
1969
+ add(job) {
1970
+ return traced("workglow.storage.queue.add", this.storageName, () => this.inner.add(job));
1971
+ }
1972
+ get(id) {
1973
+ return traced("workglow.storage.queue.get", this.storageName, () => this.inner.get(id));
1974
+ }
1975
+ next(workerId) {
1976
+ return traced("workglow.storage.queue.next", this.storageName, () => this.inner.next(workerId));
1977
+ }
1978
+ peek(status, num) {
1979
+ return traced("workglow.storage.queue.peek", this.storageName, () => this.inner.peek(status, num));
1980
+ }
1981
+ size(status) {
1982
+ return traced("workglow.storage.queue.size", this.storageName, () => this.inner.size(status));
1983
+ }
1984
+ complete(job) {
1985
+ return traced("workglow.storage.queue.complete", this.storageName, () => this.inner.complete(job));
1986
+ }
1987
+ release(id) {
1988
+ return traced("workglow.storage.queue.release", this.storageName, () => this.inner.release(id));
1989
+ }
1990
+ deleteAll() {
1991
+ return traced("workglow.storage.queue.deleteAll", this.storageName, () => this.inner.deleteAll());
1992
+ }
1993
+ outputForInput(input) {
1994
+ return traced("workglow.storage.queue.outputForInput", this.storageName, () => this.inner.outputForInput(input));
1995
+ }
1996
+ abort(id) {
1997
+ return traced("workglow.storage.queue.abort", this.storageName, () => this.inner.abort(id));
1998
+ }
1999
+ getByRunId(runId) {
2000
+ return traced("workglow.storage.queue.getByRunId", this.storageName, () => this.inner.getByRunId(runId));
2001
+ }
2002
+ saveProgress(id, progress, message, details) {
2003
+ return traced("workglow.storage.queue.saveProgress", this.storageName, () => this.inner.saveProgress(id, progress, message, details));
2004
+ }
2005
+ delete(id) {
2006
+ return traced("workglow.storage.queue.delete", this.storageName, () => this.inner.delete(id));
2007
+ }
2008
+ deleteJobsByStatusAndAge(status, olderThanMs) {
2009
+ return traced("workglow.storage.queue.deleteJobsByStatusAndAge", this.storageName, () => this.inner.deleteJobsByStatusAndAge(status, olderThanMs));
2010
+ }
2011
+ setupDatabase() {
2012
+ return this.inner.setupDatabase();
2013
+ }
2014
+ subscribeToChanges(callback, options) {
2015
+ return this.inner.subscribeToChanges(callback, options);
2016
+ }
2017
+ }
2018
+
2019
+ // src/rate-limiter-storage/IRateLimiterStorage.ts
2020
+ import { createServiceToken as createServiceToken7 } from "@workglow/util";
2021
+ var RATE_LIMITER_STORAGE = createServiceToken7("ratelimiter.storage");
2022
+
2023
+ // src/rate-limiter-storage/InMemoryRateLimiterStorage.ts
2024
+ import { createServiceToken as createServiceToken8, sleep as sleep3, uuid4 as uuid43 } from "@workglow/util";
2025
+ var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage.inMemory");
2026
+
2027
+ class InMemoryRateLimiterStorage {
2028
+ scope = "process";
2029
+ prefixValues;
2030
+ executions = new Map;
2031
+ nextAvailableTimes = new Map;
2032
+ reserveChains = new Map;
2033
+ constructor(options) {
2034
+ this.prefixValues = options?.prefixValues ?? {};
2035
+ }
2036
+ makeKey(queueName) {
2037
+ const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
2038
+ return prefixPart ? `${prefixPart}|${queueName}` : queueName;
2039
+ }
2040
+ async setupDatabase() {}
2041
+ async withKeyLock(key, fn) {
2042
+ const previous = this.reserveChains.get(key) ?? Promise.resolve();
2043
+ let release;
2044
+ const next = new Promise((resolve) => {
2045
+ release = resolve;
2046
+ });
2047
+ this.reserveChains.set(key, next);
2048
+ try {
2049
+ await previous;
2050
+ return await fn();
2051
+ } finally {
2052
+ release(undefined);
2053
+ if (this.reserveChains.get(key) === next) {
2054
+ this.reserveChains.delete(key);
2055
+ }
2056
+ }
2057
+ }
2058
+ async tryReserveExecution(queueName, maxExecutions, windowMs) {
2059
+ const key = this.makeKey(queueName);
2060
+ return this.withKeyLock(key, () => {
2061
+ const now = Date.now();
2062
+ const windowStart = new Date(now - windowMs);
2063
+ const executions = this.executions.get(key) ?? [];
2064
+ const live = executions.filter((e) => e.executedAt > windowStart);
2065
+ if (live.length >= maxExecutions) {
2066
+ if (live.length !== executions.length) {
2067
+ this.executions.set(key, live);
2068
+ }
2069
+ return null;
2070
+ }
2071
+ const next = this.nextAvailableTimes.get(key);
2072
+ if (next && next.getTime() > now) {
2073
+ return null;
2074
+ }
2075
+ const id = uuid43();
2076
+ live.push({ id, queueName, executedAt: new Date(now) });
2077
+ this.executions.set(key, live);
2078
+ return id;
2079
+ });
2080
+ }
2081
+ async releaseExecution(queueName, token) {
2082
+ if (token === null || token === undefined)
2083
+ return;
2084
+ const key = this.makeKey(queueName);
2085
+ await this.withKeyLock(key, () => {
2086
+ const executions = this.executions.get(key);
2087
+ if (!executions || executions.length === 0)
2088
+ return;
2089
+ const idx = executions.findIndex((e) => e.id === token);
2090
+ if (idx === -1)
2091
+ return;
2092
+ executions.splice(idx, 1);
2093
+ this.executions.set(key, executions);
2094
+ });
2095
+ }
2096
+ async recordExecution(queueName) {
2097
+ await sleep3(0);
2098
+ const key = this.makeKey(queueName);
2099
+ const executions = this.executions.get(key) ?? [];
2100
+ executions.push({
2101
+ id: uuid43(),
2102
+ queueName,
2103
+ executedAt: new Date
2104
+ });
2105
+ this.executions.set(key, executions);
2106
+ }
2107
+ async getExecutionCount(queueName, windowStartTime) {
2108
+ await sleep3(0);
2109
+ const key = this.makeKey(queueName);
2110
+ const executions = this.executions.get(key) ?? [];
2111
+ const windowStart = new Date(windowStartTime);
2112
+ return executions.filter((e) => e.executedAt > windowStart).length;
2113
+ }
2114
+ async getOldestExecutionAtOffset(queueName, offset) {
2115
+ await sleep3(0);
2116
+ const key = this.makeKey(queueName);
2117
+ const executions = this.executions.get(key) ?? [];
2118
+ const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
2119
+ const execution = sorted[offset];
2120
+ return execution?.executedAt.toISOString();
2121
+ }
2122
+ async getNextAvailableTime(queueName) {
2123
+ await sleep3(0);
2124
+ const key = this.makeKey(queueName);
2125
+ const time = this.nextAvailableTimes.get(key);
2126
+ return time?.toISOString();
2127
+ }
2128
+ async setNextAvailableTime(queueName, nextAvailableAt) {
2129
+ await sleep3(0);
2130
+ const key = this.makeKey(queueName);
2131
+ this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
2132
+ }
2133
+ async clear(queueName) {
2134
+ await sleep3(0);
2135
+ const key = this.makeKey(queueName);
2136
+ this.executions.delete(key);
2137
+ this.nextAvailableTimes.delete(key);
2138
+ }
2139
+ }
1733
2140
  export {
1734
2141
  withJobErrorDiagnostics,
1735
2142
  storageToClass,
1736
2143
  formatErrorChainForDiagnostics,
1737
2144
  classToStorage,
1738
2145
  applyPersistedDiagnosticsToStack,
2146
+ TelemetryQueueStorage,
1739
2147
  RetryableJobError,
1740
2148
  RateLimiter,
2149
+ RATE_LIMITER_STORAGE,
2150
+ QUEUE_STORAGE,
1741
2151
  PermanentJobError,
1742
2152
  NullLimiter,
1743
2153
  NULL_JOB_LIMITER,
@@ -1751,6 +2161,10 @@ export {
1751
2161
  Job,
1752
2162
  JOB_LIMITER,
1753
2163
  JOB_ERROR_DIAGNOSTICS_MARKER,
2164
+ InMemoryRateLimiterStorage,
2165
+ InMemoryQueueStorage,
2166
+ IN_MEMORY_RATE_LIMITER_STORAGE,
2167
+ IN_MEMORY_QUEUE_STORAGE,
1754
2168
  EvenlySpacedRateLimiter,
1755
2169
  EVENLY_SPACED_JOB_RATE_LIMITER,
1756
2170
  DelayLimiter,
@@ -1760,4 +2174,4 @@ export {
1760
2174
  AbortSignalJobError
1761
2175
  };
1762
2176
 
1763
- //# debugId=81121BC83C83DF5F64756E2164756E21
2177
+ //# debugId=4ACCF006917528EA64756E2164756E21