@workglow/job-queue 0.2.27 → 0.2.28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/browser.js +457 -43
  2. package/dist/browser.js.map +15 -10
  3. package/dist/bun.js +457 -43
  4. package/dist/bun.js.map +15 -10
  5. package/dist/common.d.ts +5 -0
  6. package/dist/common.d.ts.map +1 -1
  7. package/dist/job/Job.d.ts +1 -1
  8. package/dist/job/Job.d.ts.map +1 -1
  9. package/dist/job/JobQueueClient.d.ts +2 -1
  10. package/dist/job/JobQueueClient.d.ts.map +1 -1
  11. package/dist/job/JobQueueServer.d.ts +1 -1
  12. package/dist/job/JobQueueServer.d.ts.map +1 -1
  13. package/dist/job/JobQueueWorker.d.ts +1 -1
  14. package/dist/job/JobQueueWorker.d.ts.map +1 -1
  15. package/dist/job/JobStorageConverters.d.ts +1 -1
  16. package/dist/job/JobStorageConverters.d.ts.map +1 -1
  17. package/dist/limiter/CompositeLimiter.d.ts.map +1 -1
  18. package/dist/limiter/RateLimiter.d.ts +1 -1
  19. package/dist/limiter/RateLimiter.d.ts.map +1 -1
  20. package/dist/node.js +457 -43
  21. package/dist/node.js.map +15 -10
  22. package/dist/queue-storage/IQueueStorage.d.ts +229 -0
  23. package/dist/queue-storage/IQueueStorage.d.ts.map +1 -0
  24. package/dist/queue-storage/InMemoryQueueStorage.d.ts +149 -0
  25. package/dist/queue-storage/InMemoryQueueStorage.d.ts.map +1 -0
  26. package/dist/queue-storage/TelemetryQueueStorage.d.ts +33 -0
  27. package/dist/queue-storage/TelemetryQueueStorage.d.ts.map +1 -0
  28. package/dist/rate-limiter-storage/IRateLimiterStorage.d.ts +127 -0
  29. package/dist/rate-limiter-storage/IRateLimiterStorage.d.ts.map +1 -0
  30. package/dist/rate-limiter-storage/InMemoryRateLimiterStorage.d.ts +43 -0
  31. package/dist/rate-limiter-storage/InMemoryRateLimiterStorage.d.ts.map +1 -0
  32. package/package.json +3 -8
package/dist/node.js CHANGED
@@ -1,5 +1,14 @@
1
- // src/job/Job.ts
2
- import { JobStatus } from "@workglow/storage";
1
+ // src/queue-storage/IQueueStorage.ts
2
+ import { createServiceToken } from "@workglow/util";
3
+ var QUEUE_STORAGE = createServiceToken("jobqueue.storage");
4
+ var JobStatus = {
5
+ PENDING: "PENDING",
6
+ PROCESSING: "PROCESSING",
7
+ COMPLETED: "COMPLETED",
8
+ ABORTING: "ABORTING",
9
+ FAILED: "FAILED",
10
+ DISABLED: "DISABLED"
11
+ };
3
12
 
4
13
  // src/job/JobError.ts
5
14
  import { BaseError } from "@workglow/util";
@@ -175,7 +184,6 @@ ${fullMessage}`;
175
184
  }
176
185
 
177
186
  // src/job/JobQueueClient.ts
178
- import { JobStatus as JobStatus2 } from "@workglow/storage";
179
187
  import { EventEmitter } from "@workglow/util";
180
188
 
181
189
  // src/job/JobStorageConverters.ts
@@ -300,7 +308,7 @@ class JobQueueClient {
300
308
  run_after: options?.runAfter?.toISOString() ?? new Date().toISOString(),
301
309
  deadline_at: options?.deadlineAt?.toISOString() ?? null,
302
310
  completed_at: null,
303
- status: JobStatus2.PENDING
311
+ status: JobStatus.PENDING
304
312
  };
305
313
  const id = await this.storage.add(job);
306
314
  this.server?.handleJobAdded(id);
@@ -353,15 +361,15 @@ class JobQueueClient {
353
361
  this.removePromise(jobId, resolve, reject);
354
362
  throw new JobNotFoundError(`Job ${jobId} not found`);
355
363
  }
356
- if (job.status === JobStatus2.COMPLETED) {
364
+ if (job.status === JobStatus.COMPLETED) {
357
365
  this.removePromise(jobId, resolve, reject);
358
366
  return job.output;
359
367
  }
360
- if (job.status === JobStatus2.DISABLED) {
368
+ if (job.status === JobStatus.DISABLED) {
361
369
  this.removePromise(jobId, resolve, reject);
362
370
  throw new JobDisabledError(`Job ${jobId} was disabled`);
363
371
  }
364
- if (job.status === JobStatus2.FAILED) {
372
+ if (job.status === JobStatus.FAILED) {
365
373
  this.removePromise(jobId, resolve, reject);
366
374
  throw this.buildErrorFromJob(job);
367
375
  }
@@ -396,7 +404,7 @@ class JobQueueClient {
396
404
  throw new JobNotFoundError("Cannot abort job run with undefined jobRunId");
397
405
  const jobs = await this.getJobsByRunId(jobRunId);
398
406
  await Promise.allSettled(jobs.map((job) => {
399
- if (job.status === JobStatus2.PROCESSING || job.status === JobStatus2.PENDING) {
407
+ if (job.status === JobStatus.PROCESSING || job.status === JobStatus.PENDING) {
400
408
  return this.abort(job.id);
401
409
  }
402
410
  }));
@@ -503,15 +511,15 @@ class JobQueueClient {
503
511
  if (change.type === "UPDATE" && change.new) {
504
512
  const newStatus = change.new.status;
505
513
  const oldStatus = change.old?.status;
506
- if (newStatus === JobStatus2.PROCESSING && oldStatus === JobStatus2.PENDING) {
514
+ if (newStatus === JobStatus.PROCESSING && oldStatus === JobStatus.PENDING) {
507
515
  this.handleJobStart(jobId);
508
- } else if (newStatus === JobStatus2.COMPLETED) {
516
+ } else if (newStatus === JobStatus.COMPLETED) {
509
517
  this.handleJobComplete(jobId, change.new.output);
510
- } else if (newStatus === JobStatus2.FAILED) {
518
+ } else if (newStatus === JobStatus.FAILED) {
511
519
  this.handleJobError(jobId, change.new.error ?? "Job failed", change.new.error_code ?? undefined);
512
- } else if (newStatus === JobStatus2.DISABLED) {
520
+ } else if (newStatus === JobStatus.DISABLED) {
513
521
  this.handleJobDisabled(jobId);
514
- } else if (newStatus === JobStatus2.PENDING && oldStatus === JobStatus2.PROCESSING) {
522
+ } else if (newStatus === JobStatus.PENDING && oldStatus === JobStatus.PROCESSING) {
515
523
  const runAfter = change.new.run_after ? new Date(change.new.run_after) : new Date;
516
524
  this.handleJobRetry(jobId, runAfter);
517
525
  }
@@ -554,12 +562,11 @@ class JobQueueClient {
554
562
  }
555
563
 
556
564
  // src/job/JobQueueServer.ts
557
- import { JobStatus as JobStatus4 } from "@workglow/storage";
558
565
  import { EventEmitter as EventEmitter3, getLogger as getLogger2 } from "@workglow/util";
559
566
 
560
567
  // src/limiter/NullLimiter.ts
561
- import { createServiceToken } from "@workglow/util";
562
- var NULL_JOB_LIMITER = createServiceToken("jobqueue.limiter.null");
568
+ import { createServiceToken as createServiceToken2 } from "@workglow/util";
569
+ var NULL_JOB_LIMITER = createServiceToken2("jobqueue.limiter.null");
563
570
 
564
571
  class NullLimiter {
565
572
  scope = "process";
@@ -581,7 +588,6 @@ class NullLimiter {
581
588
  }
582
589
 
583
590
  // src/job/JobQueueWorker.ts
584
- import { JobStatus as JobStatus3 } from "@workglow/storage";
585
591
  import {
586
592
  EventEmitter as EventEmitter2,
587
593
  getLogger,
@@ -757,7 +763,7 @@ class JobQueueWorker {
757
763
  }
758
764
  async getIdleDelay() {
759
765
  try {
760
- const pending = await this.storage.peek(JobStatus3.PENDING, 1);
766
+ const pending = await this.storage.peek(JobStatus.PENDING, 1);
761
767
  if (pending.length > 0 && pending[0].run_after) {
762
768
  const delay = new Date(pending[0].run_after).getTime() - Date.now();
763
769
  if (delay > 0) {
@@ -789,7 +795,7 @@ class JobQueueWorker {
789
795
  if (this.activeJobAbortControllers.size === 0) {
790
796
  return;
791
797
  }
792
- const abortingJobs = await this.storage.peek(JobStatus3.ABORTING);
798
+ const abortingJobs = await this.storage.peek(JobStatus.ABORTING);
793
799
  for (const jobData of abortingJobs) {
794
800
  const controller = this.activeJobAbortControllers.get(jobData.id);
795
801
  if (controller && !controller.signal.aborted) {
@@ -885,7 +891,7 @@ class JobQueueWorker {
885
891
  }
886
892
  async completeJob(job, output) {
887
893
  try {
888
- job.status = JobStatus3.COMPLETED;
894
+ job.status = JobStatus.COMPLETED;
889
895
  job.progress = 100;
890
896
  job.progressMessage = "";
891
897
  job.progressDetails = null;
@@ -903,7 +909,7 @@ class JobQueueWorker {
903
909
  }
904
910
  async failJob(job, error) {
905
911
  try {
906
- job.status = JobStatus3.FAILED;
912
+ job.status = JobStatus.FAILED;
907
913
  job.progress = 100;
908
914
  job.completedAt = new Date;
909
915
  job.progressMessage = "";
@@ -920,7 +926,7 @@ class JobQueueWorker {
920
926
  }
921
927
  async disableJob(job) {
922
928
  try {
923
- job.status = JobStatus3.DISABLED;
929
+ job.status = JobStatus.DISABLED;
924
930
  job.progress = 100;
925
931
  job.completedAt = new Date;
926
932
  job.progressMessage = "";
@@ -942,7 +948,7 @@ class JobQueueWorker {
942
948
  }
943
949
  async rescheduleJob(job, retryDate) {
944
950
  try {
945
- job.status = JobStatus3.PENDING;
951
+ job.status = JobStatus.PENDING;
946
952
  const nextAvailableTime = await this.limiter.getNextAvailableTime();
947
953
  job.runAfter = retryDate instanceof Date ? retryDate : nextAvailableTime;
948
954
  job.progress = 0;
@@ -978,7 +984,7 @@ class JobQueueWorker {
978
984
  getLogger().error("handleAbort: job not found", { jobId });
979
985
  return;
980
986
  }
981
- if (job.status === JobStatus3.COMPLETED || job.status === JobStatus3.FAILED || job.status === JobStatus3.DISABLED) {
987
+ if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED || job.status === JobStatus.DISABLED) {
982
988
  return;
983
989
  }
984
990
  await this.failJob(job, new AbortSignalJobError("Job Aborted"));
@@ -990,19 +996,19 @@ class JobQueueWorker {
990
996
  return this.storageToClass(job);
991
997
  }
992
998
  async validateJobState(job) {
993
- if (job.status === JobStatus3.COMPLETED) {
999
+ if (job.status === JobStatus.COMPLETED) {
994
1000
  throw new PermanentJobError(`Job ${job.id} is already completed`);
995
1001
  }
996
- if (job.status === JobStatus3.FAILED) {
1002
+ if (job.status === JobStatus.FAILED) {
997
1003
  throw new PermanentJobError(`Job ${job.id} has failed`);
998
1004
  }
999
- if (job.status === JobStatus3.ABORTING || this.activeJobAbortControllers.get(job.id)?.signal.aborted) {
1005
+ if (job.status === JobStatus.ABORTING || this.activeJobAbortControllers.get(job.id)?.signal.aborted) {
1000
1006
  throw new AbortSignalJobError(`Job ${job.id} is being aborted`);
1001
1007
  }
1002
1008
  if (job.deadlineAt && job.deadlineAt < new Date) {
1003
1009
  throw new PermanentJobError(`Job ${job.id} has exceeded its deadline`);
1004
1010
  }
1005
- if (job.status === JobStatus3.DISABLED) {
1011
+ if (job.status === JobStatus.DISABLED) {
1006
1012
  throw new JobDisabledError(`Job ${job.id} has been disabled`);
1007
1013
  }
1008
1014
  }
@@ -1084,7 +1090,7 @@ class JobQueueServer {
1084
1090
  await this.fixupJobs();
1085
1091
  try {
1086
1092
  this.storageUnsubscribe = this.storage.subscribeToChanges((change) => {
1087
- if (change.type === "INSERT" || change.type === "RESYNC" || change.type === "UPDATE" && change.new?.status === JobStatus4.PENDING) {
1093
+ if (change.type === "INSERT" || change.type === "RESYNC" || change.type === "UPDATE" && change.new?.status === JobStatus.PENDING) {
1088
1094
  this.notifyWorkers();
1089
1095
  }
1090
1096
  });
@@ -1279,13 +1285,13 @@ class JobQueueServer {
1279
1285
  async cleanupJobs() {
1280
1286
  try {
1281
1287
  if (this.deleteAfterCompletionMs !== undefined && this.deleteAfterCompletionMs > 0) {
1282
- await this.storage.deleteJobsByStatusAndAge(JobStatus4.COMPLETED, this.deleteAfterCompletionMs);
1288
+ await this.storage.deleteJobsByStatusAndAge(JobStatus.COMPLETED, this.deleteAfterCompletionMs);
1283
1289
  }
1284
1290
  if (this.deleteAfterFailureMs !== undefined && this.deleteAfterFailureMs > 0) {
1285
- await this.storage.deleteJobsByStatusAndAge(JobStatus4.FAILED, this.deleteAfterFailureMs);
1291
+ await this.storage.deleteJobsByStatusAndAge(JobStatus.FAILED, this.deleteAfterFailureMs);
1286
1292
  }
1287
1293
  if (this.deleteAfterDisabledMs !== undefined && this.deleteAfterDisabledMs > 0) {
1288
- await this.storage.deleteJobsByStatusAndAge(JobStatus4.DISABLED, this.deleteAfterDisabledMs);
1294
+ await this.storage.deleteJobsByStatusAndAge(JobStatus.DISABLED, this.deleteAfterDisabledMs);
1289
1295
  }
1290
1296
  } catch (error) {
1291
1297
  console.error("Error in cleanup:", error);
@@ -1293,8 +1299,8 @@ class JobQueueServer {
1293
1299
  }
1294
1300
  async fixupJobs() {
1295
1301
  try {
1296
- const stuckProcessingJobs = await this.storage.peek(JobStatus4.PROCESSING);
1297
- const stuckAbortingJobs = await this.storage.peek(JobStatus4.ABORTING);
1302
+ const stuckProcessingJobs = await this.storage.peek(JobStatus.PROCESSING);
1303
+ const stuckAbortingJobs = await this.storage.peek(JobStatus.ABORTING);
1298
1304
  const stuckJobs = [...stuckProcessingJobs, ...stuckAbortingJobs];
1299
1305
  const currentWorkerIds = new Set(this.getWorkerIds());
1300
1306
  for (const jobData of stuckJobs) {
@@ -1303,12 +1309,12 @@ class JobQueueServer {
1303
1309
  }
1304
1310
  const job = this.storageToClass(jobData);
1305
1311
  if (job.runAttempts >= job.maxRetries) {
1306
- job.status = JobStatus4.FAILED;
1312
+ job.status = JobStatus.FAILED;
1307
1313
  job.error = "Max retries reached";
1308
1314
  job.errorCode = "MAX_RETRIES_REACHED";
1309
1315
  job.workerId = null;
1310
1316
  } else {
1311
- job.status = JobStatus4.PENDING;
1317
+ job.status = JobStatus.PENDING;
1312
1318
  job.runAfter = job.lastRanAt || new Date;
1313
1319
  job.progress = 0;
1314
1320
  job.progressMessage = "";
@@ -1401,8 +1407,8 @@ class CompositeLimiter {
1401
1407
  }
1402
1408
 
1403
1409
  // src/limiter/ConcurrencyLimiter.ts
1404
- import { createServiceToken as createServiceToken2 } from "@workglow/util";
1405
- var CONCURRENT_JOB_LIMITER = createServiceToken2("jobqueue.limiter.concurrent");
1410
+ import { createServiceToken as createServiceToken3 } from "@workglow/util";
1411
+ var CONCURRENT_JOB_LIMITER = createServiceToken3("jobqueue.limiter.concurrent");
1406
1412
 
1407
1413
  class ConcurrencyLimiter {
1408
1414
  scope = "process";
@@ -1495,8 +1501,8 @@ class DelayLimiter {
1495
1501
  }
1496
1502
 
1497
1503
  // src/limiter/EvenlySpacedRateLimiter.ts
1498
- import { createServiceToken as createServiceToken3 } from "@workglow/util";
1499
- var EVENLY_SPACED_JOB_RATE_LIMITER = createServiceToken3("jobqueue.limiter.rate.evenlyspaced");
1504
+ import { createServiceToken as createServiceToken4 } from "@workglow/util";
1505
+ var EVENLY_SPACED_JOB_RATE_LIMITER = createServiceToken4("jobqueue.limiter.rate.evenlyspaced");
1500
1506
 
1501
1507
  class EvenlySpacedRateLimiter {
1502
1508
  scope = "process";
@@ -1596,8 +1602,8 @@ class EvenlySpacedRateLimiter {
1596
1602
  }
1597
1603
 
1598
1604
  // src/limiter/ILimiter.ts
1599
- import { createServiceToken as createServiceToken4 } from "@workglow/util";
1600
- var JOB_LIMITER = createServiceToken4("jobqueue.limiter");
1605
+ import { createServiceToken as createServiceToken5 } from "@workglow/util";
1606
+ var JOB_LIMITER = createServiceToken5("jobqueue.limiter");
1601
1607
 
1602
1608
  // src/limiter/RateLimiter.ts
1603
1609
  class RateLimiter {
@@ -1729,14 +1735,418 @@ class RateLimiter {
1729
1735
  this.localBackoffUntilMs = 0;
1730
1736
  }
1731
1737
  }
1738
+
1739
+ // src/queue-storage/InMemoryQueueStorage.ts
1740
+ import {
1741
+ createServiceToken as createServiceToken6,
1742
+ EventEmitter as EventEmitter4,
1743
+ getLogger as getLogger3,
1744
+ makeFingerprint,
1745
+ sleep as sleep2,
1746
+ uuid4 as uuid42
1747
+ } from "@workglow/util";
1748
+ var IN_MEMORY_QUEUE_STORAGE = createServiceToken6("jobqueue.storage.inMemory");
1749
+
1750
+ class InMemoryQueueStorage {
1751
+ queueName;
1752
+ scope = "process";
1753
+ prefixValues;
1754
+ events = new EventEmitter4;
1755
+ constructor(queueName, options) {
1756
+ this.queueName = queueName;
1757
+ this.jobQueue = [];
1758
+ this.prefixValues = options?.prefixValues ?? {};
1759
+ }
1760
+ jobQueue;
1761
+ matchesPrefixes(job) {
1762
+ for (const [key, value] of Object.entries(this.prefixValues)) {
1763
+ if (job[key] !== value) {
1764
+ return false;
1765
+ }
1766
+ }
1767
+ return true;
1768
+ }
1769
+ pendingQueue() {
1770
+ const now = new Date().toISOString();
1771
+ return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === JobStatus.PENDING).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
1772
+ }
1773
+ async add(job) {
1774
+ await sleep2(0);
1775
+ const now = new Date().toISOString();
1776
+ const jobWithPrefixes = job;
1777
+ jobWithPrefixes.id = jobWithPrefixes.id ?? uuid42();
1778
+ jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid42();
1779
+ jobWithPrefixes.queue = this.queueName;
1780
+ jobWithPrefixes.fingerprint = await makeFingerprint(jobWithPrefixes.input);
1781
+ jobWithPrefixes.status = JobStatus.PENDING;
1782
+ jobWithPrefixes.progress = 0;
1783
+ jobWithPrefixes.progress_message = "";
1784
+ jobWithPrefixes.progress_details = null;
1785
+ jobWithPrefixes.created_at = now;
1786
+ jobWithPrefixes.run_after = now;
1787
+ for (const [key, value] of Object.entries(this.prefixValues)) {
1788
+ jobWithPrefixes[key] = value;
1789
+ }
1790
+ this.jobQueue.push(jobWithPrefixes);
1791
+ this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
1792
+ return jobWithPrefixes.id;
1793
+ }
1794
+ async get(id) {
1795
+ await sleep2(0);
1796
+ const job = this.jobQueue.find((j) => j.id === id);
1797
+ if (job && this.matchesPrefixes(job)) {
1798
+ return job;
1799
+ }
1800
+ return;
1801
+ }
1802
+ async peek(status = JobStatus.PENDING, num = 100) {
1803
+ await sleep2(0);
1804
+ num = Number(num) || 100;
1805
+ return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
1806
+ }
1807
+ async next(workerId) {
1808
+ await sleep2(0);
1809
+ const top = this.pendingQueue();
1810
+ const job = top[0];
1811
+ if (job) {
1812
+ const oldJob = { ...job };
1813
+ job.status = JobStatus.PROCESSING;
1814
+ job.last_ran_at = new Date().toISOString();
1815
+ job.worker_id = workerId;
1816
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1817
+ return job;
1818
+ }
1819
+ }
1820
+ async size(status = JobStatus.PENDING) {
1821
+ await sleep2(0);
1822
+ return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
1823
+ }
1824
+ async saveProgress(id, progress, message, details) {
1825
+ await sleep2(0);
1826
+ const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
1827
+ if (!job) {
1828
+ const jobWithAnyPrefix = this.jobQueue.find((j) => j.id === id);
1829
+ getLogger3().warn("Job not found for progress update", {
1830
+ id,
1831
+ reason: jobWithAnyPrefix ? "prefix_mismatch" : "missing",
1832
+ existingStatus: jobWithAnyPrefix?.status,
1833
+ queueName: this.queueName,
1834
+ prefixValues: this.prefixValues
1835
+ });
1836
+ return;
1837
+ }
1838
+ if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED) {
1839
+ getLogger3().warn("Job already completed or failed for progress update", {
1840
+ id,
1841
+ status: job.status,
1842
+ completedAt: job.completed_at,
1843
+ error: job.error
1844
+ });
1845
+ return;
1846
+ }
1847
+ const oldJob = { ...job };
1848
+ job.progress = progress;
1849
+ job.progress_message = message;
1850
+ job.progress_details = details;
1851
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1852
+ }
1853
+ async complete(job) {
1854
+ await sleep2(0);
1855
+ const jobWithPrefixes = job;
1856
+ const index = this.jobQueue.findIndex((j) => j.id === job.id && this.matchesPrefixes(j));
1857
+ if (index !== -1) {
1858
+ const existing = this.jobQueue[index];
1859
+ const currentAttempts = existing?.run_attempts ?? 0;
1860
+ jobWithPrefixes.run_attempts = currentAttempts + 1;
1861
+ for (const [key, value] of Object.entries(this.prefixValues)) {
1862
+ jobWithPrefixes[key] = value;
1863
+ }
1864
+ this.jobQueue[index] = jobWithPrefixes;
1865
+ this.events.emit("change", { type: "UPDATE", old: existing, new: jobWithPrefixes });
1866
+ }
1867
+ }
1868
+ async release(id) {
1869
+ await sleep2(0);
1870
+ const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
1871
+ if (job) {
1872
+ const oldJob = { ...job };
1873
+ job.status = JobStatus.PENDING;
1874
+ job.worker_id = null;
1875
+ job.progress = 0;
1876
+ job.progress_message = "";
1877
+ job.progress_details = null;
1878
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1879
+ }
1880
+ }
1881
+ async abort(id) {
1882
+ await sleep2(0);
1883
+ const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
1884
+ if (job) {
1885
+ const oldJob = { ...job };
1886
+ job.status = JobStatus.ABORTING;
1887
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1888
+ }
1889
+ }
1890
+ async getByRunId(runId) {
1891
+ await sleep2(0);
1892
+ return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
1893
+ }
1894
+ async deleteAll() {
1895
+ await sleep2(0);
1896
+ const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
1897
+ this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
1898
+ for (const job of deletedJobs) {
1899
+ this.events.emit("change", { type: "DELETE", old: job });
1900
+ }
1901
+ }
1902
+ async outputForInput(input) {
1903
+ await sleep2(0);
1904
+ const fingerprint = await makeFingerprint(input);
1905
+ return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === JobStatus.COMPLETED)?.output ?? null;
1906
+ }
1907
+ async delete(id) {
1908
+ await sleep2(0);
1909
+ const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
1910
+ this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
1911
+ if (deletedJob) {
1912
+ this.events.emit("change", { type: "DELETE", old: deletedJob });
1913
+ }
1914
+ }
1915
+ async deleteJobsByStatusAndAge(status, olderThanMs) {
1916
+ await sleep2(0);
1917
+ const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
1918
+ const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
1919
+ this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
1920
+ for (const job of deletedJobs) {
1921
+ this.events.emit("change", { type: "DELETE", old: job });
1922
+ }
1923
+ }
1924
+ async setupDatabase() {}
1925
+ matchesPrefixFilter(job, prefixFilter) {
1926
+ if (prefixFilter && Object.keys(prefixFilter).length === 0) {
1927
+ return true;
1928
+ }
1929
+ const filterValues = prefixFilter ?? this.prefixValues;
1930
+ if (Object.keys(filterValues).length === 0) {
1931
+ return true;
1932
+ }
1933
+ const jobWithPrefixes = job;
1934
+ for (const [key, value] of Object.entries(filterValues)) {
1935
+ if (jobWithPrefixes[key] !== value) {
1936
+ return false;
1937
+ }
1938
+ }
1939
+ return true;
1940
+ }
1941
+ subscribeToChanges(callback, options) {
1942
+ const prefixFilter = options?.prefixFilter;
1943
+ const filteredCallback = (change) => {
1944
+ const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
1945
+ const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
1946
+ if (!newMatches && !oldMatches) {
1947
+ return;
1948
+ }
1949
+ callback(change);
1950
+ };
1951
+ return this.events.subscribe("change", filteredCallback);
1952
+ }
1953
+ }
1954
+
1955
+ // src/queue-storage/TelemetryQueueStorage.ts
1956
+ import { traced } from "@workglow/util";
1957
+
1958
+ class TelemetryQueueStorage {
1959
+ storageName;
1960
+ inner;
1961
+ constructor(storageName, inner) {
1962
+ this.storageName = storageName;
1963
+ this.inner = inner;
1964
+ }
1965
+ get scope() {
1966
+ return this.inner.scope;
1967
+ }
1968
+ add(job) {
1969
+ return traced("workglow.storage.queue.add", this.storageName, () => this.inner.add(job));
1970
+ }
1971
+ get(id) {
1972
+ return traced("workglow.storage.queue.get", this.storageName, () => this.inner.get(id));
1973
+ }
1974
+ next(workerId) {
1975
+ return traced("workglow.storage.queue.next", this.storageName, () => this.inner.next(workerId));
1976
+ }
1977
+ peek(status, num) {
1978
+ return traced("workglow.storage.queue.peek", this.storageName, () => this.inner.peek(status, num));
1979
+ }
1980
+ size(status) {
1981
+ return traced("workglow.storage.queue.size", this.storageName, () => this.inner.size(status));
1982
+ }
1983
+ complete(job) {
1984
+ return traced("workglow.storage.queue.complete", this.storageName, () => this.inner.complete(job));
1985
+ }
1986
+ release(id) {
1987
+ return traced("workglow.storage.queue.release", this.storageName, () => this.inner.release(id));
1988
+ }
1989
+ deleteAll() {
1990
+ return traced("workglow.storage.queue.deleteAll", this.storageName, () => this.inner.deleteAll());
1991
+ }
1992
+ outputForInput(input) {
1993
+ return traced("workglow.storage.queue.outputForInput", this.storageName, () => this.inner.outputForInput(input));
1994
+ }
1995
+ abort(id) {
1996
+ return traced("workglow.storage.queue.abort", this.storageName, () => this.inner.abort(id));
1997
+ }
1998
+ getByRunId(runId) {
1999
+ return traced("workglow.storage.queue.getByRunId", this.storageName, () => this.inner.getByRunId(runId));
2000
+ }
2001
+ saveProgress(id, progress, message, details) {
2002
+ return traced("workglow.storage.queue.saveProgress", this.storageName, () => this.inner.saveProgress(id, progress, message, details));
2003
+ }
2004
+ delete(id) {
2005
+ return traced("workglow.storage.queue.delete", this.storageName, () => this.inner.delete(id));
2006
+ }
2007
+ deleteJobsByStatusAndAge(status, olderThanMs) {
2008
+ return traced("workglow.storage.queue.deleteJobsByStatusAndAge", this.storageName, () => this.inner.deleteJobsByStatusAndAge(status, olderThanMs));
2009
+ }
2010
+ setupDatabase() {
2011
+ return this.inner.setupDatabase();
2012
+ }
2013
+ subscribeToChanges(callback, options) {
2014
+ return this.inner.subscribeToChanges(callback, options);
2015
+ }
2016
+ }
2017
+
2018
+ // src/rate-limiter-storage/IRateLimiterStorage.ts
2019
+ import { createServiceToken as createServiceToken7 } from "@workglow/util";
2020
+ var RATE_LIMITER_STORAGE = createServiceToken7("ratelimiter.storage");
2021
+
2022
+ // src/rate-limiter-storage/InMemoryRateLimiterStorage.ts
2023
+ import { createServiceToken as createServiceToken8, sleep as sleep3, uuid4 as uuid43 } from "@workglow/util";
2024
+ var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage.inMemory");
2025
+
2026
+ class InMemoryRateLimiterStorage {
2027
+ scope = "process";
2028
+ prefixValues;
2029
+ executions = new Map;
2030
+ nextAvailableTimes = new Map;
2031
+ reserveChains = new Map;
2032
+ constructor(options) {
2033
+ this.prefixValues = options?.prefixValues ?? {};
2034
+ }
2035
+ makeKey(queueName) {
2036
+ const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
2037
+ return prefixPart ? `${prefixPart}|${queueName}` : queueName;
2038
+ }
2039
+ async setupDatabase() {}
2040
+ async withKeyLock(key, fn) {
2041
+ const previous = this.reserveChains.get(key) ?? Promise.resolve();
2042
+ let release;
2043
+ const next = new Promise((resolve) => {
2044
+ release = resolve;
2045
+ });
2046
+ this.reserveChains.set(key, next);
2047
+ try {
2048
+ await previous;
2049
+ return await fn();
2050
+ } finally {
2051
+ release(undefined);
2052
+ if (this.reserveChains.get(key) === next) {
2053
+ this.reserveChains.delete(key);
2054
+ }
2055
+ }
2056
+ }
2057
+ async tryReserveExecution(queueName, maxExecutions, windowMs) {
2058
+ const key = this.makeKey(queueName);
2059
+ return this.withKeyLock(key, () => {
2060
+ const now = Date.now();
2061
+ const windowStart = new Date(now - windowMs);
2062
+ const executions = this.executions.get(key) ?? [];
2063
+ const live = executions.filter((e) => e.executedAt > windowStart);
2064
+ if (live.length >= maxExecutions) {
2065
+ if (live.length !== executions.length) {
2066
+ this.executions.set(key, live);
2067
+ }
2068
+ return null;
2069
+ }
2070
+ const next = this.nextAvailableTimes.get(key);
2071
+ if (next && next.getTime() > now) {
2072
+ return null;
2073
+ }
2074
+ const id = uuid43();
2075
+ live.push({ id, queueName, executedAt: new Date(now) });
2076
+ this.executions.set(key, live);
2077
+ return id;
2078
+ });
2079
+ }
2080
+ async releaseExecution(queueName, token) {
2081
+ if (token === null || token === undefined)
2082
+ return;
2083
+ const key = this.makeKey(queueName);
2084
+ await this.withKeyLock(key, () => {
2085
+ const executions = this.executions.get(key);
2086
+ if (!executions || executions.length === 0)
2087
+ return;
2088
+ const idx = executions.findIndex((e) => e.id === token);
2089
+ if (idx === -1)
2090
+ return;
2091
+ executions.splice(idx, 1);
2092
+ this.executions.set(key, executions);
2093
+ });
2094
+ }
2095
+ async recordExecution(queueName) {
2096
+ await sleep3(0);
2097
+ const key = this.makeKey(queueName);
2098
+ const executions = this.executions.get(key) ?? [];
2099
+ executions.push({
2100
+ id: uuid43(),
2101
+ queueName,
2102
+ executedAt: new Date
2103
+ });
2104
+ this.executions.set(key, executions);
2105
+ }
2106
+ async getExecutionCount(queueName, windowStartTime) {
2107
+ await sleep3(0);
2108
+ const key = this.makeKey(queueName);
2109
+ const executions = this.executions.get(key) ?? [];
2110
+ const windowStart = new Date(windowStartTime);
2111
+ return executions.filter((e) => e.executedAt > windowStart).length;
2112
+ }
2113
+ async getOldestExecutionAtOffset(queueName, offset) {
2114
+ await sleep3(0);
2115
+ const key = this.makeKey(queueName);
2116
+ const executions = this.executions.get(key) ?? [];
2117
+ const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
2118
+ const execution = sorted[offset];
2119
+ return execution?.executedAt.toISOString();
2120
+ }
2121
+ async getNextAvailableTime(queueName) {
2122
+ await sleep3(0);
2123
+ const key = this.makeKey(queueName);
2124
+ const time = this.nextAvailableTimes.get(key);
2125
+ return time?.toISOString();
2126
+ }
2127
+ async setNextAvailableTime(queueName, nextAvailableAt) {
2128
+ await sleep3(0);
2129
+ const key = this.makeKey(queueName);
2130
+ this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
2131
+ }
2132
+ async clear(queueName) {
2133
+ await sleep3(0);
2134
+ const key = this.makeKey(queueName);
2135
+ this.executions.delete(key);
2136
+ this.nextAvailableTimes.delete(key);
2137
+ }
2138
+ }
1732
2139
  export {
1733
2140
  withJobErrorDiagnostics,
1734
2141
  storageToClass,
1735
2142
  formatErrorChainForDiagnostics,
1736
2143
  classToStorage,
1737
2144
  applyPersistedDiagnosticsToStack,
2145
+ TelemetryQueueStorage,
1738
2146
  RetryableJobError,
1739
2147
  RateLimiter,
2148
+ RATE_LIMITER_STORAGE,
2149
+ QUEUE_STORAGE,
1740
2150
  PermanentJobError,
1741
2151
  NullLimiter,
1742
2152
  NULL_JOB_LIMITER,
@@ -1750,6 +2160,10 @@ export {
1750
2160
  Job,
1751
2161
  JOB_LIMITER,
1752
2162
  JOB_ERROR_DIAGNOSTICS_MARKER,
2163
+ InMemoryRateLimiterStorage,
2164
+ InMemoryQueueStorage,
2165
+ IN_MEMORY_RATE_LIMITER_STORAGE,
2166
+ IN_MEMORY_QUEUE_STORAGE,
1753
2167
  EvenlySpacedRateLimiter,
1754
2168
  EVENLY_SPACED_JOB_RATE_LIMITER,
1755
2169
  DelayLimiter,
@@ -1759,4 +2173,4 @@ export {
1759
2173
  AbortSignalJobError
1760
2174
  };
1761
2175
 
1762
- //# debugId=E843281190707FDF64756E2164756E21
2176
+ //# debugId=F5B8A6C859447D5F64756E2164756E21