@workglow/job-queue 0.2.27 → 0.2.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +457 -43
- package/dist/browser.js.map +15 -10
- package/dist/bun.js +457 -43
- package/dist/bun.js.map +15 -10
- package/dist/common.d.ts +5 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/job/Job.d.ts +1 -1
- package/dist/job/Job.d.ts.map +1 -1
- package/dist/job/JobQueueClient.d.ts +2 -1
- package/dist/job/JobQueueClient.d.ts.map +1 -1
- package/dist/job/JobQueueServer.d.ts +1 -1
- package/dist/job/JobQueueServer.d.ts.map +1 -1
- package/dist/job/JobQueueWorker.d.ts +1 -1
- package/dist/job/JobQueueWorker.d.ts.map +1 -1
- package/dist/job/JobStorageConverters.d.ts +1 -1
- package/dist/job/JobStorageConverters.d.ts.map +1 -1
- package/dist/limiter/CompositeLimiter.d.ts.map +1 -1
- package/dist/limiter/RateLimiter.d.ts +1 -1
- package/dist/limiter/RateLimiter.d.ts.map +1 -1
- package/dist/node.js +457 -43
- package/dist/node.js.map +15 -10
- package/dist/queue-storage/IQueueStorage.d.ts +229 -0
- package/dist/queue-storage/IQueueStorage.d.ts.map +1 -0
- package/dist/queue-storage/InMemoryQueueStorage.d.ts +149 -0
- package/dist/queue-storage/InMemoryQueueStorage.d.ts.map +1 -0
- package/dist/queue-storage/TelemetryQueueStorage.d.ts +33 -0
- package/dist/queue-storage/TelemetryQueueStorage.d.ts.map +1 -0
- package/dist/rate-limiter-storage/IRateLimiterStorage.d.ts +127 -0
- package/dist/rate-limiter-storage/IRateLimiterStorage.d.ts.map +1 -0
- package/dist/rate-limiter-storage/InMemoryRateLimiterStorage.d.ts +43 -0
- package/dist/rate-limiter-storage/InMemoryRateLimiterStorage.d.ts.map +1 -0
- package/package.json +3 -8
package/dist/browser.js
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
|
-
// src/
|
|
2
|
-
import {
|
|
1
|
+
// src/queue-storage/IQueueStorage.ts
|
|
2
|
+
import { createServiceToken } from "@workglow/util";
|
|
3
|
+
var QUEUE_STORAGE = createServiceToken("jobqueue.storage");
|
|
4
|
+
var JobStatus = {
|
|
5
|
+
PENDING: "PENDING",
|
|
6
|
+
PROCESSING: "PROCESSING",
|
|
7
|
+
COMPLETED: "COMPLETED",
|
|
8
|
+
ABORTING: "ABORTING",
|
|
9
|
+
FAILED: "FAILED",
|
|
10
|
+
DISABLED: "DISABLED"
|
|
11
|
+
};
|
|
3
12
|
|
|
4
13
|
// src/job/JobError.ts
|
|
5
14
|
import { BaseError } from "@workglow/util";
|
|
@@ -175,7 +184,6 @@ ${fullMessage}`;
|
|
|
175
184
|
}
|
|
176
185
|
|
|
177
186
|
// src/job/JobQueueClient.ts
|
|
178
|
-
import { JobStatus as JobStatus2 } from "@workglow/storage";
|
|
179
187
|
import { EventEmitter } from "@workglow/util";
|
|
180
188
|
|
|
181
189
|
// src/job/JobStorageConverters.ts
|
|
@@ -300,7 +308,7 @@ class JobQueueClient {
|
|
|
300
308
|
run_after: options?.runAfter?.toISOString() ?? new Date().toISOString(),
|
|
301
309
|
deadline_at: options?.deadlineAt?.toISOString() ?? null,
|
|
302
310
|
completed_at: null,
|
|
303
|
-
status:
|
|
311
|
+
status: JobStatus.PENDING
|
|
304
312
|
};
|
|
305
313
|
const id = await this.storage.add(job);
|
|
306
314
|
this.server?.handleJobAdded(id);
|
|
@@ -353,15 +361,15 @@ class JobQueueClient {
|
|
|
353
361
|
this.removePromise(jobId, resolve, reject);
|
|
354
362
|
throw new JobNotFoundError(`Job ${jobId} not found`);
|
|
355
363
|
}
|
|
356
|
-
if (job.status ===
|
|
364
|
+
if (job.status === JobStatus.COMPLETED) {
|
|
357
365
|
this.removePromise(jobId, resolve, reject);
|
|
358
366
|
return job.output;
|
|
359
367
|
}
|
|
360
|
-
if (job.status ===
|
|
368
|
+
if (job.status === JobStatus.DISABLED) {
|
|
361
369
|
this.removePromise(jobId, resolve, reject);
|
|
362
370
|
throw new JobDisabledError(`Job ${jobId} was disabled`);
|
|
363
371
|
}
|
|
364
|
-
if (job.status ===
|
|
372
|
+
if (job.status === JobStatus.FAILED) {
|
|
365
373
|
this.removePromise(jobId, resolve, reject);
|
|
366
374
|
throw this.buildErrorFromJob(job);
|
|
367
375
|
}
|
|
@@ -396,7 +404,7 @@ class JobQueueClient {
|
|
|
396
404
|
throw new JobNotFoundError("Cannot abort job run with undefined jobRunId");
|
|
397
405
|
const jobs = await this.getJobsByRunId(jobRunId);
|
|
398
406
|
await Promise.allSettled(jobs.map((job) => {
|
|
399
|
-
if (job.status ===
|
|
407
|
+
if (job.status === JobStatus.PROCESSING || job.status === JobStatus.PENDING) {
|
|
400
408
|
return this.abort(job.id);
|
|
401
409
|
}
|
|
402
410
|
}));
|
|
@@ -503,15 +511,15 @@ class JobQueueClient {
|
|
|
503
511
|
if (change.type === "UPDATE" && change.new) {
|
|
504
512
|
const newStatus = change.new.status;
|
|
505
513
|
const oldStatus = change.old?.status;
|
|
506
|
-
if (newStatus ===
|
|
514
|
+
if (newStatus === JobStatus.PROCESSING && oldStatus === JobStatus.PENDING) {
|
|
507
515
|
this.handleJobStart(jobId);
|
|
508
|
-
} else if (newStatus ===
|
|
516
|
+
} else if (newStatus === JobStatus.COMPLETED) {
|
|
509
517
|
this.handleJobComplete(jobId, change.new.output);
|
|
510
|
-
} else if (newStatus ===
|
|
518
|
+
} else if (newStatus === JobStatus.FAILED) {
|
|
511
519
|
this.handleJobError(jobId, change.new.error ?? "Job failed", change.new.error_code ?? undefined);
|
|
512
|
-
} else if (newStatus ===
|
|
520
|
+
} else if (newStatus === JobStatus.DISABLED) {
|
|
513
521
|
this.handleJobDisabled(jobId);
|
|
514
|
-
} else if (newStatus ===
|
|
522
|
+
} else if (newStatus === JobStatus.PENDING && oldStatus === JobStatus.PROCESSING) {
|
|
515
523
|
const runAfter = change.new.run_after ? new Date(change.new.run_after) : new Date;
|
|
516
524
|
this.handleJobRetry(jobId, runAfter);
|
|
517
525
|
}
|
|
@@ -554,12 +562,11 @@ class JobQueueClient {
|
|
|
554
562
|
}
|
|
555
563
|
|
|
556
564
|
// src/job/JobQueueServer.ts
|
|
557
|
-
import { JobStatus as JobStatus4 } from "@workglow/storage";
|
|
558
565
|
import { EventEmitter as EventEmitter3, getLogger as getLogger2 } from "@workglow/util";
|
|
559
566
|
|
|
560
567
|
// src/limiter/NullLimiter.ts
|
|
561
|
-
import { createServiceToken } from "@workglow/util";
|
|
562
|
-
var NULL_JOB_LIMITER =
|
|
568
|
+
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
569
|
+
var NULL_JOB_LIMITER = createServiceToken2("jobqueue.limiter.null");
|
|
563
570
|
|
|
564
571
|
class NullLimiter {
|
|
565
572
|
scope = "process";
|
|
@@ -581,7 +588,6 @@ class NullLimiter {
|
|
|
581
588
|
}
|
|
582
589
|
|
|
583
590
|
// src/job/JobQueueWorker.ts
|
|
584
|
-
import { JobStatus as JobStatus3 } from "@workglow/storage";
|
|
585
591
|
import {
|
|
586
592
|
EventEmitter as EventEmitter2,
|
|
587
593
|
getLogger,
|
|
@@ -757,7 +763,7 @@ class JobQueueWorker {
|
|
|
757
763
|
}
|
|
758
764
|
async getIdleDelay() {
|
|
759
765
|
try {
|
|
760
|
-
const pending = await this.storage.peek(
|
|
766
|
+
const pending = await this.storage.peek(JobStatus.PENDING, 1);
|
|
761
767
|
if (pending.length > 0 && pending[0].run_after) {
|
|
762
768
|
const delay = new Date(pending[0].run_after).getTime() - Date.now();
|
|
763
769
|
if (delay > 0) {
|
|
@@ -789,7 +795,7 @@ class JobQueueWorker {
|
|
|
789
795
|
if (this.activeJobAbortControllers.size === 0) {
|
|
790
796
|
return;
|
|
791
797
|
}
|
|
792
|
-
const abortingJobs = await this.storage.peek(
|
|
798
|
+
const abortingJobs = await this.storage.peek(JobStatus.ABORTING);
|
|
793
799
|
for (const jobData of abortingJobs) {
|
|
794
800
|
const controller = this.activeJobAbortControllers.get(jobData.id);
|
|
795
801
|
if (controller && !controller.signal.aborted) {
|
|
@@ -885,7 +891,7 @@ class JobQueueWorker {
|
|
|
885
891
|
}
|
|
886
892
|
async completeJob(job, output) {
|
|
887
893
|
try {
|
|
888
|
-
job.status =
|
|
894
|
+
job.status = JobStatus.COMPLETED;
|
|
889
895
|
job.progress = 100;
|
|
890
896
|
job.progressMessage = "";
|
|
891
897
|
job.progressDetails = null;
|
|
@@ -903,7 +909,7 @@ class JobQueueWorker {
|
|
|
903
909
|
}
|
|
904
910
|
async failJob(job, error) {
|
|
905
911
|
try {
|
|
906
|
-
job.status =
|
|
912
|
+
job.status = JobStatus.FAILED;
|
|
907
913
|
job.progress = 100;
|
|
908
914
|
job.completedAt = new Date;
|
|
909
915
|
job.progressMessage = "";
|
|
@@ -920,7 +926,7 @@ class JobQueueWorker {
|
|
|
920
926
|
}
|
|
921
927
|
async disableJob(job) {
|
|
922
928
|
try {
|
|
923
|
-
job.status =
|
|
929
|
+
job.status = JobStatus.DISABLED;
|
|
924
930
|
job.progress = 100;
|
|
925
931
|
job.completedAt = new Date;
|
|
926
932
|
job.progressMessage = "";
|
|
@@ -942,7 +948,7 @@ class JobQueueWorker {
|
|
|
942
948
|
}
|
|
943
949
|
async rescheduleJob(job, retryDate) {
|
|
944
950
|
try {
|
|
945
|
-
job.status =
|
|
951
|
+
job.status = JobStatus.PENDING;
|
|
946
952
|
const nextAvailableTime = await this.limiter.getNextAvailableTime();
|
|
947
953
|
job.runAfter = retryDate instanceof Date ? retryDate : nextAvailableTime;
|
|
948
954
|
job.progress = 0;
|
|
@@ -978,7 +984,7 @@ class JobQueueWorker {
|
|
|
978
984
|
getLogger().error("handleAbort: job not found", { jobId });
|
|
979
985
|
return;
|
|
980
986
|
}
|
|
981
|
-
if (job.status ===
|
|
987
|
+
if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED || job.status === JobStatus.DISABLED) {
|
|
982
988
|
return;
|
|
983
989
|
}
|
|
984
990
|
await this.failJob(job, new AbortSignalJobError("Job Aborted"));
|
|
@@ -990,19 +996,19 @@ class JobQueueWorker {
|
|
|
990
996
|
return this.storageToClass(job);
|
|
991
997
|
}
|
|
992
998
|
async validateJobState(job) {
|
|
993
|
-
if (job.status ===
|
|
999
|
+
if (job.status === JobStatus.COMPLETED) {
|
|
994
1000
|
throw new PermanentJobError(`Job ${job.id} is already completed`);
|
|
995
1001
|
}
|
|
996
|
-
if (job.status ===
|
|
1002
|
+
if (job.status === JobStatus.FAILED) {
|
|
997
1003
|
throw new PermanentJobError(`Job ${job.id} has failed`);
|
|
998
1004
|
}
|
|
999
|
-
if (job.status ===
|
|
1005
|
+
if (job.status === JobStatus.ABORTING || this.activeJobAbortControllers.get(job.id)?.signal.aborted) {
|
|
1000
1006
|
throw new AbortSignalJobError(`Job ${job.id} is being aborted`);
|
|
1001
1007
|
}
|
|
1002
1008
|
if (job.deadlineAt && job.deadlineAt < new Date) {
|
|
1003
1009
|
throw new PermanentJobError(`Job ${job.id} has exceeded its deadline`);
|
|
1004
1010
|
}
|
|
1005
|
-
if (job.status ===
|
|
1011
|
+
if (job.status === JobStatus.DISABLED) {
|
|
1006
1012
|
throw new JobDisabledError(`Job ${job.id} has been disabled`);
|
|
1007
1013
|
}
|
|
1008
1014
|
}
|
|
@@ -1084,7 +1090,7 @@ class JobQueueServer {
|
|
|
1084
1090
|
await this.fixupJobs();
|
|
1085
1091
|
try {
|
|
1086
1092
|
this.storageUnsubscribe = this.storage.subscribeToChanges((change) => {
|
|
1087
|
-
if (change.type === "INSERT" || change.type === "RESYNC" || change.type === "UPDATE" && change.new?.status ===
|
|
1093
|
+
if (change.type === "INSERT" || change.type === "RESYNC" || change.type === "UPDATE" && change.new?.status === JobStatus.PENDING) {
|
|
1088
1094
|
this.notifyWorkers();
|
|
1089
1095
|
}
|
|
1090
1096
|
});
|
|
@@ -1279,13 +1285,13 @@ class JobQueueServer {
|
|
|
1279
1285
|
async cleanupJobs() {
|
|
1280
1286
|
try {
|
|
1281
1287
|
if (this.deleteAfterCompletionMs !== undefined && this.deleteAfterCompletionMs > 0) {
|
|
1282
|
-
await this.storage.deleteJobsByStatusAndAge(
|
|
1288
|
+
await this.storage.deleteJobsByStatusAndAge(JobStatus.COMPLETED, this.deleteAfterCompletionMs);
|
|
1283
1289
|
}
|
|
1284
1290
|
if (this.deleteAfterFailureMs !== undefined && this.deleteAfterFailureMs > 0) {
|
|
1285
|
-
await this.storage.deleteJobsByStatusAndAge(
|
|
1291
|
+
await this.storage.deleteJobsByStatusAndAge(JobStatus.FAILED, this.deleteAfterFailureMs);
|
|
1286
1292
|
}
|
|
1287
1293
|
if (this.deleteAfterDisabledMs !== undefined && this.deleteAfterDisabledMs > 0) {
|
|
1288
|
-
await this.storage.deleteJobsByStatusAndAge(
|
|
1294
|
+
await this.storage.deleteJobsByStatusAndAge(JobStatus.DISABLED, this.deleteAfterDisabledMs);
|
|
1289
1295
|
}
|
|
1290
1296
|
} catch (error) {
|
|
1291
1297
|
console.error("Error in cleanup:", error);
|
|
@@ -1293,8 +1299,8 @@ class JobQueueServer {
|
|
|
1293
1299
|
}
|
|
1294
1300
|
async fixupJobs() {
|
|
1295
1301
|
try {
|
|
1296
|
-
const stuckProcessingJobs = await this.storage.peek(
|
|
1297
|
-
const stuckAbortingJobs = await this.storage.peek(
|
|
1302
|
+
const stuckProcessingJobs = await this.storage.peek(JobStatus.PROCESSING);
|
|
1303
|
+
const stuckAbortingJobs = await this.storage.peek(JobStatus.ABORTING);
|
|
1298
1304
|
const stuckJobs = [...stuckProcessingJobs, ...stuckAbortingJobs];
|
|
1299
1305
|
const currentWorkerIds = new Set(this.getWorkerIds());
|
|
1300
1306
|
for (const jobData of stuckJobs) {
|
|
@@ -1303,12 +1309,12 @@ class JobQueueServer {
|
|
|
1303
1309
|
}
|
|
1304
1310
|
const job = this.storageToClass(jobData);
|
|
1305
1311
|
if (job.runAttempts >= job.maxRetries) {
|
|
1306
|
-
job.status =
|
|
1312
|
+
job.status = JobStatus.FAILED;
|
|
1307
1313
|
job.error = "Max retries reached";
|
|
1308
1314
|
job.errorCode = "MAX_RETRIES_REACHED";
|
|
1309
1315
|
job.workerId = null;
|
|
1310
1316
|
} else {
|
|
1311
|
-
job.status =
|
|
1317
|
+
job.status = JobStatus.PENDING;
|
|
1312
1318
|
job.runAfter = job.lastRanAt || new Date;
|
|
1313
1319
|
job.progress = 0;
|
|
1314
1320
|
job.progressMessage = "";
|
|
@@ -1401,8 +1407,8 @@ class CompositeLimiter {
|
|
|
1401
1407
|
}
|
|
1402
1408
|
|
|
1403
1409
|
// src/limiter/ConcurrencyLimiter.ts
|
|
1404
|
-
import { createServiceToken as
|
|
1405
|
-
var CONCURRENT_JOB_LIMITER =
|
|
1410
|
+
import { createServiceToken as createServiceToken3 } from "@workglow/util";
|
|
1411
|
+
var CONCURRENT_JOB_LIMITER = createServiceToken3("jobqueue.limiter.concurrent");
|
|
1406
1412
|
|
|
1407
1413
|
class ConcurrencyLimiter {
|
|
1408
1414
|
scope = "process";
|
|
@@ -1495,8 +1501,8 @@ class DelayLimiter {
|
|
|
1495
1501
|
}
|
|
1496
1502
|
|
|
1497
1503
|
// src/limiter/EvenlySpacedRateLimiter.ts
|
|
1498
|
-
import { createServiceToken as
|
|
1499
|
-
var EVENLY_SPACED_JOB_RATE_LIMITER =
|
|
1504
|
+
import { createServiceToken as createServiceToken4 } from "@workglow/util";
|
|
1505
|
+
var EVENLY_SPACED_JOB_RATE_LIMITER = createServiceToken4("jobqueue.limiter.rate.evenlyspaced");
|
|
1500
1506
|
|
|
1501
1507
|
class EvenlySpacedRateLimiter {
|
|
1502
1508
|
scope = "process";
|
|
@@ -1596,8 +1602,8 @@ class EvenlySpacedRateLimiter {
|
|
|
1596
1602
|
}
|
|
1597
1603
|
|
|
1598
1604
|
// src/limiter/ILimiter.ts
|
|
1599
|
-
import { createServiceToken as
|
|
1600
|
-
var JOB_LIMITER =
|
|
1605
|
+
import { createServiceToken as createServiceToken5 } from "@workglow/util";
|
|
1606
|
+
var JOB_LIMITER = createServiceToken5("jobqueue.limiter");
|
|
1601
1607
|
|
|
1602
1608
|
// src/limiter/RateLimiter.ts
|
|
1603
1609
|
class RateLimiter {
|
|
@@ -1729,14 +1735,418 @@ class RateLimiter {
|
|
|
1729
1735
|
this.localBackoffUntilMs = 0;
|
|
1730
1736
|
}
|
|
1731
1737
|
}
|
|
1738
|
+
|
|
1739
|
+
// src/queue-storage/InMemoryQueueStorage.ts
|
|
1740
|
+
import {
|
|
1741
|
+
createServiceToken as createServiceToken6,
|
|
1742
|
+
EventEmitter as EventEmitter4,
|
|
1743
|
+
getLogger as getLogger3,
|
|
1744
|
+
makeFingerprint,
|
|
1745
|
+
sleep as sleep2,
|
|
1746
|
+
uuid4 as uuid42
|
|
1747
|
+
} from "@workglow/util";
|
|
1748
|
+
var IN_MEMORY_QUEUE_STORAGE = createServiceToken6("jobqueue.storage.inMemory");
|
|
1749
|
+
|
|
1750
|
+
class InMemoryQueueStorage {
|
|
1751
|
+
queueName;
|
|
1752
|
+
scope = "process";
|
|
1753
|
+
prefixValues;
|
|
1754
|
+
events = new EventEmitter4;
|
|
1755
|
+
constructor(queueName, options) {
|
|
1756
|
+
this.queueName = queueName;
|
|
1757
|
+
this.jobQueue = [];
|
|
1758
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
1759
|
+
}
|
|
1760
|
+
jobQueue;
|
|
1761
|
+
matchesPrefixes(job) {
|
|
1762
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
1763
|
+
if (job[key] !== value) {
|
|
1764
|
+
return false;
|
|
1765
|
+
}
|
|
1766
|
+
}
|
|
1767
|
+
return true;
|
|
1768
|
+
}
|
|
1769
|
+
pendingQueue() {
|
|
1770
|
+
const now = new Date().toISOString();
|
|
1771
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === JobStatus.PENDING).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
1772
|
+
}
|
|
1773
|
+
async add(job) {
|
|
1774
|
+
await sleep2(0);
|
|
1775
|
+
const now = new Date().toISOString();
|
|
1776
|
+
const jobWithPrefixes = job;
|
|
1777
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid42();
|
|
1778
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid42();
|
|
1779
|
+
jobWithPrefixes.queue = this.queueName;
|
|
1780
|
+
jobWithPrefixes.fingerprint = await makeFingerprint(jobWithPrefixes.input);
|
|
1781
|
+
jobWithPrefixes.status = JobStatus.PENDING;
|
|
1782
|
+
jobWithPrefixes.progress = 0;
|
|
1783
|
+
jobWithPrefixes.progress_message = "";
|
|
1784
|
+
jobWithPrefixes.progress_details = null;
|
|
1785
|
+
jobWithPrefixes.created_at = now;
|
|
1786
|
+
jobWithPrefixes.run_after = now;
|
|
1787
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
1788
|
+
jobWithPrefixes[key] = value;
|
|
1789
|
+
}
|
|
1790
|
+
this.jobQueue.push(jobWithPrefixes);
|
|
1791
|
+
this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
|
|
1792
|
+
return jobWithPrefixes.id;
|
|
1793
|
+
}
|
|
1794
|
+
async get(id) {
|
|
1795
|
+
await sleep2(0);
|
|
1796
|
+
const job = this.jobQueue.find((j) => j.id === id);
|
|
1797
|
+
if (job && this.matchesPrefixes(job)) {
|
|
1798
|
+
return job;
|
|
1799
|
+
}
|
|
1800
|
+
return;
|
|
1801
|
+
}
|
|
1802
|
+
async peek(status = JobStatus.PENDING, num = 100) {
|
|
1803
|
+
await sleep2(0);
|
|
1804
|
+
num = Number(num) || 100;
|
|
1805
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
1806
|
+
}
|
|
1807
|
+
async next(workerId) {
|
|
1808
|
+
await sleep2(0);
|
|
1809
|
+
const top = this.pendingQueue();
|
|
1810
|
+
const job = top[0];
|
|
1811
|
+
if (job) {
|
|
1812
|
+
const oldJob = { ...job };
|
|
1813
|
+
job.status = JobStatus.PROCESSING;
|
|
1814
|
+
job.last_ran_at = new Date().toISOString();
|
|
1815
|
+
job.worker_id = workerId;
|
|
1816
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1817
|
+
return job;
|
|
1818
|
+
}
|
|
1819
|
+
}
|
|
1820
|
+
async size(status = JobStatus.PENDING) {
|
|
1821
|
+
await sleep2(0);
|
|
1822
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
|
|
1823
|
+
}
|
|
1824
|
+
async saveProgress(id, progress, message, details) {
|
|
1825
|
+
await sleep2(0);
|
|
1826
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
1827
|
+
if (!job) {
|
|
1828
|
+
const jobWithAnyPrefix = this.jobQueue.find((j) => j.id === id);
|
|
1829
|
+
getLogger3().warn("Job not found for progress update", {
|
|
1830
|
+
id,
|
|
1831
|
+
reason: jobWithAnyPrefix ? "prefix_mismatch" : "missing",
|
|
1832
|
+
existingStatus: jobWithAnyPrefix?.status,
|
|
1833
|
+
queueName: this.queueName,
|
|
1834
|
+
prefixValues: this.prefixValues
|
|
1835
|
+
});
|
|
1836
|
+
return;
|
|
1837
|
+
}
|
|
1838
|
+
if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED) {
|
|
1839
|
+
getLogger3().warn("Job already completed or failed for progress update", {
|
|
1840
|
+
id,
|
|
1841
|
+
status: job.status,
|
|
1842
|
+
completedAt: job.completed_at,
|
|
1843
|
+
error: job.error
|
|
1844
|
+
});
|
|
1845
|
+
return;
|
|
1846
|
+
}
|
|
1847
|
+
const oldJob = { ...job };
|
|
1848
|
+
job.progress = progress;
|
|
1849
|
+
job.progress_message = message;
|
|
1850
|
+
job.progress_details = details;
|
|
1851
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1852
|
+
}
|
|
1853
|
+
async complete(job) {
|
|
1854
|
+
await sleep2(0);
|
|
1855
|
+
const jobWithPrefixes = job;
|
|
1856
|
+
const index = this.jobQueue.findIndex((j) => j.id === job.id && this.matchesPrefixes(j));
|
|
1857
|
+
if (index !== -1) {
|
|
1858
|
+
const existing = this.jobQueue[index];
|
|
1859
|
+
const currentAttempts = existing?.run_attempts ?? 0;
|
|
1860
|
+
jobWithPrefixes.run_attempts = currentAttempts + 1;
|
|
1861
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
1862
|
+
jobWithPrefixes[key] = value;
|
|
1863
|
+
}
|
|
1864
|
+
this.jobQueue[index] = jobWithPrefixes;
|
|
1865
|
+
this.events.emit("change", { type: "UPDATE", old: existing, new: jobWithPrefixes });
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
async release(id) {
|
|
1869
|
+
await sleep2(0);
|
|
1870
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
1871
|
+
if (job) {
|
|
1872
|
+
const oldJob = { ...job };
|
|
1873
|
+
job.status = JobStatus.PENDING;
|
|
1874
|
+
job.worker_id = null;
|
|
1875
|
+
job.progress = 0;
|
|
1876
|
+
job.progress_message = "";
|
|
1877
|
+
job.progress_details = null;
|
|
1878
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
async abort(id) {
|
|
1882
|
+
await sleep2(0);
|
|
1883
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
1884
|
+
if (job) {
|
|
1885
|
+
const oldJob = { ...job };
|
|
1886
|
+
job.status = JobStatus.ABORTING;
|
|
1887
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
async getByRunId(runId) {
|
|
1891
|
+
await sleep2(0);
|
|
1892
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
|
|
1893
|
+
}
|
|
1894
|
+
async deleteAll() {
|
|
1895
|
+
await sleep2(0);
|
|
1896
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
|
|
1897
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
|
|
1898
|
+
for (const job of deletedJobs) {
|
|
1899
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
1900
|
+
}
|
|
1901
|
+
}
|
|
1902
|
+
async outputForInput(input) {
|
|
1903
|
+
await sleep2(0);
|
|
1904
|
+
const fingerprint = await makeFingerprint(input);
|
|
1905
|
+
return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === JobStatus.COMPLETED)?.output ?? null;
|
|
1906
|
+
}
|
|
1907
|
+
async delete(id) {
|
|
1908
|
+
await sleep2(0);
|
|
1909
|
+
const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
|
|
1910
|
+
this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
|
|
1911
|
+
if (deletedJob) {
|
|
1912
|
+
this.events.emit("change", { type: "DELETE", old: deletedJob });
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1915
|
+
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
1916
|
+
await sleep2(0);
|
|
1917
|
+
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
1918
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
|
|
1919
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
|
|
1920
|
+
for (const job of deletedJobs) {
|
|
1921
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
1922
|
+
}
|
|
1923
|
+
}
|
|
1924
|
+
async setupDatabase() {}
|
|
1925
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
1926
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
1927
|
+
return true;
|
|
1928
|
+
}
|
|
1929
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
1930
|
+
if (Object.keys(filterValues).length === 0) {
|
|
1931
|
+
return true;
|
|
1932
|
+
}
|
|
1933
|
+
const jobWithPrefixes = job;
|
|
1934
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
1935
|
+
if (jobWithPrefixes[key] !== value) {
|
|
1936
|
+
return false;
|
|
1937
|
+
}
|
|
1938
|
+
}
|
|
1939
|
+
return true;
|
|
1940
|
+
}
|
|
1941
|
+
subscribeToChanges(callback, options) {
|
|
1942
|
+
const prefixFilter = options?.prefixFilter;
|
|
1943
|
+
const filteredCallback = (change) => {
|
|
1944
|
+
const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
|
|
1945
|
+
const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
|
|
1946
|
+
if (!newMatches && !oldMatches) {
|
|
1947
|
+
return;
|
|
1948
|
+
}
|
|
1949
|
+
callback(change);
|
|
1950
|
+
};
|
|
1951
|
+
return this.events.subscribe("change", filteredCallback);
|
|
1952
|
+
}
|
|
1953
|
+
}
|
|
1954
|
+
|
|
1955
|
+
// src/queue-storage/TelemetryQueueStorage.ts
|
|
1956
|
+
import { traced } from "@workglow/util";
|
|
1957
|
+
|
|
1958
|
+
class TelemetryQueueStorage {
|
|
1959
|
+
storageName;
|
|
1960
|
+
inner;
|
|
1961
|
+
constructor(storageName, inner) {
|
|
1962
|
+
this.storageName = storageName;
|
|
1963
|
+
this.inner = inner;
|
|
1964
|
+
}
|
|
1965
|
+
get scope() {
|
|
1966
|
+
return this.inner.scope;
|
|
1967
|
+
}
|
|
1968
|
+
add(job) {
|
|
1969
|
+
return traced("workglow.storage.queue.add", this.storageName, () => this.inner.add(job));
|
|
1970
|
+
}
|
|
1971
|
+
get(id) {
|
|
1972
|
+
return traced("workglow.storage.queue.get", this.storageName, () => this.inner.get(id));
|
|
1973
|
+
}
|
|
1974
|
+
next(workerId) {
|
|
1975
|
+
return traced("workglow.storage.queue.next", this.storageName, () => this.inner.next(workerId));
|
|
1976
|
+
}
|
|
1977
|
+
peek(status, num) {
|
|
1978
|
+
return traced("workglow.storage.queue.peek", this.storageName, () => this.inner.peek(status, num));
|
|
1979
|
+
}
|
|
1980
|
+
size(status) {
|
|
1981
|
+
return traced("workglow.storage.queue.size", this.storageName, () => this.inner.size(status));
|
|
1982
|
+
}
|
|
1983
|
+
complete(job) {
|
|
1984
|
+
return traced("workglow.storage.queue.complete", this.storageName, () => this.inner.complete(job));
|
|
1985
|
+
}
|
|
1986
|
+
release(id) {
|
|
1987
|
+
return traced("workglow.storage.queue.release", this.storageName, () => this.inner.release(id));
|
|
1988
|
+
}
|
|
1989
|
+
deleteAll() {
|
|
1990
|
+
return traced("workglow.storage.queue.deleteAll", this.storageName, () => this.inner.deleteAll());
|
|
1991
|
+
}
|
|
1992
|
+
outputForInput(input) {
|
|
1993
|
+
return traced("workglow.storage.queue.outputForInput", this.storageName, () => this.inner.outputForInput(input));
|
|
1994
|
+
}
|
|
1995
|
+
abort(id) {
|
|
1996
|
+
return traced("workglow.storage.queue.abort", this.storageName, () => this.inner.abort(id));
|
|
1997
|
+
}
|
|
1998
|
+
getByRunId(runId) {
|
|
1999
|
+
return traced("workglow.storage.queue.getByRunId", this.storageName, () => this.inner.getByRunId(runId));
|
|
2000
|
+
}
|
|
2001
|
+
saveProgress(id, progress, message, details) {
|
|
2002
|
+
return traced("workglow.storage.queue.saveProgress", this.storageName, () => this.inner.saveProgress(id, progress, message, details));
|
|
2003
|
+
}
|
|
2004
|
+
delete(id) {
|
|
2005
|
+
return traced("workglow.storage.queue.delete", this.storageName, () => this.inner.delete(id));
|
|
2006
|
+
}
|
|
2007
|
+
deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2008
|
+
return traced("workglow.storage.queue.deleteJobsByStatusAndAge", this.storageName, () => this.inner.deleteJobsByStatusAndAge(status, olderThanMs));
|
|
2009
|
+
}
|
|
2010
|
+
setupDatabase() {
|
|
2011
|
+
return this.inner.setupDatabase();
|
|
2012
|
+
}
|
|
2013
|
+
subscribeToChanges(callback, options) {
|
|
2014
|
+
return this.inner.subscribeToChanges(callback, options);
|
|
2015
|
+
}
|
|
2016
|
+
}
|
|
2017
|
+
|
|
2018
|
+
// src/rate-limiter-storage/IRateLimiterStorage.ts
|
|
2019
|
+
import { createServiceToken as createServiceToken7 } from "@workglow/util";
|
|
2020
|
+
var RATE_LIMITER_STORAGE = createServiceToken7("ratelimiter.storage");
|
|
2021
|
+
|
|
2022
|
+
// src/rate-limiter-storage/InMemoryRateLimiterStorage.ts
|
|
2023
|
+
import { createServiceToken as createServiceToken8, sleep as sleep3, uuid4 as uuid43 } from "@workglow/util";
|
|
2024
|
+
var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage.inMemory");
|
|
2025
|
+
|
|
2026
|
+
class InMemoryRateLimiterStorage {
|
|
2027
|
+
scope = "process";
|
|
2028
|
+
prefixValues;
|
|
2029
|
+
executions = new Map;
|
|
2030
|
+
nextAvailableTimes = new Map;
|
|
2031
|
+
reserveChains = new Map;
|
|
2032
|
+
constructor(options) {
|
|
2033
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
2034
|
+
}
|
|
2035
|
+
makeKey(queueName) {
|
|
2036
|
+
const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
|
|
2037
|
+
return prefixPart ? `${prefixPart}|${queueName}` : queueName;
|
|
2038
|
+
}
|
|
2039
|
+
async setupDatabase() {}
|
|
2040
|
+
async withKeyLock(key, fn) {
|
|
2041
|
+
const previous = this.reserveChains.get(key) ?? Promise.resolve();
|
|
2042
|
+
let release;
|
|
2043
|
+
const next = new Promise((resolve) => {
|
|
2044
|
+
release = resolve;
|
|
2045
|
+
});
|
|
2046
|
+
this.reserveChains.set(key, next);
|
|
2047
|
+
try {
|
|
2048
|
+
await previous;
|
|
2049
|
+
return await fn();
|
|
2050
|
+
} finally {
|
|
2051
|
+
release(undefined);
|
|
2052
|
+
if (this.reserveChains.get(key) === next) {
|
|
2053
|
+
this.reserveChains.delete(key);
|
|
2054
|
+
}
|
|
2055
|
+
}
|
|
2056
|
+
}
|
|
2057
|
+
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
2058
|
+
const key = this.makeKey(queueName);
|
|
2059
|
+
return this.withKeyLock(key, () => {
|
|
2060
|
+
const now = Date.now();
|
|
2061
|
+
const windowStart = new Date(now - windowMs);
|
|
2062
|
+
const executions = this.executions.get(key) ?? [];
|
|
2063
|
+
const live = executions.filter((e) => e.executedAt > windowStart);
|
|
2064
|
+
if (live.length >= maxExecutions) {
|
|
2065
|
+
if (live.length !== executions.length) {
|
|
2066
|
+
this.executions.set(key, live);
|
|
2067
|
+
}
|
|
2068
|
+
return null;
|
|
2069
|
+
}
|
|
2070
|
+
const next = this.nextAvailableTimes.get(key);
|
|
2071
|
+
if (next && next.getTime() > now) {
|
|
2072
|
+
return null;
|
|
2073
|
+
}
|
|
2074
|
+
const id = uuid43();
|
|
2075
|
+
live.push({ id, queueName, executedAt: new Date(now) });
|
|
2076
|
+
this.executions.set(key, live);
|
|
2077
|
+
return id;
|
|
2078
|
+
});
|
|
2079
|
+
}
|
|
2080
|
+
async releaseExecution(queueName, token) {
|
|
2081
|
+
if (token === null || token === undefined)
|
|
2082
|
+
return;
|
|
2083
|
+
const key = this.makeKey(queueName);
|
|
2084
|
+
await this.withKeyLock(key, () => {
|
|
2085
|
+
const executions = this.executions.get(key);
|
|
2086
|
+
if (!executions || executions.length === 0)
|
|
2087
|
+
return;
|
|
2088
|
+
const idx = executions.findIndex((e) => e.id === token);
|
|
2089
|
+
if (idx === -1)
|
|
2090
|
+
return;
|
|
2091
|
+
executions.splice(idx, 1);
|
|
2092
|
+
this.executions.set(key, executions);
|
|
2093
|
+
});
|
|
2094
|
+
}
|
|
2095
|
+
async recordExecution(queueName) {
|
|
2096
|
+
await sleep3(0);
|
|
2097
|
+
const key = this.makeKey(queueName);
|
|
2098
|
+
const executions = this.executions.get(key) ?? [];
|
|
2099
|
+
executions.push({
|
|
2100
|
+
id: uuid43(),
|
|
2101
|
+
queueName,
|
|
2102
|
+
executedAt: new Date
|
|
2103
|
+
});
|
|
2104
|
+
this.executions.set(key, executions);
|
|
2105
|
+
}
|
|
2106
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
2107
|
+
await sleep3(0);
|
|
2108
|
+
const key = this.makeKey(queueName);
|
|
2109
|
+
const executions = this.executions.get(key) ?? [];
|
|
2110
|
+
const windowStart = new Date(windowStartTime);
|
|
2111
|
+
return executions.filter((e) => e.executedAt > windowStart).length;
|
|
2112
|
+
}
|
|
2113
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
2114
|
+
await sleep3(0);
|
|
2115
|
+
const key = this.makeKey(queueName);
|
|
2116
|
+
const executions = this.executions.get(key) ?? [];
|
|
2117
|
+
const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
|
|
2118
|
+
const execution = sorted[offset];
|
|
2119
|
+
return execution?.executedAt.toISOString();
|
|
2120
|
+
}
|
|
2121
|
+
async getNextAvailableTime(queueName) {
|
|
2122
|
+
await sleep3(0);
|
|
2123
|
+
const key = this.makeKey(queueName);
|
|
2124
|
+
const time = this.nextAvailableTimes.get(key);
|
|
2125
|
+
return time?.toISOString();
|
|
2126
|
+
}
|
|
2127
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
2128
|
+
await sleep3(0);
|
|
2129
|
+
const key = this.makeKey(queueName);
|
|
2130
|
+
this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
|
|
2131
|
+
}
|
|
2132
|
+
async clear(queueName) {
|
|
2133
|
+
await sleep3(0);
|
|
2134
|
+
const key = this.makeKey(queueName);
|
|
2135
|
+
this.executions.delete(key);
|
|
2136
|
+
this.nextAvailableTimes.delete(key);
|
|
2137
|
+
}
|
|
2138
|
+
}
|
|
1732
2139
|
export {
|
|
1733
2140
|
withJobErrorDiagnostics,
|
|
1734
2141
|
storageToClass,
|
|
1735
2142
|
formatErrorChainForDiagnostics,
|
|
1736
2143
|
classToStorage,
|
|
1737
2144
|
applyPersistedDiagnosticsToStack,
|
|
2145
|
+
TelemetryQueueStorage,
|
|
1738
2146
|
RetryableJobError,
|
|
1739
2147
|
RateLimiter,
|
|
2148
|
+
RATE_LIMITER_STORAGE,
|
|
2149
|
+
QUEUE_STORAGE,
|
|
1740
2150
|
PermanentJobError,
|
|
1741
2151
|
NullLimiter,
|
|
1742
2152
|
NULL_JOB_LIMITER,
|
|
@@ -1750,6 +2160,10 @@ export {
|
|
|
1750
2160
|
Job,
|
|
1751
2161
|
JOB_LIMITER,
|
|
1752
2162
|
JOB_ERROR_DIAGNOSTICS_MARKER,
|
|
2163
|
+
InMemoryRateLimiterStorage,
|
|
2164
|
+
InMemoryQueueStorage,
|
|
2165
|
+
IN_MEMORY_RATE_LIMITER_STORAGE,
|
|
2166
|
+
IN_MEMORY_QUEUE_STORAGE,
|
|
1753
2167
|
EvenlySpacedRateLimiter,
|
|
1754
2168
|
EVENLY_SPACED_JOB_RATE_LIMITER,
|
|
1755
2169
|
DelayLimiter,
|
|
@@ -1759,4 +2173,4 @@ export {
|
|
|
1759
2173
|
AbortSignalJobError
|
|
1760
2174
|
};
|
|
1761
2175
|
|
|
1762
|
-
//# debugId=
|
|
2176
|
+
//# debugId=E30CFA4D7AC3B9B064756E2164756E21
|