@workglow/job-queue 0.2.27 → 0.2.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +457 -43
- package/dist/browser.js.map +15 -10
- package/dist/bun.js +457 -43
- package/dist/bun.js.map +15 -10
- package/dist/common.d.ts +5 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/job/Job.d.ts +1 -1
- package/dist/job/Job.d.ts.map +1 -1
- package/dist/job/JobQueueClient.d.ts +2 -1
- package/dist/job/JobQueueClient.d.ts.map +1 -1
- package/dist/job/JobQueueServer.d.ts +1 -1
- package/dist/job/JobQueueServer.d.ts.map +1 -1
- package/dist/job/JobQueueWorker.d.ts +1 -1
- package/dist/job/JobQueueWorker.d.ts.map +1 -1
- package/dist/job/JobStorageConverters.d.ts +1 -1
- package/dist/job/JobStorageConverters.d.ts.map +1 -1
- package/dist/limiter/CompositeLimiter.d.ts.map +1 -1
- package/dist/limiter/RateLimiter.d.ts +1 -1
- package/dist/limiter/RateLimiter.d.ts.map +1 -1
- package/dist/node.js +457 -43
- package/dist/node.js.map +15 -10
- package/dist/queue-storage/IQueueStorage.d.ts +229 -0
- package/dist/queue-storage/IQueueStorage.d.ts.map +1 -0
- package/dist/queue-storage/InMemoryQueueStorage.d.ts +149 -0
- package/dist/queue-storage/InMemoryQueueStorage.d.ts.map +1 -0
- package/dist/queue-storage/TelemetryQueueStorage.d.ts +33 -0
- package/dist/queue-storage/TelemetryQueueStorage.d.ts.map +1 -0
- package/dist/rate-limiter-storage/IRateLimiterStorage.d.ts +127 -0
- package/dist/rate-limiter-storage/IRateLimiterStorage.d.ts.map +1 -0
- package/dist/rate-limiter-storage/InMemoryRateLimiterStorage.d.ts +43 -0
- package/dist/rate-limiter-storage/InMemoryRateLimiterStorage.d.ts.map +1 -0
- package/package.json +3 -8
package/dist/bun.js
CHANGED
|
@@ -1,6 +1,15 @@
|
|
|
1
1
|
// @bun
|
|
2
|
-
// src/
|
|
3
|
-
import {
|
|
2
|
+
// src/queue-storage/IQueueStorage.ts
|
|
3
|
+
import { createServiceToken } from "@workglow/util";
|
|
4
|
+
var QUEUE_STORAGE = createServiceToken("jobqueue.storage");
|
|
5
|
+
var JobStatus = {
|
|
6
|
+
PENDING: "PENDING",
|
|
7
|
+
PROCESSING: "PROCESSING",
|
|
8
|
+
COMPLETED: "COMPLETED",
|
|
9
|
+
ABORTING: "ABORTING",
|
|
10
|
+
FAILED: "FAILED",
|
|
11
|
+
DISABLED: "DISABLED"
|
|
12
|
+
};
|
|
4
13
|
|
|
5
14
|
// src/job/JobError.ts
|
|
6
15
|
import { BaseError } from "@workglow/util";
|
|
@@ -176,7 +185,6 @@ ${fullMessage}`;
|
|
|
176
185
|
}
|
|
177
186
|
|
|
178
187
|
// src/job/JobQueueClient.ts
|
|
179
|
-
import { JobStatus as JobStatus2 } from "@workglow/storage";
|
|
180
188
|
import { EventEmitter } from "@workglow/util";
|
|
181
189
|
|
|
182
190
|
// src/job/JobStorageConverters.ts
|
|
@@ -301,7 +309,7 @@ class JobQueueClient {
|
|
|
301
309
|
run_after: options?.runAfter?.toISOString() ?? new Date().toISOString(),
|
|
302
310
|
deadline_at: options?.deadlineAt?.toISOString() ?? null,
|
|
303
311
|
completed_at: null,
|
|
304
|
-
status:
|
|
312
|
+
status: JobStatus.PENDING
|
|
305
313
|
};
|
|
306
314
|
const id = await this.storage.add(job);
|
|
307
315
|
this.server?.handleJobAdded(id);
|
|
@@ -354,15 +362,15 @@ class JobQueueClient {
|
|
|
354
362
|
this.removePromise(jobId, resolve, reject);
|
|
355
363
|
throw new JobNotFoundError(`Job ${jobId} not found`);
|
|
356
364
|
}
|
|
357
|
-
if (job.status ===
|
|
365
|
+
if (job.status === JobStatus.COMPLETED) {
|
|
358
366
|
this.removePromise(jobId, resolve, reject);
|
|
359
367
|
return job.output;
|
|
360
368
|
}
|
|
361
|
-
if (job.status ===
|
|
369
|
+
if (job.status === JobStatus.DISABLED) {
|
|
362
370
|
this.removePromise(jobId, resolve, reject);
|
|
363
371
|
throw new JobDisabledError(`Job ${jobId} was disabled`);
|
|
364
372
|
}
|
|
365
|
-
if (job.status ===
|
|
373
|
+
if (job.status === JobStatus.FAILED) {
|
|
366
374
|
this.removePromise(jobId, resolve, reject);
|
|
367
375
|
throw this.buildErrorFromJob(job);
|
|
368
376
|
}
|
|
@@ -397,7 +405,7 @@ class JobQueueClient {
|
|
|
397
405
|
throw new JobNotFoundError("Cannot abort job run with undefined jobRunId");
|
|
398
406
|
const jobs = await this.getJobsByRunId(jobRunId);
|
|
399
407
|
await Promise.allSettled(jobs.map((job) => {
|
|
400
|
-
if (job.status ===
|
|
408
|
+
if (job.status === JobStatus.PROCESSING || job.status === JobStatus.PENDING) {
|
|
401
409
|
return this.abort(job.id);
|
|
402
410
|
}
|
|
403
411
|
}));
|
|
@@ -504,15 +512,15 @@ class JobQueueClient {
|
|
|
504
512
|
if (change.type === "UPDATE" && change.new) {
|
|
505
513
|
const newStatus = change.new.status;
|
|
506
514
|
const oldStatus = change.old?.status;
|
|
507
|
-
if (newStatus ===
|
|
515
|
+
if (newStatus === JobStatus.PROCESSING && oldStatus === JobStatus.PENDING) {
|
|
508
516
|
this.handleJobStart(jobId);
|
|
509
|
-
} else if (newStatus ===
|
|
517
|
+
} else if (newStatus === JobStatus.COMPLETED) {
|
|
510
518
|
this.handleJobComplete(jobId, change.new.output);
|
|
511
|
-
} else if (newStatus ===
|
|
519
|
+
} else if (newStatus === JobStatus.FAILED) {
|
|
512
520
|
this.handleJobError(jobId, change.new.error ?? "Job failed", change.new.error_code ?? undefined);
|
|
513
|
-
} else if (newStatus ===
|
|
521
|
+
} else if (newStatus === JobStatus.DISABLED) {
|
|
514
522
|
this.handleJobDisabled(jobId);
|
|
515
|
-
} else if (newStatus ===
|
|
523
|
+
} else if (newStatus === JobStatus.PENDING && oldStatus === JobStatus.PROCESSING) {
|
|
516
524
|
const runAfter = change.new.run_after ? new Date(change.new.run_after) : new Date;
|
|
517
525
|
this.handleJobRetry(jobId, runAfter);
|
|
518
526
|
}
|
|
@@ -555,12 +563,11 @@ class JobQueueClient {
|
|
|
555
563
|
}
|
|
556
564
|
|
|
557
565
|
// src/job/JobQueueServer.ts
|
|
558
|
-
import { JobStatus as JobStatus4 } from "@workglow/storage";
|
|
559
566
|
import { EventEmitter as EventEmitter3, getLogger as getLogger2 } from "@workglow/util";
|
|
560
567
|
|
|
561
568
|
// src/limiter/NullLimiter.ts
|
|
562
|
-
import { createServiceToken } from "@workglow/util";
|
|
563
|
-
var NULL_JOB_LIMITER =
|
|
569
|
+
import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
570
|
+
var NULL_JOB_LIMITER = createServiceToken2("jobqueue.limiter.null");
|
|
564
571
|
|
|
565
572
|
class NullLimiter {
|
|
566
573
|
scope = "process";
|
|
@@ -582,7 +589,6 @@ class NullLimiter {
|
|
|
582
589
|
}
|
|
583
590
|
|
|
584
591
|
// src/job/JobQueueWorker.ts
|
|
585
|
-
import { JobStatus as JobStatus3 } from "@workglow/storage";
|
|
586
592
|
import {
|
|
587
593
|
EventEmitter as EventEmitter2,
|
|
588
594
|
getLogger,
|
|
@@ -758,7 +764,7 @@ class JobQueueWorker {
|
|
|
758
764
|
}
|
|
759
765
|
async getIdleDelay() {
|
|
760
766
|
try {
|
|
761
|
-
const pending = await this.storage.peek(
|
|
767
|
+
const pending = await this.storage.peek(JobStatus.PENDING, 1);
|
|
762
768
|
if (pending.length > 0 && pending[0].run_after) {
|
|
763
769
|
const delay = new Date(pending[0].run_after).getTime() - Date.now();
|
|
764
770
|
if (delay > 0) {
|
|
@@ -790,7 +796,7 @@ class JobQueueWorker {
|
|
|
790
796
|
if (this.activeJobAbortControllers.size === 0) {
|
|
791
797
|
return;
|
|
792
798
|
}
|
|
793
|
-
const abortingJobs = await this.storage.peek(
|
|
799
|
+
const abortingJobs = await this.storage.peek(JobStatus.ABORTING);
|
|
794
800
|
for (const jobData of abortingJobs) {
|
|
795
801
|
const controller = this.activeJobAbortControllers.get(jobData.id);
|
|
796
802
|
if (controller && !controller.signal.aborted) {
|
|
@@ -886,7 +892,7 @@ class JobQueueWorker {
|
|
|
886
892
|
}
|
|
887
893
|
async completeJob(job, output) {
|
|
888
894
|
try {
|
|
889
|
-
job.status =
|
|
895
|
+
job.status = JobStatus.COMPLETED;
|
|
890
896
|
job.progress = 100;
|
|
891
897
|
job.progressMessage = "";
|
|
892
898
|
job.progressDetails = null;
|
|
@@ -904,7 +910,7 @@ class JobQueueWorker {
|
|
|
904
910
|
}
|
|
905
911
|
async failJob(job, error) {
|
|
906
912
|
try {
|
|
907
|
-
job.status =
|
|
913
|
+
job.status = JobStatus.FAILED;
|
|
908
914
|
job.progress = 100;
|
|
909
915
|
job.completedAt = new Date;
|
|
910
916
|
job.progressMessage = "";
|
|
@@ -921,7 +927,7 @@ class JobQueueWorker {
|
|
|
921
927
|
}
|
|
922
928
|
async disableJob(job) {
|
|
923
929
|
try {
|
|
924
|
-
job.status =
|
|
930
|
+
job.status = JobStatus.DISABLED;
|
|
925
931
|
job.progress = 100;
|
|
926
932
|
job.completedAt = new Date;
|
|
927
933
|
job.progressMessage = "";
|
|
@@ -943,7 +949,7 @@ class JobQueueWorker {
|
|
|
943
949
|
}
|
|
944
950
|
async rescheduleJob(job, retryDate) {
|
|
945
951
|
try {
|
|
946
|
-
job.status =
|
|
952
|
+
job.status = JobStatus.PENDING;
|
|
947
953
|
const nextAvailableTime = await this.limiter.getNextAvailableTime();
|
|
948
954
|
job.runAfter = retryDate instanceof Date ? retryDate : nextAvailableTime;
|
|
949
955
|
job.progress = 0;
|
|
@@ -979,7 +985,7 @@ class JobQueueWorker {
|
|
|
979
985
|
getLogger().error("handleAbort: job not found", { jobId });
|
|
980
986
|
return;
|
|
981
987
|
}
|
|
982
|
-
if (job.status ===
|
|
988
|
+
if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED || job.status === JobStatus.DISABLED) {
|
|
983
989
|
return;
|
|
984
990
|
}
|
|
985
991
|
await this.failJob(job, new AbortSignalJobError("Job Aborted"));
|
|
@@ -991,19 +997,19 @@ class JobQueueWorker {
|
|
|
991
997
|
return this.storageToClass(job);
|
|
992
998
|
}
|
|
993
999
|
async validateJobState(job) {
|
|
994
|
-
if (job.status ===
|
|
1000
|
+
if (job.status === JobStatus.COMPLETED) {
|
|
995
1001
|
throw new PermanentJobError(`Job ${job.id} is already completed`);
|
|
996
1002
|
}
|
|
997
|
-
if (job.status ===
|
|
1003
|
+
if (job.status === JobStatus.FAILED) {
|
|
998
1004
|
throw new PermanentJobError(`Job ${job.id} has failed`);
|
|
999
1005
|
}
|
|
1000
|
-
if (job.status ===
|
|
1006
|
+
if (job.status === JobStatus.ABORTING || this.activeJobAbortControllers.get(job.id)?.signal.aborted) {
|
|
1001
1007
|
throw new AbortSignalJobError(`Job ${job.id} is being aborted`);
|
|
1002
1008
|
}
|
|
1003
1009
|
if (job.deadlineAt && job.deadlineAt < new Date) {
|
|
1004
1010
|
throw new PermanentJobError(`Job ${job.id} has exceeded its deadline`);
|
|
1005
1011
|
}
|
|
1006
|
-
if (job.status ===
|
|
1012
|
+
if (job.status === JobStatus.DISABLED) {
|
|
1007
1013
|
throw new JobDisabledError(`Job ${job.id} has been disabled`);
|
|
1008
1014
|
}
|
|
1009
1015
|
}
|
|
@@ -1085,7 +1091,7 @@ class JobQueueServer {
|
|
|
1085
1091
|
await this.fixupJobs();
|
|
1086
1092
|
try {
|
|
1087
1093
|
this.storageUnsubscribe = this.storage.subscribeToChanges((change) => {
|
|
1088
|
-
if (change.type === "INSERT" || change.type === "RESYNC" || change.type === "UPDATE" && change.new?.status ===
|
|
1094
|
+
if (change.type === "INSERT" || change.type === "RESYNC" || change.type === "UPDATE" && change.new?.status === JobStatus.PENDING) {
|
|
1089
1095
|
this.notifyWorkers();
|
|
1090
1096
|
}
|
|
1091
1097
|
});
|
|
@@ -1280,13 +1286,13 @@ class JobQueueServer {
|
|
|
1280
1286
|
async cleanupJobs() {
|
|
1281
1287
|
try {
|
|
1282
1288
|
if (this.deleteAfterCompletionMs !== undefined && this.deleteAfterCompletionMs > 0) {
|
|
1283
|
-
await this.storage.deleteJobsByStatusAndAge(
|
|
1289
|
+
await this.storage.deleteJobsByStatusAndAge(JobStatus.COMPLETED, this.deleteAfterCompletionMs);
|
|
1284
1290
|
}
|
|
1285
1291
|
if (this.deleteAfterFailureMs !== undefined && this.deleteAfterFailureMs > 0) {
|
|
1286
|
-
await this.storage.deleteJobsByStatusAndAge(
|
|
1292
|
+
await this.storage.deleteJobsByStatusAndAge(JobStatus.FAILED, this.deleteAfterFailureMs);
|
|
1287
1293
|
}
|
|
1288
1294
|
if (this.deleteAfterDisabledMs !== undefined && this.deleteAfterDisabledMs > 0) {
|
|
1289
|
-
await this.storage.deleteJobsByStatusAndAge(
|
|
1295
|
+
await this.storage.deleteJobsByStatusAndAge(JobStatus.DISABLED, this.deleteAfterDisabledMs);
|
|
1290
1296
|
}
|
|
1291
1297
|
} catch (error) {
|
|
1292
1298
|
console.error("Error in cleanup:", error);
|
|
@@ -1294,8 +1300,8 @@ class JobQueueServer {
|
|
|
1294
1300
|
}
|
|
1295
1301
|
async fixupJobs() {
|
|
1296
1302
|
try {
|
|
1297
|
-
const stuckProcessingJobs = await this.storage.peek(
|
|
1298
|
-
const stuckAbortingJobs = await this.storage.peek(
|
|
1303
|
+
const stuckProcessingJobs = await this.storage.peek(JobStatus.PROCESSING);
|
|
1304
|
+
const stuckAbortingJobs = await this.storage.peek(JobStatus.ABORTING);
|
|
1299
1305
|
const stuckJobs = [...stuckProcessingJobs, ...stuckAbortingJobs];
|
|
1300
1306
|
const currentWorkerIds = new Set(this.getWorkerIds());
|
|
1301
1307
|
for (const jobData of stuckJobs) {
|
|
@@ -1304,12 +1310,12 @@ class JobQueueServer {
|
|
|
1304
1310
|
}
|
|
1305
1311
|
const job = this.storageToClass(jobData);
|
|
1306
1312
|
if (job.runAttempts >= job.maxRetries) {
|
|
1307
|
-
job.status =
|
|
1313
|
+
job.status = JobStatus.FAILED;
|
|
1308
1314
|
job.error = "Max retries reached";
|
|
1309
1315
|
job.errorCode = "MAX_RETRIES_REACHED";
|
|
1310
1316
|
job.workerId = null;
|
|
1311
1317
|
} else {
|
|
1312
|
-
job.status =
|
|
1318
|
+
job.status = JobStatus.PENDING;
|
|
1313
1319
|
job.runAfter = job.lastRanAt || new Date;
|
|
1314
1320
|
job.progress = 0;
|
|
1315
1321
|
job.progressMessage = "";
|
|
@@ -1402,8 +1408,8 @@ class CompositeLimiter {
|
|
|
1402
1408
|
}
|
|
1403
1409
|
|
|
1404
1410
|
// src/limiter/ConcurrencyLimiter.ts
|
|
1405
|
-
import { createServiceToken as
|
|
1406
|
-
var CONCURRENT_JOB_LIMITER =
|
|
1411
|
+
import { createServiceToken as createServiceToken3 } from "@workglow/util";
|
|
1412
|
+
var CONCURRENT_JOB_LIMITER = createServiceToken3("jobqueue.limiter.concurrent");
|
|
1407
1413
|
|
|
1408
1414
|
class ConcurrencyLimiter {
|
|
1409
1415
|
scope = "process";
|
|
@@ -1496,8 +1502,8 @@ class DelayLimiter {
|
|
|
1496
1502
|
}
|
|
1497
1503
|
|
|
1498
1504
|
// src/limiter/EvenlySpacedRateLimiter.ts
|
|
1499
|
-
import { createServiceToken as
|
|
1500
|
-
var EVENLY_SPACED_JOB_RATE_LIMITER =
|
|
1505
|
+
import { createServiceToken as createServiceToken4 } from "@workglow/util";
|
|
1506
|
+
var EVENLY_SPACED_JOB_RATE_LIMITER = createServiceToken4("jobqueue.limiter.rate.evenlyspaced");
|
|
1501
1507
|
|
|
1502
1508
|
class EvenlySpacedRateLimiter {
|
|
1503
1509
|
scope = "process";
|
|
@@ -1597,8 +1603,8 @@ class EvenlySpacedRateLimiter {
|
|
|
1597
1603
|
}
|
|
1598
1604
|
|
|
1599
1605
|
// src/limiter/ILimiter.ts
|
|
1600
|
-
import { createServiceToken as
|
|
1601
|
-
var JOB_LIMITER =
|
|
1606
|
+
import { createServiceToken as createServiceToken5 } from "@workglow/util";
|
|
1607
|
+
var JOB_LIMITER = createServiceToken5("jobqueue.limiter");
|
|
1602
1608
|
|
|
1603
1609
|
// src/limiter/RateLimiter.ts
|
|
1604
1610
|
class RateLimiter {
|
|
@@ -1730,14 +1736,418 @@ class RateLimiter {
|
|
|
1730
1736
|
this.localBackoffUntilMs = 0;
|
|
1731
1737
|
}
|
|
1732
1738
|
}
|
|
1739
|
+
|
|
1740
|
+
// src/queue-storage/InMemoryQueueStorage.ts
|
|
1741
|
+
import {
|
|
1742
|
+
createServiceToken as createServiceToken6,
|
|
1743
|
+
EventEmitter as EventEmitter4,
|
|
1744
|
+
getLogger as getLogger3,
|
|
1745
|
+
makeFingerprint,
|
|
1746
|
+
sleep as sleep2,
|
|
1747
|
+
uuid4 as uuid42
|
|
1748
|
+
} from "@workglow/util";
|
|
1749
|
+
var IN_MEMORY_QUEUE_STORAGE = createServiceToken6("jobqueue.storage.inMemory");
|
|
1750
|
+
|
|
1751
|
+
class InMemoryQueueStorage {
|
|
1752
|
+
queueName;
|
|
1753
|
+
scope = "process";
|
|
1754
|
+
prefixValues;
|
|
1755
|
+
events = new EventEmitter4;
|
|
1756
|
+
constructor(queueName, options) {
|
|
1757
|
+
this.queueName = queueName;
|
|
1758
|
+
this.jobQueue = [];
|
|
1759
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
1760
|
+
}
|
|
1761
|
+
jobQueue;
|
|
1762
|
+
matchesPrefixes(job) {
|
|
1763
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
1764
|
+
if (job[key] !== value) {
|
|
1765
|
+
return false;
|
|
1766
|
+
}
|
|
1767
|
+
}
|
|
1768
|
+
return true;
|
|
1769
|
+
}
|
|
1770
|
+
pendingQueue() {
|
|
1771
|
+
const now = new Date().toISOString();
|
|
1772
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === JobStatus.PENDING).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
1773
|
+
}
|
|
1774
|
+
async add(job) {
|
|
1775
|
+
await sleep2(0);
|
|
1776
|
+
const now = new Date().toISOString();
|
|
1777
|
+
const jobWithPrefixes = job;
|
|
1778
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid42();
|
|
1779
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid42();
|
|
1780
|
+
jobWithPrefixes.queue = this.queueName;
|
|
1781
|
+
jobWithPrefixes.fingerprint = await makeFingerprint(jobWithPrefixes.input);
|
|
1782
|
+
jobWithPrefixes.status = JobStatus.PENDING;
|
|
1783
|
+
jobWithPrefixes.progress = 0;
|
|
1784
|
+
jobWithPrefixes.progress_message = "";
|
|
1785
|
+
jobWithPrefixes.progress_details = null;
|
|
1786
|
+
jobWithPrefixes.created_at = now;
|
|
1787
|
+
jobWithPrefixes.run_after = now;
|
|
1788
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
1789
|
+
jobWithPrefixes[key] = value;
|
|
1790
|
+
}
|
|
1791
|
+
this.jobQueue.push(jobWithPrefixes);
|
|
1792
|
+
this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
|
|
1793
|
+
return jobWithPrefixes.id;
|
|
1794
|
+
}
|
|
1795
|
+
async get(id) {
|
|
1796
|
+
await sleep2(0);
|
|
1797
|
+
const job = this.jobQueue.find((j) => j.id === id);
|
|
1798
|
+
if (job && this.matchesPrefixes(job)) {
|
|
1799
|
+
return job;
|
|
1800
|
+
}
|
|
1801
|
+
return;
|
|
1802
|
+
}
|
|
1803
|
+
async peek(status = JobStatus.PENDING, num = 100) {
|
|
1804
|
+
await sleep2(0);
|
|
1805
|
+
num = Number(num) || 100;
|
|
1806
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
1807
|
+
}
|
|
1808
|
+
async next(workerId) {
|
|
1809
|
+
await sleep2(0);
|
|
1810
|
+
const top = this.pendingQueue();
|
|
1811
|
+
const job = top[0];
|
|
1812
|
+
if (job) {
|
|
1813
|
+
const oldJob = { ...job };
|
|
1814
|
+
job.status = JobStatus.PROCESSING;
|
|
1815
|
+
job.last_ran_at = new Date().toISOString();
|
|
1816
|
+
job.worker_id = workerId;
|
|
1817
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1818
|
+
return job;
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
async size(status = JobStatus.PENDING) {
|
|
1822
|
+
await sleep2(0);
|
|
1823
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
|
|
1824
|
+
}
|
|
1825
|
+
async saveProgress(id, progress, message, details) {
|
|
1826
|
+
await sleep2(0);
|
|
1827
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
1828
|
+
if (!job) {
|
|
1829
|
+
const jobWithAnyPrefix = this.jobQueue.find((j) => j.id === id);
|
|
1830
|
+
getLogger3().warn("Job not found for progress update", {
|
|
1831
|
+
id,
|
|
1832
|
+
reason: jobWithAnyPrefix ? "prefix_mismatch" : "missing",
|
|
1833
|
+
existingStatus: jobWithAnyPrefix?.status,
|
|
1834
|
+
queueName: this.queueName,
|
|
1835
|
+
prefixValues: this.prefixValues
|
|
1836
|
+
});
|
|
1837
|
+
return;
|
|
1838
|
+
}
|
|
1839
|
+
if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED) {
|
|
1840
|
+
getLogger3().warn("Job already completed or failed for progress update", {
|
|
1841
|
+
id,
|
|
1842
|
+
status: job.status,
|
|
1843
|
+
completedAt: job.completed_at,
|
|
1844
|
+
error: job.error
|
|
1845
|
+
});
|
|
1846
|
+
return;
|
|
1847
|
+
}
|
|
1848
|
+
const oldJob = { ...job };
|
|
1849
|
+
job.progress = progress;
|
|
1850
|
+
job.progress_message = message;
|
|
1851
|
+
job.progress_details = details;
|
|
1852
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1853
|
+
}
|
|
1854
|
+
async complete(job) {
|
|
1855
|
+
await sleep2(0);
|
|
1856
|
+
const jobWithPrefixes = job;
|
|
1857
|
+
const index = this.jobQueue.findIndex((j) => j.id === job.id && this.matchesPrefixes(j));
|
|
1858
|
+
if (index !== -1) {
|
|
1859
|
+
const existing = this.jobQueue[index];
|
|
1860
|
+
const currentAttempts = existing?.run_attempts ?? 0;
|
|
1861
|
+
jobWithPrefixes.run_attempts = currentAttempts + 1;
|
|
1862
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
1863
|
+
jobWithPrefixes[key] = value;
|
|
1864
|
+
}
|
|
1865
|
+
this.jobQueue[index] = jobWithPrefixes;
|
|
1866
|
+
this.events.emit("change", { type: "UPDATE", old: existing, new: jobWithPrefixes });
|
|
1867
|
+
}
|
|
1868
|
+
}
|
|
1869
|
+
async release(id) {
|
|
1870
|
+
await sleep2(0);
|
|
1871
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
1872
|
+
if (job) {
|
|
1873
|
+
const oldJob = { ...job };
|
|
1874
|
+
job.status = JobStatus.PENDING;
|
|
1875
|
+
job.worker_id = null;
|
|
1876
|
+
job.progress = 0;
|
|
1877
|
+
job.progress_message = "";
|
|
1878
|
+
job.progress_details = null;
|
|
1879
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1880
|
+
}
|
|
1881
|
+
}
|
|
1882
|
+
async abort(id) {
|
|
1883
|
+
await sleep2(0);
|
|
1884
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
1885
|
+
if (job) {
|
|
1886
|
+
const oldJob = { ...job };
|
|
1887
|
+
job.status = JobStatus.ABORTING;
|
|
1888
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
async getByRunId(runId) {
|
|
1892
|
+
await sleep2(0);
|
|
1893
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
|
|
1894
|
+
}
|
|
1895
|
+
async deleteAll() {
|
|
1896
|
+
await sleep2(0);
|
|
1897
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
|
|
1898
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
|
|
1899
|
+
for (const job of deletedJobs) {
|
|
1900
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
async outputForInput(input) {
|
|
1904
|
+
await sleep2(0);
|
|
1905
|
+
const fingerprint = await makeFingerprint(input);
|
|
1906
|
+
return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === JobStatus.COMPLETED)?.output ?? null;
|
|
1907
|
+
}
|
|
1908
|
+
async delete(id) {
|
|
1909
|
+
await sleep2(0);
|
|
1910
|
+
const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
|
|
1911
|
+
this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
|
|
1912
|
+
if (deletedJob) {
|
|
1913
|
+
this.events.emit("change", { type: "DELETE", old: deletedJob });
|
|
1914
|
+
}
|
|
1915
|
+
}
|
|
1916
|
+
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
1917
|
+
await sleep2(0);
|
|
1918
|
+
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
1919
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
|
|
1920
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
|
|
1921
|
+
for (const job of deletedJobs) {
|
|
1922
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
1923
|
+
}
|
|
1924
|
+
}
|
|
1925
|
+
async setupDatabase() {}
|
|
1926
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
1927
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
1928
|
+
return true;
|
|
1929
|
+
}
|
|
1930
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
1931
|
+
if (Object.keys(filterValues).length === 0) {
|
|
1932
|
+
return true;
|
|
1933
|
+
}
|
|
1934
|
+
const jobWithPrefixes = job;
|
|
1935
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
1936
|
+
if (jobWithPrefixes[key] !== value) {
|
|
1937
|
+
return false;
|
|
1938
|
+
}
|
|
1939
|
+
}
|
|
1940
|
+
return true;
|
|
1941
|
+
}
|
|
1942
|
+
subscribeToChanges(callback, options) {
|
|
1943
|
+
const prefixFilter = options?.prefixFilter;
|
|
1944
|
+
const filteredCallback = (change) => {
|
|
1945
|
+
const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
|
|
1946
|
+
const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
|
|
1947
|
+
if (!newMatches && !oldMatches) {
|
|
1948
|
+
return;
|
|
1949
|
+
}
|
|
1950
|
+
callback(change);
|
|
1951
|
+
};
|
|
1952
|
+
return this.events.subscribe("change", filteredCallback);
|
|
1953
|
+
}
|
|
1954
|
+
}
|
|
1955
|
+
|
|
1956
|
+
// src/queue-storage/TelemetryQueueStorage.ts
|
|
1957
|
+
import { traced } from "@workglow/util";
|
|
1958
|
+
|
|
1959
|
+
class TelemetryQueueStorage {
|
|
1960
|
+
storageName;
|
|
1961
|
+
inner;
|
|
1962
|
+
constructor(storageName, inner) {
|
|
1963
|
+
this.storageName = storageName;
|
|
1964
|
+
this.inner = inner;
|
|
1965
|
+
}
|
|
1966
|
+
get scope() {
|
|
1967
|
+
return this.inner.scope;
|
|
1968
|
+
}
|
|
1969
|
+
add(job) {
|
|
1970
|
+
return traced("workglow.storage.queue.add", this.storageName, () => this.inner.add(job));
|
|
1971
|
+
}
|
|
1972
|
+
get(id) {
|
|
1973
|
+
return traced("workglow.storage.queue.get", this.storageName, () => this.inner.get(id));
|
|
1974
|
+
}
|
|
1975
|
+
next(workerId) {
|
|
1976
|
+
return traced("workglow.storage.queue.next", this.storageName, () => this.inner.next(workerId));
|
|
1977
|
+
}
|
|
1978
|
+
peek(status, num) {
|
|
1979
|
+
return traced("workglow.storage.queue.peek", this.storageName, () => this.inner.peek(status, num));
|
|
1980
|
+
}
|
|
1981
|
+
size(status) {
|
|
1982
|
+
return traced("workglow.storage.queue.size", this.storageName, () => this.inner.size(status));
|
|
1983
|
+
}
|
|
1984
|
+
complete(job) {
|
|
1985
|
+
return traced("workglow.storage.queue.complete", this.storageName, () => this.inner.complete(job));
|
|
1986
|
+
}
|
|
1987
|
+
release(id) {
|
|
1988
|
+
return traced("workglow.storage.queue.release", this.storageName, () => this.inner.release(id));
|
|
1989
|
+
}
|
|
1990
|
+
deleteAll() {
|
|
1991
|
+
return traced("workglow.storage.queue.deleteAll", this.storageName, () => this.inner.deleteAll());
|
|
1992
|
+
}
|
|
1993
|
+
outputForInput(input) {
|
|
1994
|
+
return traced("workglow.storage.queue.outputForInput", this.storageName, () => this.inner.outputForInput(input));
|
|
1995
|
+
}
|
|
1996
|
+
abort(id) {
|
|
1997
|
+
return traced("workglow.storage.queue.abort", this.storageName, () => this.inner.abort(id));
|
|
1998
|
+
}
|
|
1999
|
+
getByRunId(runId) {
|
|
2000
|
+
return traced("workglow.storage.queue.getByRunId", this.storageName, () => this.inner.getByRunId(runId));
|
|
2001
|
+
}
|
|
2002
|
+
saveProgress(id, progress, message, details) {
|
|
2003
|
+
return traced("workglow.storage.queue.saveProgress", this.storageName, () => this.inner.saveProgress(id, progress, message, details));
|
|
2004
|
+
}
|
|
2005
|
+
delete(id) {
|
|
2006
|
+
return traced("workglow.storage.queue.delete", this.storageName, () => this.inner.delete(id));
|
|
2007
|
+
}
|
|
2008
|
+
deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2009
|
+
return traced("workglow.storage.queue.deleteJobsByStatusAndAge", this.storageName, () => this.inner.deleteJobsByStatusAndAge(status, olderThanMs));
|
|
2010
|
+
}
|
|
2011
|
+
setupDatabase() {
|
|
2012
|
+
return this.inner.setupDatabase();
|
|
2013
|
+
}
|
|
2014
|
+
subscribeToChanges(callback, options) {
|
|
2015
|
+
return this.inner.subscribeToChanges(callback, options);
|
|
2016
|
+
}
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
// src/rate-limiter-storage/IRateLimiterStorage.ts
|
|
2020
|
+
import { createServiceToken as createServiceToken7 } from "@workglow/util";
|
|
2021
|
+
var RATE_LIMITER_STORAGE = createServiceToken7("ratelimiter.storage");
|
|
2022
|
+
|
|
2023
|
+
// src/rate-limiter-storage/InMemoryRateLimiterStorage.ts
|
|
2024
|
+
import { createServiceToken as createServiceToken8, sleep as sleep3, uuid4 as uuid43 } from "@workglow/util";
|
|
2025
|
+
var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage.inMemory");
|
|
2026
|
+
|
|
2027
|
+
class InMemoryRateLimiterStorage {
|
|
2028
|
+
scope = "process";
|
|
2029
|
+
prefixValues;
|
|
2030
|
+
executions = new Map;
|
|
2031
|
+
nextAvailableTimes = new Map;
|
|
2032
|
+
reserveChains = new Map;
|
|
2033
|
+
constructor(options) {
|
|
2034
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
2035
|
+
}
|
|
2036
|
+
makeKey(queueName) {
|
|
2037
|
+
const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
|
|
2038
|
+
return prefixPart ? `${prefixPart}|${queueName}` : queueName;
|
|
2039
|
+
}
|
|
2040
|
+
async setupDatabase() {}
|
|
2041
|
+
async withKeyLock(key, fn) {
|
|
2042
|
+
const previous = this.reserveChains.get(key) ?? Promise.resolve();
|
|
2043
|
+
let release;
|
|
2044
|
+
const next = new Promise((resolve) => {
|
|
2045
|
+
release = resolve;
|
|
2046
|
+
});
|
|
2047
|
+
this.reserveChains.set(key, next);
|
|
2048
|
+
try {
|
|
2049
|
+
await previous;
|
|
2050
|
+
return await fn();
|
|
2051
|
+
} finally {
|
|
2052
|
+
release(undefined);
|
|
2053
|
+
if (this.reserveChains.get(key) === next) {
|
|
2054
|
+
this.reserveChains.delete(key);
|
|
2055
|
+
}
|
|
2056
|
+
}
|
|
2057
|
+
}
|
|
2058
|
+
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
2059
|
+
const key = this.makeKey(queueName);
|
|
2060
|
+
return this.withKeyLock(key, () => {
|
|
2061
|
+
const now = Date.now();
|
|
2062
|
+
const windowStart = new Date(now - windowMs);
|
|
2063
|
+
const executions = this.executions.get(key) ?? [];
|
|
2064
|
+
const live = executions.filter((e) => e.executedAt > windowStart);
|
|
2065
|
+
if (live.length >= maxExecutions) {
|
|
2066
|
+
if (live.length !== executions.length) {
|
|
2067
|
+
this.executions.set(key, live);
|
|
2068
|
+
}
|
|
2069
|
+
return null;
|
|
2070
|
+
}
|
|
2071
|
+
const next = this.nextAvailableTimes.get(key);
|
|
2072
|
+
if (next && next.getTime() > now) {
|
|
2073
|
+
return null;
|
|
2074
|
+
}
|
|
2075
|
+
const id = uuid43();
|
|
2076
|
+
live.push({ id, queueName, executedAt: new Date(now) });
|
|
2077
|
+
this.executions.set(key, live);
|
|
2078
|
+
return id;
|
|
2079
|
+
});
|
|
2080
|
+
}
|
|
2081
|
+
async releaseExecution(queueName, token) {
|
|
2082
|
+
if (token === null || token === undefined)
|
|
2083
|
+
return;
|
|
2084
|
+
const key = this.makeKey(queueName);
|
|
2085
|
+
await this.withKeyLock(key, () => {
|
|
2086
|
+
const executions = this.executions.get(key);
|
|
2087
|
+
if (!executions || executions.length === 0)
|
|
2088
|
+
return;
|
|
2089
|
+
const idx = executions.findIndex((e) => e.id === token);
|
|
2090
|
+
if (idx === -1)
|
|
2091
|
+
return;
|
|
2092
|
+
executions.splice(idx, 1);
|
|
2093
|
+
this.executions.set(key, executions);
|
|
2094
|
+
});
|
|
2095
|
+
}
|
|
2096
|
+
async recordExecution(queueName) {
|
|
2097
|
+
await sleep3(0);
|
|
2098
|
+
const key = this.makeKey(queueName);
|
|
2099
|
+
const executions = this.executions.get(key) ?? [];
|
|
2100
|
+
executions.push({
|
|
2101
|
+
id: uuid43(),
|
|
2102
|
+
queueName,
|
|
2103
|
+
executedAt: new Date
|
|
2104
|
+
});
|
|
2105
|
+
this.executions.set(key, executions);
|
|
2106
|
+
}
|
|
2107
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
2108
|
+
await sleep3(0);
|
|
2109
|
+
const key = this.makeKey(queueName);
|
|
2110
|
+
const executions = this.executions.get(key) ?? [];
|
|
2111
|
+
const windowStart = new Date(windowStartTime);
|
|
2112
|
+
return executions.filter((e) => e.executedAt > windowStart).length;
|
|
2113
|
+
}
|
|
2114
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
2115
|
+
await sleep3(0);
|
|
2116
|
+
const key = this.makeKey(queueName);
|
|
2117
|
+
const executions = this.executions.get(key) ?? [];
|
|
2118
|
+
const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
|
|
2119
|
+
const execution = sorted[offset];
|
|
2120
|
+
return execution?.executedAt.toISOString();
|
|
2121
|
+
}
|
|
2122
|
+
async getNextAvailableTime(queueName) {
|
|
2123
|
+
await sleep3(0);
|
|
2124
|
+
const key = this.makeKey(queueName);
|
|
2125
|
+
const time = this.nextAvailableTimes.get(key);
|
|
2126
|
+
return time?.toISOString();
|
|
2127
|
+
}
|
|
2128
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
2129
|
+
await sleep3(0);
|
|
2130
|
+
const key = this.makeKey(queueName);
|
|
2131
|
+
this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
|
|
2132
|
+
}
|
|
2133
|
+
async clear(queueName) {
|
|
2134
|
+
await sleep3(0);
|
|
2135
|
+
const key = this.makeKey(queueName);
|
|
2136
|
+
this.executions.delete(key);
|
|
2137
|
+
this.nextAvailableTimes.delete(key);
|
|
2138
|
+
}
|
|
2139
|
+
}
|
|
1733
2140
|
export {
|
|
1734
2141
|
withJobErrorDiagnostics,
|
|
1735
2142
|
storageToClass,
|
|
1736
2143
|
formatErrorChainForDiagnostics,
|
|
1737
2144
|
classToStorage,
|
|
1738
2145
|
applyPersistedDiagnosticsToStack,
|
|
2146
|
+
TelemetryQueueStorage,
|
|
1739
2147
|
RetryableJobError,
|
|
1740
2148
|
RateLimiter,
|
|
2149
|
+
RATE_LIMITER_STORAGE,
|
|
2150
|
+
QUEUE_STORAGE,
|
|
1741
2151
|
PermanentJobError,
|
|
1742
2152
|
NullLimiter,
|
|
1743
2153
|
NULL_JOB_LIMITER,
|
|
@@ -1751,6 +2161,10 @@ export {
|
|
|
1751
2161
|
Job,
|
|
1752
2162
|
JOB_LIMITER,
|
|
1753
2163
|
JOB_ERROR_DIAGNOSTICS_MARKER,
|
|
2164
|
+
InMemoryRateLimiterStorage,
|
|
2165
|
+
InMemoryQueueStorage,
|
|
2166
|
+
IN_MEMORY_RATE_LIMITER_STORAGE,
|
|
2167
|
+
IN_MEMORY_QUEUE_STORAGE,
|
|
1754
2168
|
EvenlySpacedRateLimiter,
|
|
1755
2169
|
EVENLY_SPACED_JOB_RATE_LIMITER,
|
|
1756
2170
|
DelayLimiter,
|
|
@@ -1760,4 +2174,4 @@ export {
|
|
|
1760
2174
|
AbortSignalJobError
|
|
1761
2175
|
};
|
|
1762
2176
|
|
|
1763
|
-
//# debugId=
|
|
2177
|
+
//# debugId=4ACCF006917528EA64756E2164756E21
|