@workglow/storage 0.0.56 → 0.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +1266 -121
- package/dist/browser.js.map +15 -10
- package/dist/bun.js +2017 -252
- package/dist/bun.js.map +19 -12
- package/dist/common-server.d.ts +4 -0
- package/dist/common-server.d.ts.map +1 -1
- package/dist/common.d.ts +2 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/limiter/IRateLimiterStorage.d.ts +81 -0
- package/dist/limiter/IRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/InMemoryRateLimiterStorage.d.ts +32 -0
- package/dist/limiter/InMemoryRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/IndexedDbRateLimiterStorage.d.ts +52 -0
- package/dist/limiter/IndexedDbRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/PostgresRateLimiterStorage.d.ts +54 -0
- package/dist/limiter/PostgresRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/SqliteRateLimiterStorage.d.ts +53 -0
- package/dist/limiter/SqliteRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/SupabaseRateLimiterStorage.d.ts +53 -0
- package/dist/limiter/SupabaseRateLimiterStorage.d.ts.map +1 -0
- package/dist/node.js +2017 -252
- package/dist/node.js.map +19 -12
- package/dist/queue/IQueueStorage.d.ts +72 -1
- package/dist/queue/IQueueStorage.d.ts.map +1 -1
- package/dist/queue/InMemoryQueueStorage.d.ts +44 -11
- package/dist/queue/InMemoryQueueStorage.d.ts.map +1 -1
- package/dist/queue/IndexedDbQueueStorage.d.ts +70 -5
- package/dist/queue/IndexedDbQueueStorage.d.ts.map +1 -1
- package/dist/queue/PostgresQueueStorage.d.ts +80 -8
- package/dist/queue/PostgresQueueStorage.d.ts.map +1 -1
- package/dist/queue/SqliteQueueStorage.d.ts +90 -34
- package/dist/queue/SqliteQueueStorage.d.ts.map +1 -1
- package/dist/queue/SupabaseQueueStorage.d.ts +98 -4
- package/dist/queue/SupabaseQueueStorage.d.ts.map +1 -1
- package/dist/tabular/ITabularRepository.d.ts +18 -0
- package/dist/tabular/ITabularRepository.d.ts.map +1 -1
- package/dist/tabular/InMemoryTabularRepository.d.ts +9 -1
- package/dist/tabular/InMemoryTabularRepository.d.ts.map +1 -1
- package/dist/tabular/SupabaseTabularRepository.d.ts +21 -1
- package/dist/tabular/SupabaseTabularRepository.d.ts.map +1 -1
- package/dist/tabular/TabularRepository.d.ts +10 -1
- package/dist/tabular/TabularRepository.d.ts.map +1 -1
- package/dist/util/PollingSubscriptionManager.d.ts +112 -0
- package/dist/util/PollingSubscriptionManager.d.ts.map +1 -0
- package/package.json +5 -5
package/dist/node.js
CHANGED
|
@@ -106,6 +106,9 @@ class TabularRepository {
|
|
|
106
106
|
waitOn(name) {
|
|
107
107
|
return this.events.waitOn(name);
|
|
108
108
|
}
|
|
109
|
+
subscribeToChanges(_callback) {
|
|
110
|
+
throw new Error(`subscribeToChanges is not implemented for ${this.constructor.name}. ` + `Use InMemoryTabularRepository or SupabaseTabularRepository for subscription support.`);
|
|
111
|
+
}
|
|
109
112
|
primaryKeyColumns() {
|
|
110
113
|
const columns = [];
|
|
111
114
|
for (const key of Object.keys(this.primaryKeySchema.properties)) {
|
|
@@ -273,12 +276,30 @@ class InMemoryTabularRepository extends TabularRepository {
|
|
|
273
276
|
return false;
|
|
274
277
|
}
|
|
275
278
|
});
|
|
276
|
-
for (const [id,
|
|
279
|
+
for (const [id, entity] of entriesToDelete) {
|
|
277
280
|
this.values.delete(id);
|
|
281
|
+
const { key } = this.separateKeyValueFromCombined(entity);
|
|
282
|
+
this.events.emit("delete", key);
|
|
278
283
|
}
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
284
|
+
}
|
|
285
|
+
subscribeToChanges(callback) {
|
|
286
|
+
const handlePut = (entity) => {
|
|
287
|
+
callback({ type: "UPDATE", new: entity });
|
|
288
|
+
};
|
|
289
|
+
const handleDelete = (_key) => {
|
|
290
|
+
callback({ type: "DELETE" });
|
|
291
|
+
};
|
|
292
|
+
const handleClearAll = () => {
|
|
293
|
+
callback({ type: "DELETE" });
|
|
294
|
+
};
|
|
295
|
+
this.events.on("put", handlePut);
|
|
296
|
+
this.events.on("delete", handleDelete);
|
|
297
|
+
this.events.on("clearall", handleClearAll);
|
|
298
|
+
return () => {
|
|
299
|
+
this.events.off("put", handlePut);
|
|
300
|
+
this.events.off("delete", handleDelete);
|
|
301
|
+
this.events.off("clearall", handleClearAll);
|
|
302
|
+
};
|
|
282
303
|
}
|
|
283
304
|
destroy() {
|
|
284
305
|
this.values.clear();
|
|
@@ -540,7 +561,7 @@ class InMemoryKvRepository extends KvViaTabularRepository {
|
|
|
540
561
|
}
|
|
541
562
|
}
|
|
542
563
|
// src/queue/InMemoryQueueStorage.ts
|
|
543
|
-
import { createServiceToken as createServiceToken7, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
|
|
564
|
+
import { createServiceToken as createServiceToken7, EventEmitter as EventEmitter3, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
|
|
544
565
|
|
|
545
566
|
// src/queue/IQueueStorage.ts
|
|
546
567
|
import { createServiceToken as createServiceToken6 } from "@workglow/util";
|
|
@@ -560,63 +581,88 @@ var IN_MEMORY_QUEUE_STORAGE = createServiceToken7("jobqueue.storage.inMemory");
|
|
|
560
581
|
|
|
561
582
|
class InMemoryQueueStorage {
|
|
562
583
|
queueName;
|
|
563
|
-
|
|
584
|
+
prefixValues;
|
|
585
|
+
events = new EventEmitter3;
|
|
586
|
+
constructor(queueName, options) {
|
|
564
587
|
this.queueName = queueName;
|
|
565
588
|
this.jobQueue = [];
|
|
589
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
566
590
|
}
|
|
567
591
|
jobQueue;
|
|
592
|
+
matchesPrefixes(job) {
|
|
593
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
594
|
+
if (job[key] !== value) {
|
|
595
|
+
return false;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
return true;
|
|
599
|
+
}
|
|
568
600
|
pendingQueue() {
|
|
569
601
|
const now = new Date().toISOString();
|
|
570
|
-
return this.jobQueue.filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
602
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
571
603
|
}
|
|
572
604
|
async add(job) {
|
|
573
605
|
await sleep(0);
|
|
574
606
|
const now = new Date().toISOString();
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
607
|
+
const jobWithPrefixes = job;
|
|
608
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid4();
|
|
609
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid4();
|
|
610
|
+
jobWithPrefixes.queue = this.queueName;
|
|
611
|
+
jobWithPrefixes.fingerprint = await makeFingerprint4(jobWithPrefixes.input);
|
|
612
|
+
jobWithPrefixes.status = "PENDING" /* PENDING */;
|
|
613
|
+
jobWithPrefixes.progress = 0;
|
|
614
|
+
jobWithPrefixes.progress_message = "";
|
|
615
|
+
jobWithPrefixes.progress_details = null;
|
|
616
|
+
jobWithPrefixes.created_at = now;
|
|
617
|
+
jobWithPrefixes.run_after = now;
|
|
618
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
619
|
+
jobWithPrefixes[key] = value;
|
|
620
|
+
}
|
|
621
|
+
this.jobQueue.push(jobWithPrefixes);
|
|
622
|
+
this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
|
|
623
|
+
return jobWithPrefixes.id;
|
|
587
624
|
}
|
|
588
625
|
async get(id) {
|
|
589
626
|
await sleep(0);
|
|
590
|
-
|
|
627
|
+
const job = this.jobQueue.find((j) => j.id === id);
|
|
628
|
+
if (job && this.matchesPrefixes(job)) {
|
|
629
|
+
return job;
|
|
630
|
+
}
|
|
631
|
+
return;
|
|
591
632
|
}
|
|
592
633
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
593
634
|
await sleep(0);
|
|
594
635
|
num = Number(num) || 100;
|
|
595
|
-
return this.jobQueue.sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
636
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
596
637
|
}
|
|
597
|
-
async next() {
|
|
638
|
+
async next(workerId) {
|
|
598
639
|
await sleep(0);
|
|
599
640
|
const top = this.pendingQueue();
|
|
600
641
|
const job = top[0];
|
|
601
642
|
if (job) {
|
|
643
|
+
const oldJob = { ...job };
|
|
602
644
|
job.status = "PROCESSING" /* PROCESSING */;
|
|
603
645
|
job.last_ran_at = new Date().toISOString();
|
|
646
|
+
job.worker_id = workerId ?? null;
|
|
647
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
604
648
|
return job;
|
|
605
649
|
}
|
|
606
650
|
}
|
|
607
651
|
async size(status = "PENDING" /* PENDING */) {
|
|
608
652
|
await sleep(0);
|
|
609
|
-
return this.jobQueue.filter((j) => j.status === status).length;
|
|
653
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
|
|
610
654
|
}
|
|
611
655
|
async saveProgress(id, progress, message, details) {
|
|
612
656
|
await sleep(0);
|
|
613
|
-
const job = this.jobQueue.find((j) => j.id === id);
|
|
657
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
614
658
|
if (!job) {
|
|
615
659
|
throw new Error(`Job ${id} not found`);
|
|
616
660
|
}
|
|
661
|
+
const oldJob = { ...job };
|
|
617
662
|
job.progress = progress;
|
|
618
663
|
job.progress_message = message;
|
|
619
664
|
job.progress_details = details;
|
|
665
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
620
666
|
}
|
|
621
667
|
async complete(job) {
|
|
622
668
|
await sleep(0);
|
|
@@ -626,44 +672,149 @@ class InMemoryQueueStorage {
|
|
|
626
672
|
const currentAttempts = existing?.run_attempts ?? 0;
|
|
627
673
|
job.run_attempts = currentAttempts + 1;
|
|
628
674
|
this.jobQueue[index] = job;
|
|
675
|
+
this.events.emit("change", { type: "UPDATE", old: existing, new: job });
|
|
629
676
|
}
|
|
630
677
|
}
|
|
631
678
|
async abort(id) {
|
|
632
679
|
await sleep(0);
|
|
633
|
-
const job = this.jobQueue.find((j) => j.id === id);
|
|
680
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
634
681
|
if (job) {
|
|
682
|
+
const oldJob = { ...job };
|
|
635
683
|
job.status = "ABORTING" /* ABORTING */;
|
|
684
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
636
685
|
}
|
|
637
686
|
}
|
|
638
687
|
async getByRunId(runId) {
|
|
639
688
|
await sleep(0);
|
|
640
|
-
return this.jobQueue.filter((job) => job.job_run_id === runId);
|
|
689
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
|
|
641
690
|
}
|
|
642
691
|
async deleteAll() {
|
|
643
692
|
await sleep(0);
|
|
644
|
-
this.jobQueue
|
|
693
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
|
|
694
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
|
|
695
|
+
for (const job of deletedJobs) {
|
|
696
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
697
|
+
}
|
|
645
698
|
}
|
|
646
699
|
async outputForInput(input) {
|
|
647
700
|
await sleep(0);
|
|
648
701
|
const fingerprint = await makeFingerprint4(input);
|
|
649
|
-
return this.jobQueue.find((j) => j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
|
|
702
|
+
return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
|
|
650
703
|
}
|
|
651
704
|
async delete(id) {
|
|
652
705
|
await sleep(0);
|
|
653
|
-
|
|
706
|
+
const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
|
|
707
|
+
this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
|
|
708
|
+
if (deletedJob) {
|
|
709
|
+
this.events.emit("change", { type: "DELETE", old: deletedJob });
|
|
710
|
+
}
|
|
654
711
|
}
|
|
655
712
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
656
713
|
await sleep(0);
|
|
657
714
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
658
|
-
|
|
715
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
|
|
716
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
|
|
717
|
+
for (const job of deletedJobs) {
|
|
718
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
async setupDatabase() {}
|
|
722
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
723
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
724
|
+
return true;
|
|
725
|
+
}
|
|
726
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
727
|
+
if (Object.keys(filterValues).length === 0) {
|
|
728
|
+
return true;
|
|
729
|
+
}
|
|
730
|
+
const jobWithPrefixes = job;
|
|
731
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
732
|
+
if (jobWithPrefixes[key] !== value) {
|
|
733
|
+
return false;
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
return true;
|
|
737
|
+
}
|
|
738
|
+
subscribeToChanges(callback, options) {
|
|
739
|
+
const prefixFilter = options?.prefixFilter;
|
|
740
|
+
const filteredCallback = (change) => {
|
|
741
|
+
const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
|
|
742
|
+
const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
|
|
743
|
+
if (!newMatches && !oldMatches) {
|
|
744
|
+
return;
|
|
745
|
+
}
|
|
746
|
+
callback(change);
|
|
747
|
+
};
|
|
748
|
+
return this.events.subscribe("change", filteredCallback);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
// src/limiter/IRateLimiterStorage.ts
|
|
752
|
+
import { createServiceToken as createServiceToken8 } from "@workglow/util";
|
|
753
|
+
var RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage");
|
|
754
|
+
// src/limiter/InMemoryRateLimiterStorage.ts
|
|
755
|
+
import { createServiceToken as createServiceToken9, sleep as sleep2 } from "@workglow/util";
|
|
756
|
+
var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken9("ratelimiter.storage.inMemory");
|
|
757
|
+
|
|
758
|
+
class InMemoryRateLimiterStorage {
|
|
759
|
+
prefixValues;
|
|
760
|
+
executions = new Map;
|
|
761
|
+
nextAvailableTimes = new Map;
|
|
762
|
+
constructor(options) {
|
|
763
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
764
|
+
}
|
|
765
|
+
makeKey(queueName) {
|
|
766
|
+
const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
|
|
767
|
+
return prefixPart ? `${prefixPart}|${queueName}` : queueName;
|
|
659
768
|
}
|
|
660
769
|
async setupDatabase() {}
|
|
770
|
+
async recordExecution(queueName) {
|
|
771
|
+
await sleep2(0);
|
|
772
|
+
const key = this.makeKey(queueName);
|
|
773
|
+
const executions = this.executions.get(key) ?? [];
|
|
774
|
+
executions.push({
|
|
775
|
+
queueName,
|
|
776
|
+
executedAt: new Date
|
|
777
|
+
});
|
|
778
|
+
this.executions.set(key, executions);
|
|
779
|
+
}
|
|
780
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
781
|
+
await sleep2(0);
|
|
782
|
+
const key = this.makeKey(queueName);
|
|
783
|
+
const executions = this.executions.get(key) ?? [];
|
|
784
|
+
const windowStart = new Date(windowStartTime);
|
|
785
|
+
return executions.filter((e) => e.executedAt > windowStart).length;
|
|
786
|
+
}
|
|
787
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
788
|
+
await sleep2(0);
|
|
789
|
+
const key = this.makeKey(queueName);
|
|
790
|
+
const executions = this.executions.get(key) ?? [];
|
|
791
|
+
const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
|
|
792
|
+
const execution = sorted[offset];
|
|
793
|
+
return execution?.executedAt.toISOString();
|
|
794
|
+
}
|
|
795
|
+
async getNextAvailableTime(queueName) {
|
|
796
|
+
await sleep2(0);
|
|
797
|
+
const key = this.makeKey(queueName);
|
|
798
|
+
const time = this.nextAvailableTimes.get(key);
|
|
799
|
+
return time?.toISOString();
|
|
800
|
+
}
|
|
801
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
802
|
+
await sleep2(0);
|
|
803
|
+
const key = this.makeKey(queueName);
|
|
804
|
+
this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
|
|
805
|
+
}
|
|
806
|
+
async clear(queueName) {
|
|
807
|
+
await sleep2(0);
|
|
808
|
+
const key = this.makeKey(queueName);
|
|
809
|
+
this.executions.delete(key);
|
|
810
|
+
this.nextAvailableTimes.delete(key);
|
|
811
|
+
}
|
|
661
812
|
}
|
|
662
813
|
// src/tabular/FsFolderTabularRepository.ts
|
|
663
|
-
import { createServiceToken as
|
|
814
|
+
import { createServiceToken as createServiceToken10, sleep as sleep3 } from "@workglow/util";
|
|
664
815
|
import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
665
816
|
import path from "node:path";
|
|
666
|
-
var FS_FOLDER_TABULAR_REPOSITORY =
|
|
817
|
+
var FS_FOLDER_TABULAR_REPOSITORY = createServiceToken10("storage.tabularRepository.fsFolder");
|
|
667
818
|
|
|
668
819
|
class FsFolderTabularRepository extends TabularRepository {
|
|
669
820
|
folderPath;
|
|
@@ -688,7 +839,7 @@ class FsFolderTabularRepository extends TabularRepository {
|
|
|
688
839
|
await writeFile(filePath, JSON.stringify(entity));
|
|
689
840
|
} catch (error) {
|
|
690
841
|
try {
|
|
691
|
-
await
|
|
842
|
+
await sleep3(1);
|
|
692
843
|
await writeFile(filePath, JSON.stringify(entity));
|
|
693
844
|
} catch (error2) {
|
|
694
845
|
console.error("Error writing file", filePath, error2);
|
|
@@ -772,7 +923,7 @@ class FsFolderTabularRepository extends TabularRepository {
|
|
|
772
923
|
}
|
|
773
924
|
}
|
|
774
925
|
// src/tabular/PostgresTabularRepository.ts
|
|
775
|
-
import { createServiceToken as
|
|
926
|
+
import { createServiceToken as createServiceToken11 } from "@workglow/util";
|
|
776
927
|
|
|
777
928
|
// src/tabular/BaseSqlTabularRepository.ts
|
|
778
929
|
class BaseSqlTabularRepository extends TabularRepository {
|
|
@@ -958,7 +1109,7 @@ class BaseSqlTabularRepository extends TabularRepository {
|
|
|
958
1109
|
}
|
|
959
1110
|
|
|
960
1111
|
// src/tabular/PostgresTabularRepository.ts
|
|
961
|
-
var POSTGRES_TABULAR_REPOSITORY =
|
|
1112
|
+
var POSTGRES_TABULAR_REPOSITORY = createServiceToken11("storage.tabularRepository.postgres");
|
|
962
1113
|
|
|
963
1114
|
class PostgresTabularRepository extends BaseSqlTabularRepository {
|
|
964
1115
|
db;
|
|
@@ -1316,8 +1467,8 @@ class PostgresTabularRepository extends BaseSqlTabularRepository {
|
|
|
1316
1467
|
}
|
|
1317
1468
|
// src/tabular/SqliteTabularRepository.ts
|
|
1318
1469
|
import { Sqlite } from "@workglow/sqlite";
|
|
1319
|
-
import { createServiceToken as
|
|
1320
|
-
var SQLITE_TABULAR_REPOSITORY =
|
|
1470
|
+
import { createServiceToken as createServiceToken12 } from "@workglow/util";
|
|
1471
|
+
var SQLITE_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.sqlite");
|
|
1321
1472
|
var Database = Sqlite.Database;
|
|
1322
1473
|
|
|
1323
1474
|
class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
@@ -1594,11 +1745,12 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1594
1745
|
}
|
|
1595
1746
|
}
|
|
1596
1747
|
// src/tabular/SupabaseTabularRepository.ts
|
|
1597
|
-
import { createServiceToken as
|
|
1598
|
-
var SUPABASE_TABULAR_REPOSITORY =
|
|
1748
|
+
import { createServiceToken as createServiceToken13 } from "@workglow/util";
|
|
1749
|
+
var SUPABASE_TABULAR_REPOSITORY = createServiceToken13("storage.tabularRepository.supabase");
|
|
1599
1750
|
|
|
1600
1751
|
class SupabaseTabularRepository extends BaseSqlTabularRepository {
|
|
1601
1752
|
client;
|
|
1753
|
+
realtimeChannel = null;
|
|
1602
1754
|
constructor(client, table = "tabular_store", schema, primaryKeyNames, indexes = []) {
|
|
1603
1755
|
super(table, schema, primaryKeyNames, indexes);
|
|
1604
1756
|
this.client = client;
|
|
@@ -1971,10 +2123,44 @@ class SupabaseTabularRepository extends BaseSqlTabularRepository {
|
|
|
1971
2123
|
throw error;
|
|
1972
2124
|
this.events.emit("delete", column);
|
|
1973
2125
|
}
|
|
2126
|
+
convertRealtimeRow(row) {
|
|
2127
|
+
const entity = { ...row };
|
|
2128
|
+
for (const key in this.schema.properties) {
|
|
2129
|
+
entity[key] = this.sqlToJsValue(key, row[key]);
|
|
2130
|
+
}
|
|
2131
|
+
return entity;
|
|
2132
|
+
}
|
|
2133
|
+
subscribeToChanges(callback) {
|
|
2134
|
+
const channelName = `tabular-${this.table}-${Date.now()}`;
|
|
2135
|
+
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
2136
|
+
event: "*",
|
|
2137
|
+
schema: "public",
|
|
2138
|
+
table: this.table
|
|
2139
|
+
}, (payload) => {
|
|
2140
|
+
const change = {
|
|
2141
|
+
type: payload.eventType.toUpperCase(),
|
|
2142
|
+
old: payload.old && Object.keys(payload.old).length > 0 ? this.convertRealtimeRow(payload.old) : undefined,
|
|
2143
|
+
new: payload.new && Object.keys(payload.new).length > 0 ? this.convertRealtimeRow(payload.new) : undefined
|
|
2144
|
+
};
|
|
2145
|
+
callback(change);
|
|
2146
|
+
}).subscribe();
|
|
2147
|
+
return () => {
|
|
2148
|
+
if (this.realtimeChannel) {
|
|
2149
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
2150
|
+
this.realtimeChannel = null;
|
|
2151
|
+
}
|
|
2152
|
+
};
|
|
2153
|
+
}
|
|
2154
|
+
destroy() {
|
|
2155
|
+
if (this.realtimeChannel) {
|
|
2156
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
2157
|
+
this.realtimeChannel = null;
|
|
2158
|
+
}
|
|
2159
|
+
}
|
|
1974
2160
|
}
|
|
1975
2161
|
// src/kv/FsFolderJsonKvRepository.ts
|
|
1976
|
-
import { createServiceToken as
|
|
1977
|
-
var FS_FOLDER_JSON_KV_REPOSITORY =
|
|
2162
|
+
import { createServiceToken as createServiceToken14 } from "@workglow/util";
|
|
2163
|
+
var FS_FOLDER_JSON_KV_REPOSITORY = createServiceToken14("storage.kvRepository.fsFolderJson");
|
|
1978
2164
|
|
|
1979
2165
|
class FsFolderJsonKvRepository extends KvViaTabularRepository {
|
|
1980
2166
|
folderPath;
|
|
@@ -1986,10 +2172,10 @@ class FsFolderJsonKvRepository extends KvViaTabularRepository {
|
|
|
1986
2172
|
}
|
|
1987
2173
|
}
|
|
1988
2174
|
// src/kv/FsFolderKvRepository.ts
|
|
1989
|
-
import { createServiceToken as
|
|
2175
|
+
import { createServiceToken as createServiceToken15 } from "@workglow/util";
|
|
1990
2176
|
import { mkdir as mkdir2, readFile as readFile2, rm as rm2, unlink, writeFile as writeFile2 } from "fs/promises";
|
|
1991
2177
|
import path2 from "path";
|
|
1992
|
-
var FS_FOLDER_KV_REPOSITORY =
|
|
2178
|
+
var FS_FOLDER_KV_REPOSITORY = createServiceToken15("storage.kvRepository.fsFolder");
|
|
1993
2179
|
|
|
1994
2180
|
class FsFolderKvRepository extends KvRepository {
|
|
1995
2181
|
folderPath;
|
|
@@ -2066,8 +2252,8 @@ class FsFolderKvRepository extends KvRepository {
|
|
|
2066
2252
|
}
|
|
2067
2253
|
}
|
|
2068
2254
|
// src/kv/PostgresKvRepository.ts
|
|
2069
|
-
import { createServiceToken as
|
|
2070
|
-
var POSTGRES_KV_REPOSITORY =
|
|
2255
|
+
import { createServiceToken as createServiceToken16 } from "@workglow/util";
|
|
2256
|
+
var POSTGRES_KV_REPOSITORY = createServiceToken16("storage.kvRepository.postgres");
|
|
2071
2257
|
|
|
2072
2258
|
class PostgresKvRepository extends KvViaTabularRepository {
|
|
2073
2259
|
db;
|
|
@@ -2081,8 +2267,8 @@ class PostgresKvRepository extends KvViaTabularRepository {
|
|
|
2081
2267
|
}
|
|
2082
2268
|
}
|
|
2083
2269
|
// src/kv/SqliteKvRepository.ts
|
|
2084
|
-
import { createServiceToken as
|
|
2085
|
-
var SQLITE_KV_REPOSITORY =
|
|
2270
|
+
import { createServiceToken as createServiceToken17 } from "@workglow/util";
|
|
2271
|
+
var SQLITE_KV_REPOSITORY = createServiceToken17("storage.kvRepository.sqlite");
|
|
2086
2272
|
|
|
2087
2273
|
class SqliteKvRepository extends KvViaTabularRepository {
|
|
2088
2274
|
db;
|
|
@@ -2096,8 +2282,8 @@ class SqliteKvRepository extends KvViaTabularRepository {
|
|
|
2096
2282
|
}
|
|
2097
2283
|
}
|
|
2098
2284
|
// src/kv/SupabaseKvRepository.ts
|
|
2099
|
-
import { createServiceToken as
|
|
2100
|
-
var SUPABASE_KV_REPOSITORY =
|
|
2285
|
+
import { createServiceToken as createServiceToken18 } from "@workglow/util";
|
|
2286
|
+
var SUPABASE_KV_REPOSITORY = createServiceToken18("storage.kvRepository.supabase");
|
|
2101
2287
|
|
|
2102
2288
|
class SupabaseKvRepository extends KvViaTabularRepository {
|
|
2103
2289
|
client;
|
|
@@ -2111,15 +2297,169 @@ class SupabaseKvRepository extends KvViaTabularRepository {
|
|
|
2111
2297
|
}
|
|
2112
2298
|
}
|
|
2113
2299
|
// src/queue/PostgresQueueStorage.ts
|
|
2114
|
-
import { createServiceToken as
|
|
2115
|
-
|
|
2300
|
+
import { createServiceToken as createServiceToken19, makeFingerprint as makeFingerprint5, uuid4 as uuid42 } from "@workglow/util";
|
|
2301
|
+
|
|
2302
|
+
// src/util/PollingSubscriptionManager.ts
|
|
2303
|
+
class PollingSubscriptionManager {
|
|
2304
|
+
intervals = new Map;
|
|
2305
|
+
lastKnownState = new Map;
|
|
2306
|
+
initialized = false;
|
|
2307
|
+
fetchState;
|
|
2308
|
+
compareItems;
|
|
2309
|
+
payloadFactory;
|
|
2310
|
+
defaultIntervalMs;
|
|
2311
|
+
constructor(fetchState, compareItems, payloadFactory, options) {
|
|
2312
|
+
this.fetchState = fetchState;
|
|
2313
|
+
this.compareItems = compareItems;
|
|
2314
|
+
this.payloadFactory = payloadFactory;
|
|
2315
|
+
this.defaultIntervalMs = options?.defaultIntervalMs ?? 1000;
|
|
2316
|
+
}
|
|
2317
|
+
subscribe(callback, options) {
|
|
2318
|
+
const interval = options?.intervalMs ?? this.defaultIntervalMs;
|
|
2319
|
+
const subscription = {
|
|
2320
|
+
callback,
|
|
2321
|
+
intervalMs: interval
|
|
2322
|
+
};
|
|
2323
|
+
let intervalGroup = this.intervals.get(interval);
|
|
2324
|
+
if (!intervalGroup) {
|
|
2325
|
+
const subscribers = new Set;
|
|
2326
|
+
const intervalId = setInterval(() => this.poll(subscribers), interval);
|
|
2327
|
+
intervalGroup = { intervalId, subscribers };
|
|
2328
|
+
this.intervals.set(interval, intervalGroup);
|
|
2329
|
+
if (!this.initialized) {
|
|
2330
|
+
this.initialized = true;
|
|
2331
|
+
this.initAndPoll(subscribers, subscription);
|
|
2332
|
+
} else {
|
|
2333
|
+
this.pollForNewSubscriber(subscription);
|
|
2334
|
+
}
|
|
2335
|
+
} else {
|
|
2336
|
+
this.pollForNewSubscriber(subscription);
|
|
2337
|
+
}
|
|
2338
|
+
intervalGroup.subscribers.add(subscription);
|
|
2339
|
+
return () => {
|
|
2340
|
+
const group = this.intervals.get(interval);
|
|
2341
|
+
if (group) {
|
|
2342
|
+
group.subscribers.delete(subscription);
|
|
2343
|
+
if (group.subscribers.size === 0) {
|
|
2344
|
+
clearInterval(group.intervalId);
|
|
2345
|
+
this.intervals.delete(interval);
|
|
2346
|
+
}
|
|
2347
|
+
}
|
|
2348
|
+
};
|
|
2349
|
+
}
|
|
2350
|
+
async initAndPoll(subscribers, newSubscription) {
|
|
2351
|
+
try {
|
|
2352
|
+
this.lastKnownState = await this.fetchState();
|
|
2353
|
+
for (const [, item] of this.lastKnownState) {
|
|
2354
|
+
const payload = this.payloadFactory.insert(item);
|
|
2355
|
+
try {
|
|
2356
|
+
newSubscription.callback(payload);
|
|
2357
|
+
} catch {}
|
|
2358
|
+
}
|
|
2359
|
+
} catch {}
|
|
2360
|
+
}
|
|
2361
|
+
pollForNewSubscriber(subscription) {
|
|
2362
|
+
for (const [, item] of this.lastKnownState) {
|
|
2363
|
+
const payload = this.payloadFactory.insert(item);
|
|
2364
|
+
try {
|
|
2365
|
+
subscription.callback(payload);
|
|
2366
|
+
} catch {}
|
|
2367
|
+
}
|
|
2368
|
+
}
|
|
2369
|
+
async poll(subscribers) {
|
|
2370
|
+
if (subscribers.size === 0)
|
|
2371
|
+
return;
|
|
2372
|
+
try {
|
|
2373
|
+
const currentState = await this.fetchState();
|
|
2374
|
+
const changes = [];
|
|
2375
|
+
for (const [key, item] of currentState) {
|
|
2376
|
+
const oldItem = this.lastKnownState.get(key);
|
|
2377
|
+
if (!oldItem) {
|
|
2378
|
+
changes.push(this.payloadFactory.insert(item));
|
|
2379
|
+
} else if (!this.compareItems(oldItem, item)) {
|
|
2380
|
+
changes.push(this.payloadFactory.update(oldItem, item));
|
|
2381
|
+
}
|
|
2382
|
+
}
|
|
2383
|
+
for (const [key, item] of this.lastKnownState) {
|
|
2384
|
+
if (!currentState.has(key)) {
|
|
2385
|
+
changes.push(this.payloadFactory.delete(item));
|
|
2386
|
+
}
|
|
2387
|
+
}
|
|
2388
|
+
this.lastKnownState = currentState;
|
|
2389
|
+
for (const change of changes) {
|
|
2390
|
+
for (const sub of subscribers) {
|
|
2391
|
+
try {
|
|
2392
|
+
sub.callback(change);
|
|
2393
|
+
} catch {}
|
|
2394
|
+
}
|
|
2395
|
+
}
|
|
2396
|
+
} catch {}
|
|
2397
|
+
}
|
|
2398
|
+
get subscriptionCount() {
|
|
2399
|
+
let count = 0;
|
|
2400
|
+
for (const group of this.intervals.values()) {
|
|
2401
|
+
count += group.subscribers.size;
|
|
2402
|
+
}
|
|
2403
|
+
return count;
|
|
2404
|
+
}
|
|
2405
|
+
get hasSubscriptions() {
|
|
2406
|
+
return this.intervals.size > 0;
|
|
2407
|
+
}
|
|
2408
|
+
destroy() {
|
|
2409
|
+
for (const group of this.intervals.values()) {
|
|
2410
|
+
clearInterval(group.intervalId);
|
|
2411
|
+
}
|
|
2412
|
+
this.intervals.clear();
|
|
2413
|
+
this.lastKnownState.clear();
|
|
2414
|
+
this.initialized = false;
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
|
|
2418
|
+
// src/queue/PostgresQueueStorage.ts
|
|
2419
|
+
var POSTGRES_QUEUE_STORAGE = createServiceToken19("jobqueue.storage.postgres");
|
|
2116
2420
|
|
|
2117
2421
|
class PostgresQueueStorage {
|
|
2118
2422
|
db;
|
|
2119
2423
|
queueName;
|
|
2120
|
-
|
|
2424
|
+
prefixes;
|
|
2425
|
+
prefixValues;
|
|
2426
|
+
tableName;
|
|
2427
|
+
pollingManager = null;
|
|
2428
|
+
constructor(db, queueName, options) {
|
|
2121
2429
|
this.db = db;
|
|
2122
2430
|
this.queueName = queueName;
|
|
2431
|
+
this.prefixes = options?.prefixes ?? [];
|
|
2432
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
2433
|
+
if (this.prefixes.length > 0) {
|
|
2434
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
2435
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
2436
|
+
} else {
|
|
2437
|
+
this.tableName = "job_queue";
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
getPrefixColumnType(type) {
|
|
2441
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
2442
|
+
}
|
|
2443
|
+
buildPrefixColumnsSql() {
|
|
2444
|
+
if (this.prefixes.length === 0)
|
|
2445
|
+
return "";
|
|
2446
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
2447
|
+
`) + `,
|
|
2448
|
+
`;
|
|
2449
|
+
}
|
|
2450
|
+
getPrefixColumnNames() {
|
|
2451
|
+
return this.prefixes.map((p) => p.name);
|
|
2452
|
+
}
|
|
2453
|
+
buildPrefixWhereClause(startParam) {
|
|
2454
|
+
if (this.prefixes.length === 0) {
|
|
2455
|
+
return { conditions: "", params: [] };
|
|
2456
|
+
}
|
|
2457
|
+
const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
|
|
2458
|
+
const params = this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2459
|
+
return { conditions: " AND " + conditions, params };
|
|
2460
|
+
}
|
|
2461
|
+
getPrefixParamValues() {
|
|
2462
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2123
2463
|
}
|
|
2124
2464
|
async setupDatabase() {
|
|
2125
2465
|
let sql;
|
|
@@ -2130,10 +2470,13 @@ class PostgresQueueStorage {
|
|
|
2130
2470
|
if (e.code !== "42710")
|
|
2131
2471
|
throw e;
|
|
2132
2472
|
}
|
|
2473
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
2474
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2475
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2133
2476
|
sql = `
|
|
2134
|
-
CREATE TABLE IF NOT EXISTS
|
|
2477
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
2135
2478
|
id SERIAL NOT NULL,
|
|
2136
|
-
fingerprint text NOT NULL,
|
|
2479
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
2137
2480
|
queue text NOT NULL,
|
|
2138
2481
|
job_run_id text NOT NULL,
|
|
2139
2482
|
status job_status NOT NULL default 'PENDING',
|
|
@@ -2150,20 +2493,22 @@ class PostgresQueueStorage {
|
|
|
2150
2493
|
error_code text,
|
|
2151
2494
|
progress real DEFAULT 0,
|
|
2152
2495
|
progress_message text DEFAULT '',
|
|
2153
|
-
progress_details jsonb
|
|
2496
|
+
progress_details jsonb,
|
|
2497
|
+
worker_id text
|
|
2154
2498
|
)`;
|
|
2155
2499
|
await this.db.query(sql);
|
|
2500
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
2156
2501
|
sql = `
|
|
2157
|
-
CREATE INDEX IF NOT EXISTS
|
|
2158
|
-
ON
|
|
2502
|
+
CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx
|
|
2503
|
+
ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`;
|
|
2159
2504
|
await this.db.query(sql);
|
|
2160
2505
|
sql = `
|
|
2161
|
-
CREATE INDEX IF NOT EXISTS
|
|
2162
|
-
ON
|
|
2506
|
+
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx
|
|
2507
|
+
ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`;
|
|
2163
2508
|
await this.db.query(sql);
|
|
2164
2509
|
sql = `
|
|
2165
|
-
CREATE INDEX IF NOT EXISTS
|
|
2166
|
-
ON
|
|
2510
|
+
CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx
|
|
2511
|
+
ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`;
|
|
2167
2512
|
await this.db.query(sql);
|
|
2168
2513
|
}
|
|
2169
2514
|
async add(job) {
|
|
@@ -2177,9 +2522,14 @@ class PostgresQueueStorage {
|
|
|
2177
2522
|
job.progress_details = null;
|
|
2178
2523
|
job.created_at = now;
|
|
2179
2524
|
job.run_after = now;
|
|
2525
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2526
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2527
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
2528
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(",") + "," : "";
|
|
2529
|
+
const baseParamStart = prefixColumnNames.length + 1;
|
|
2180
2530
|
const sql = `
|
|
2181
|
-
INSERT INTO
|
|
2182
|
-
queue,
|
|
2531
|
+
INSERT INTO ${this.tableName}(
|
|
2532
|
+
${prefixColumnsInsert}queue,
|
|
2183
2533
|
fingerprint,
|
|
2184
2534
|
input,
|
|
2185
2535
|
run_after,
|
|
@@ -2192,9 +2542,10 @@ class PostgresQueueStorage {
|
|
|
2192
2542
|
progress_details
|
|
2193
2543
|
)
|
|
2194
2544
|
VALUES
|
|
2195
|
-
($1
|
|
2545
|
+
(${prefixParamPlaceholders}$${baseParamStart},$${baseParamStart + 1},$${baseParamStart + 2},$${baseParamStart + 3},$${baseParamStart + 4},$${baseParamStart + 5},$${baseParamStart + 6},$${baseParamStart + 7},$${baseParamStart + 8},$${baseParamStart + 9},$${baseParamStart + 10})
|
|
2196
2546
|
RETURNING id`;
|
|
2197
2547
|
const params = [
|
|
2548
|
+
...prefixParamValues,
|
|
2198
2549
|
job.queue,
|
|
2199
2550
|
job.fingerprint,
|
|
2200
2551
|
JSON.stringify(job.input),
|
|
@@ -2214,68 +2565,75 @@ class PostgresQueueStorage {
|
|
|
2214
2565
|
return job.id;
|
|
2215
2566
|
}
|
|
2216
2567
|
async get(id) {
|
|
2568
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2217
2569
|
const result = await this.db.query(`SELECT *
|
|
2218
|
-
FROM
|
|
2219
|
-
WHERE id = $1 AND queue = $2
|
|
2570
|
+
FROM ${this.tableName}
|
|
2571
|
+
WHERE id = $1 AND queue = $2${prefixConditions}
|
|
2220
2572
|
FOR UPDATE SKIP LOCKED
|
|
2221
|
-
LIMIT 1`, [id, this.queueName]);
|
|
2573
|
+
LIMIT 1`, [id, this.queueName, ...prefixParams]);
|
|
2222
2574
|
if (!result || result.rows.length === 0)
|
|
2223
2575
|
return;
|
|
2224
2576
|
return result.rows[0];
|
|
2225
2577
|
}
|
|
2226
2578
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2227
2579
|
num = Number(num) || 100;
|
|
2580
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
2228
2581
|
const result = await this.db.query(`
|
|
2229
2582
|
SELECT *
|
|
2230
|
-
FROM
|
|
2583
|
+
FROM ${this.tableName}
|
|
2231
2584
|
WHERE queue = $1
|
|
2232
|
-
AND status = $2
|
|
2585
|
+
AND status = $2${prefixConditions}
|
|
2233
2586
|
ORDER BY run_after ASC
|
|
2234
2587
|
LIMIT $3
|
|
2235
|
-
FOR UPDATE SKIP LOCKED`, [this.queueName, status, num]);
|
|
2588
|
+
FOR UPDATE SKIP LOCKED`, [this.queueName, status, num, ...prefixParams]);
|
|
2236
2589
|
if (!result)
|
|
2237
2590
|
return [];
|
|
2238
2591
|
return result.rows;
|
|
2239
2592
|
}
|
|
2240
|
-
async next() {
|
|
2593
|
+
async next(workerId) {
|
|
2594
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
|
|
2241
2595
|
const result = await this.db.query(`
|
|
2242
|
-
UPDATE
|
|
2243
|
-
SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2596
|
+
UPDATE ${this.tableName}
|
|
2597
|
+
SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = $4
|
|
2244
2598
|
WHERE id = (
|
|
2245
2599
|
SELECT id
|
|
2246
|
-
FROM
|
|
2600
|
+
FROM ${this.tableName}
|
|
2247
2601
|
WHERE queue = $2
|
|
2248
|
-
AND status = $3
|
|
2602
|
+
AND status = $3${prefixConditions}
|
|
2249
2603
|
AND run_after <= NOW() AT TIME ZONE 'UTC'
|
|
2250
2604
|
ORDER BY run_after ASC
|
|
2251
2605
|
FOR UPDATE SKIP LOCKED
|
|
2252
2606
|
LIMIT 1
|
|
2253
2607
|
)
|
|
2254
|
-
RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING
|
|
2608
|
+
RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING */, workerId ?? null, ...prefixParams]);
|
|
2255
2609
|
return result?.rows?.[0] ?? undefined;
|
|
2256
2610
|
}
|
|
2257
2611
|
async size(status = "PENDING" /* PENDING */) {
|
|
2612
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2258
2613
|
const result = await this.db.query(`
|
|
2259
2614
|
SELECT COUNT(*) as count
|
|
2260
|
-
FROM
|
|
2615
|
+
FROM ${this.tableName}
|
|
2261
2616
|
WHERE queue = $1
|
|
2262
|
-
AND status = $2`, [this.queueName, status]);
|
|
2617
|
+
AND status = $2${prefixConditions}`, [this.queueName, status, ...prefixParams]);
|
|
2263
2618
|
if (!result)
|
|
2264
2619
|
return 0;
|
|
2265
2620
|
return parseInt(result.rows[0].count, 10);
|
|
2266
2621
|
}
|
|
2267
2622
|
async complete(jobDetails) {
|
|
2623
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2268
2624
|
if (jobDetails.status === "DISABLED" /* DISABLED */) {
|
|
2269
|
-
|
|
2625
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(4);
|
|
2626
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
2270
2627
|
SET
|
|
2271
2628
|
status = $1,
|
|
2272
2629
|
progress = 100,
|
|
2273
2630
|
progress_message = '',
|
|
2274
2631
|
progress_details = NULL,
|
|
2275
2632
|
completed_at = NOW() AT TIME ZONE 'UTC'
|
|
2276
|
-
WHERE id = $2 AND queue = $3`, [jobDetails.status, jobDetails.id, this.queueName]);
|
|
2633
|
+
WHERE id = $2 AND queue = $3${prefixConditions}`, [jobDetails.status, jobDetails.id, this.queueName, ...prefixParams]);
|
|
2277
2634
|
} else if (jobDetails.status === "PENDING" /* PENDING */) {
|
|
2278
|
-
|
|
2635
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
|
|
2636
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
2279
2637
|
SET
|
|
2280
2638
|
error = $1,
|
|
2281
2639
|
error_code = $2,
|
|
@@ -2286,17 +2644,19 @@ class PostgresQueueStorage {
|
|
|
2286
2644
|
progress_details = NULL,
|
|
2287
2645
|
run_attempts = run_attempts + 1,
|
|
2288
2646
|
last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2289
|
-
WHERE id = $5 AND queue = $6`, [
|
|
2647
|
+
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
2290
2648
|
jobDetails.error,
|
|
2291
2649
|
jobDetails.error_code,
|
|
2292
2650
|
jobDetails.status,
|
|
2293
2651
|
jobDetails.run_after,
|
|
2294
2652
|
jobDetails.id,
|
|
2295
|
-
this.queueName
|
|
2653
|
+
this.queueName,
|
|
2654
|
+
...prefixParams
|
|
2296
2655
|
]);
|
|
2297
2656
|
} else {
|
|
2657
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
|
|
2298
2658
|
await this.db.query(`
|
|
2299
|
-
UPDATE
|
|
2659
|
+
UPDATE ${this.tableName}
|
|
2300
2660
|
SET
|
|
2301
2661
|
output = $1,
|
|
2302
2662
|
error = $2,
|
|
@@ -2308,86 +2668,231 @@ class PostgresQueueStorage {
|
|
|
2308
2668
|
run_attempts = run_attempts + 1,
|
|
2309
2669
|
completed_at = NOW() AT TIME ZONE 'UTC',
|
|
2310
2670
|
last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2311
|
-
WHERE id = $5 AND queue = $6`, [
|
|
2671
|
+
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
2312
2672
|
jobDetails.output ? JSON.stringify(jobDetails.output) : null,
|
|
2313
2673
|
jobDetails.error ?? null,
|
|
2314
2674
|
jobDetails.error_code ?? null,
|
|
2315
2675
|
jobDetails.status,
|
|
2316
2676
|
jobDetails.id,
|
|
2317
|
-
this.queueName
|
|
2677
|
+
this.queueName,
|
|
2678
|
+
...prefixParams
|
|
2318
2679
|
]);
|
|
2319
2680
|
}
|
|
2320
2681
|
}
|
|
2321
2682
|
async deleteAll() {
|
|
2683
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
2322
2684
|
await this.db.query(`
|
|
2323
|
-
DELETE FROM
|
|
2324
|
-
WHERE queue = $1`, [this.queueName]);
|
|
2685
|
+
DELETE FROM ${this.tableName}
|
|
2686
|
+
WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
|
|
2325
2687
|
}
|
|
2326
2688
|
async outputForInput(input) {
|
|
2327
2689
|
const fingerprint = await makeFingerprint5(input);
|
|
2690
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2328
2691
|
const result = await this.db.query(`
|
|
2329
2692
|
SELECT output
|
|
2330
|
-
FROM
|
|
2331
|
-
WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'`, [fingerprint, this.queueName]);
|
|
2332
|
-
if (!result)
|
|
2693
|
+
FROM ${this.tableName}
|
|
2694
|
+
WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'${prefixConditions}`, [fingerprint, this.queueName, ...prefixParams]);
|
|
2695
|
+
if (!result || result.rows.length === 0)
|
|
2333
2696
|
return null;
|
|
2334
2697
|
return result.rows[0].output;
|
|
2335
2698
|
}
|
|
2336
2699
|
async abort(jobId) {
|
|
2337
|
-
const
|
|
2338
|
-
|
|
2700
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2701
|
+
await this.db.query(`
|
|
2702
|
+
UPDATE ${this.tableName}
|
|
2339
2703
|
SET status = 'ABORTING'
|
|
2340
|
-
WHERE id = $1 AND queue = $2`, [jobId, this.queueName]);
|
|
2704
|
+
WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
|
|
2341
2705
|
}
|
|
2342
2706
|
async getByRunId(job_run_id) {
|
|
2707
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2343
2708
|
const result = await this.db.query(`
|
|
2344
|
-
SELECT * FROM
|
|
2709
|
+
SELECT * FROM ${this.tableName} WHERE job_run_id = $1 AND queue = $2${prefixConditions}`, [job_run_id, this.queueName, ...prefixParams]);
|
|
2345
2710
|
if (!result)
|
|
2346
2711
|
return [];
|
|
2347
2712
|
return result.rows;
|
|
2348
2713
|
}
|
|
2349
2714
|
async saveProgress(jobId, progress, message, details) {
|
|
2715
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(6);
|
|
2350
2716
|
await this.db.query(`
|
|
2351
|
-
UPDATE
|
|
2717
|
+
UPDATE ${this.tableName}
|
|
2352
2718
|
SET progress = $1,
|
|
2353
2719
|
progress_message = $2,
|
|
2354
2720
|
progress_details = $3
|
|
2355
|
-
WHERE id = $4 AND queue = $5`, [
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2721
|
+
WHERE id = $4 AND queue = $5${prefixConditions}`, [
|
|
2722
|
+
progress,
|
|
2723
|
+
message,
|
|
2724
|
+
details ? JSON.stringify(details) : null,
|
|
2359
2725
|
jobId,
|
|
2360
|
-
this.queueName
|
|
2726
|
+
this.queueName,
|
|
2727
|
+
...prefixParams
|
|
2361
2728
|
]);
|
|
2362
2729
|
}
|
|
2730
|
+
async delete(jobId) {
|
|
2731
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2732
|
+
await this.db.query(`DELETE FROM ${this.tableName} WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
|
|
2733
|
+
}
|
|
2363
2734
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2364
2735
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
2365
|
-
|
|
2736
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
2737
|
+
await this.db.query(`DELETE FROM ${this.tableName}
|
|
2366
2738
|
WHERE queue = $1
|
|
2367
2739
|
AND status = $2
|
|
2368
2740
|
AND completed_at IS NOT NULL
|
|
2369
|
-
AND completed_at <= $3`, [this.queueName, status, cutoffDate]);
|
|
2741
|
+
AND completed_at <= $3${prefixConditions}`, [this.queueName, status, cutoffDate, ...prefixParams]);
|
|
2742
|
+
}
|
|
2743
|
+
async getAllJobs() {
|
|
2744
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
2745
|
+
const result = await this.db.query(`SELECT * FROM ${this.tableName} WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
|
|
2746
|
+
if (!result)
|
|
2747
|
+
return [];
|
|
2748
|
+
return result.rows;
|
|
2749
|
+
}
|
|
2750
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
2751
|
+
const filterEntries = Object.entries(prefixFilter);
|
|
2752
|
+
let query = `SELECT * FROM ${this.tableName} WHERE queue = $1`;
|
|
2753
|
+
const params = [this.queueName];
|
|
2754
|
+
filterEntries.forEach(([key, value], index) => {
|
|
2755
|
+
query += ` AND ${key} = $${index + 2}`;
|
|
2756
|
+
params.push(value);
|
|
2757
|
+
});
|
|
2758
|
+
const result = await this.db.query(query, params);
|
|
2759
|
+
if (!result)
|
|
2760
|
+
return [];
|
|
2761
|
+
return result.rows;
|
|
2762
|
+
}
|
|
2763
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
2764
|
+
if (prefixFilter === undefined) {
|
|
2765
|
+
return false;
|
|
2766
|
+
}
|
|
2767
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
2768
|
+
return true;
|
|
2769
|
+
}
|
|
2770
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
2771
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
2772
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
2773
|
+
return true;
|
|
2774
|
+
}
|
|
2775
|
+
for (const key of instanceKeys) {
|
|
2776
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
2777
|
+
return true;
|
|
2778
|
+
}
|
|
2779
|
+
}
|
|
2780
|
+
return false;
|
|
2781
|
+
}
|
|
2782
|
+
getPollingManager() {
|
|
2783
|
+
if (!this.pollingManager) {
|
|
2784
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
2785
|
+
const jobs = await this.getAllJobs();
|
|
2786
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
2787
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
2788
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
2789
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
2790
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
2791
|
+
});
|
|
2792
|
+
}
|
|
2793
|
+
return this.pollingManager;
|
|
2794
|
+
}
|
|
2795
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
2796
|
+
let lastKnownJobs = new Map;
|
|
2797
|
+
let cancelled = false;
|
|
2798
|
+
const poll = async () => {
|
|
2799
|
+
if (cancelled)
|
|
2800
|
+
return;
|
|
2801
|
+
try {
|
|
2802
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
2803
|
+
if (cancelled)
|
|
2804
|
+
return;
|
|
2805
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
2806
|
+
for (const [id, job] of currentMap) {
|
|
2807
|
+
const old = lastKnownJobs.get(id);
|
|
2808
|
+
if (!old) {
|
|
2809
|
+
callback({ type: "INSERT", new: job });
|
|
2810
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
2811
|
+
callback({ type: "UPDATE", old, new: job });
|
|
2812
|
+
}
|
|
2813
|
+
}
|
|
2814
|
+
for (const [id, job] of lastKnownJobs) {
|
|
2815
|
+
if (!currentMap.has(id)) {
|
|
2816
|
+
callback({ type: "DELETE", old: job });
|
|
2817
|
+
}
|
|
2818
|
+
}
|
|
2819
|
+
lastKnownJobs = currentMap;
|
|
2820
|
+
} catch {}
|
|
2821
|
+
};
|
|
2822
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
2823
|
+
poll();
|
|
2824
|
+
return () => {
|
|
2825
|
+
cancelled = true;
|
|
2826
|
+
clearInterval(intervalId);
|
|
2827
|
+
};
|
|
2828
|
+
}
|
|
2829
|
+
subscribeToChanges(callback, options) {
|
|
2830
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
2831
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
2832
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
2833
|
+
}
|
|
2834
|
+
const manager = this.getPollingManager();
|
|
2835
|
+
return manager.subscribe(callback, { intervalMs });
|
|
2370
2836
|
}
|
|
2371
2837
|
}
|
|
2372
2838
|
// src/queue/SqliteQueueStorage.ts
|
|
2373
|
-
import { createServiceToken as
|
|
2374
|
-
var SQLITE_QUEUE_STORAGE =
|
|
2839
|
+
import { createServiceToken as createServiceToken20, makeFingerprint as makeFingerprint6, sleep as sleep4, uuid4 as uuid43 } from "@workglow/util";
|
|
2840
|
+
var SQLITE_QUEUE_STORAGE = createServiceToken20("jobqueue.storage.sqlite");
|
|
2375
2841
|
|
|
2376
2842
|
class SqliteQueueStorage {
|
|
2377
2843
|
db;
|
|
2378
2844
|
queueName;
|
|
2379
2845
|
options;
|
|
2846
|
+
prefixes;
|
|
2847
|
+
prefixValues;
|
|
2848
|
+
tableName;
|
|
2849
|
+
pollingManager = null;
|
|
2380
2850
|
constructor(db, queueName, options) {
|
|
2381
2851
|
this.db = db;
|
|
2382
2852
|
this.queueName = queueName;
|
|
2383
2853
|
this.options = options;
|
|
2854
|
+
this.prefixes = options?.prefixes ?? [];
|
|
2855
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
2856
|
+
if (this.prefixes.length > 0) {
|
|
2857
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
2858
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
2859
|
+
} else {
|
|
2860
|
+
this.tableName = "job_queue";
|
|
2861
|
+
}
|
|
2862
|
+
}
|
|
2863
|
+
getPrefixColumnType(type) {
|
|
2864
|
+
return type === "uuid" ? "TEXT" : "INTEGER";
|
|
2865
|
+
}
|
|
2866
|
+
buildPrefixColumnsSql() {
|
|
2867
|
+
if (this.prefixes.length === 0)
|
|
2868
|
+
return "";
|
|
2869
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
2870
|
+
`) + `,
|
|
2871
|
+
`;
|
|
2872
|
+
}
|
|
2873
|
+
getPrefixColumnNames() {
|
|
2874
|
+
return this.prefixes.map((p) => p.name);
|
|
2875
|
+
}
|
|
2876
|
+
buildPrefixWhereClause() {
|
|
2877
|
+
if (this.prefixes.length === 0) {
|
|
2878
|
+
return "";
|
|
2879
|
+
}
|
|
2880
|
+
const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
|
|
2881
|
+
return " AND " + conditions;
|
|
2882
|
+
}
|
|
2883
|
+
getPrefixParamValues() {
|
|
2884
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2384
2885
|
}
|
|
2385
2886
|
async setupDatabase() {
|
|
2386
|
-
await
|
|
2887
|
+
await sleep4(0);
|
|
2888
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
2889
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2890
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2891
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
2387
2892
|
this.db.exec(`
|
|
2388
|
-
CREATE TABLE IF NOT EXISTS
|
|
2893
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
2389
2894
|
id INTEGER PRIMARY KEY,
|
|
2390
|
-
fingerprint text NOT NULL,
|
|
2895
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
2391
2896
|
queue text NOT NULL,
|
|
2392
2897
|
job_run_id text NOT NULL,
|
|
2393
2898
|
status TEXT NOT NULL default 'PENDING',
|
|
@@ -2404,12 +2909,13 @@ class SqliteQueueStorage {
|
|
|
2404
2909
|
error_code TEXT,
|
|
2405
2910
|
progress REAL DEFAULT 0,
|
|
2406
2911
|
progress_message TEXT DEFAULT '',
|
|
2407
|
-
progress_details TEXT NULL
|
|
2912
|
+
progress_details TEXT NULL,
|
|
2913
|
+
worker_id TEXT
|
|
2408
2914
|
);
|
|
2409
2915
|
|
|
2410
|
-
CREATE INDEX IF NOT EXISTS
|
|
2411
|
-
CREATE INDEX IF NOT EXISTS
|
|
2412
|
-
CREATE INDEX IF NOT EXISTS
|
|
2916
|
+
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after);
|
|
2917
|
+
CREATE INDEX IF NOT EXISTS job_queue_fingerprint${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status);
|
|
2918
|
+
CREATE INDEX IF NOT EXISTS job_queue_job_run_id${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, job_run_id);
|
|
2413
2919
|
`);
|
|
2414
2920
|
}
|
|
2415
2921
|
async add(job) {
|
|
@@ -2423,9 +2929,13 @@ class SqliteQueueStorage {
|
|
|
2423
2929
|
job.progress_details = null;
|
|
2424
2930
|
job.created_at = now;
|
|
2425
2931
|
job.run_after = now;
|
|
2932
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2933
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2934
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
2935
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
2426
2936
|
const AddQuery = `
|
|
2427
|
-
INSERT INTO
|
|
2428
|
-
queue,
|
|
2937
|
+
INSERT INTO ${this.tableName}(
|
|
2938
|
+
${prefixColumnsInsert}queue,
|
|
2429
2939
|
fingerprint,
|
|
2430
2940
|
input,
|
|
2431
2941
|
run_after,
|
|
@@ -2437,21 +2947,23 @@ class SqliteQueueStorage {
|
|
|
2437
2947
|
progress_details,
|
|
2438
2948
|
created_at
|
|
2439
2949
|
)
|
|
2440
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2950
|
+
VALUES (${prefixPlaceholders}?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2441
2951
|
RETURNING id`;
|
|
2442
2952
|
const stmt = this.db.prepare(AddQuery);
|
|
2443
|
-
const result = stmt.get(job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
|
|
2953
|
+
const result = stmt.get(...prefixParamValues, job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
|
|
2444
2954
|
job.id = result?.id;
|
|
2445
2955
|
return result?.id;
|
|
2446
2956
|
}
|
|
2447
2957
|
async get(id) {
|
|
2958
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
2959
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2448
2960
|
const JobQuery = `
|
|
2449
2961
|
SELECT *
|
|
2450
|
-
FROM
|
|
2451
|
-
WHERE id = ? AND queue =
|
|
2962
|
+
FROM ${this.tableName}
|
|
2963
|
+
WHERE id = ? AND queue = ?${prefixConditions}
|
|
2452
2964
|
LIMIT 1`;
|
|
2453
2965
|
const stmt = this.db.prepare(JobQuery);
|
|
2454
|
-
const result = stmt.get(id, this.queueName);
|
|
2966
|
+
const result = stmt.get(String(id), this.queueName, ...prefixParams);
|
|
2455
2967
|
if (!result)
|
|
2456
2968
|
return;
|
|
2457
2969
|
if (result.input)
|
|
@@ -2464,15 +2976,17 @@ class SqliteQueueStorage {
|
|
|
2464
2976
|
}
|
|
2465
2977
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2466
2978
|
num = Number(num) || 100;
|
|
2979
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
2980
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2467
2981
|
const FutureJobQuery = `
|
|
2468
2982
|
SELECT *
|
|
2469
|
-
FROM
|
|
2983
|
+
FROM ${this.tableName}
|
|
2470
2984
|
WHERE queue = ?
|
|
2471
|
-
AND status =
|
|
2985
|
+
AND status = ?${prefixConditions}
|
|
2472
2986
|
ORDER BY run_after ASC
|
|
2473
2987
|
LIMIT ${num}`;
|
|
2474
2988
|
const stmt = this.db.prepare(FutureJobQuery);
|
|
2475
|
-
const result = stmt.all(this.queueName, status);
|
|
2989
|
+
const result = stmt.all(this.queueName, status, ...prefixParams);
|
|
2476
2990
|
return (result || []).map((details) => {
|
|
2477
2991
|
if (details.input)
|
|
2478
2992
|
details.input = JSON.parse(details.input);
|
|
@@ -2484,20 +2998,24 @@ class SqliteQueueStorage {
|
|
|
2484
2998
|
});
|
|
2485
2999
|
}
|
|
2486
3000
|
async abort(jobId) {
|
|
3001
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3002
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2487
3003
|
const AbortQuery = `
|
|
2488
|
-
UPDATE
|
|
3004
|
+
UPDATE ${this.tableName}
|
|
2489
3005
|
SET status = ?
|
|
2490
|
-
WHERE id = ? AND queue =
|
|
3006
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2491
3007
|
const stmt = this.db.prepare(AbortQuery);
|
|
2492
|
-
stmt.run("ABORTING" /* ABORTING */, jobId, this.queueName);
|
|
3008
|
+
stmt.run("ABORTING" /* ABORTING */, String(jobId), this.queueName, ...prefixParams);
|
|
2493
3009
|
}
|
|
2494
3010
|
async getByRunId(job_run_id) {
|
|
3011
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3012
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2495
3013
|
const JobsByRunIdQuery = `
|
|
2496
3014
|
SELECT *
|
|
2497
|
-
FROM
|
|
2498
|
-
WHERE job_run_id = ? AND queue =
|
|
3015
|
+
FROM ${this.tableName}
|
|
3016
|
+
WHERE job_run_id = ? AND queue = ?${prefixConditions}`;
|
|
2499
3017
|
const stmt = this.db.prepare(JobsByRunIdQuery);
|
|
2500
|
-
const result = stmt.all(job_run_id, this.queueName);
|
|
3018
|
+
const result = stmt.all(job_run_id, this.queueName, ...prefixParams);
|
|
2501
3019
|
return (result || []).map((details) => {
|
|
2502
3020
|
if (details.input)
|
|
2503
3021
|
details.input = JSON.parse(details.input);
|
|
@@ -2508,22 +3026,24 @@ class SqliteQueueStorage {
|
|
|
2508
3026
|
return details;
|
|
2509
3027
|
});
|
|
2510
3028
|
}
|
|
2511
|
-
async next() {
|
|
3029
|
+
async next(workerId) {
|
|
2512
3030
|
const now = new Date().toISOString();
|
|
3031
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3032
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2513
3033
|
const stmt = this.db.prepare(`
|
|
2514
|
-
UPDATE
|
|
2515
|
-
SET status = ?, last_ran_at = ?
|
|
3034
|
+
UPDATE ${this.tableName}
|
|
3035
|
+
SET status = ?, last_ran_at = ?, worker_id = ?
|
|
2516
3036
|
WHERE id = (
|
|
2517
3037
|
SELECT id
|
|
2518
|
-
FROM
|
|
3038
|
+
FROM ${this.tableName}
|
|
2519
3039
|
WHERE queue = ?
|
|
2520
|
-
AND status =
|
|
3040
|
+
AND status = ?${prefixConditions}
|
|
2521
3041
|
AND run_after <= ?
|
|
2522
3042
|
ORDER BY run_after ASC
|
|
2523
3043
|
LIMIT 1
|
|
2524
3044
|
)
|
|
2525
3045
|
RETURNING *`);
|
|
2526
|
-
const result = stmt.get("PROCESSING" /* PROCESSING */, now, this.queueName, "PENDING" /* PENDING */, now);
|
|
3046
|
+
const result = stmt.get("PROCESSING" /* PROCESSING */, now, workerId ?? null, this.queueName, "PENDING" /* PENDING */, ...prefixParams, now);
|
|
2527
3047
|
if (!result)
|
|
2528
3048
|
return;
|
|
2529
3049
|
if (result.input)
|
|
@@ -2535,33 +3055,37 @@ class SqliteQueueStorage {
|
|
|
2535
3055
|
return result;
|
|
2536
3056
|
}
|
|
2537
3057
|
async size(status = "PENDING" /* PENDING */) {
|
|
3058
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3059
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2538
3060
|
const sizeQuery = `
|
|
2539
3061
|
SELECT COUNT(*) as count
|
|
2540
|
-
FROM
|
|
3062
|
+
FROM ${this.tableName}
|
|
2541
3063
|
WHERE queue = ?
|
|
2542
|
-
AND status =
|
|
3064
|
+
AND status = ?${prefixConditions}`;
|
|
2543
3065
|
const stmt = this.db.prepare(sizeQuery);
|
|
2544
|
-
const result = stmt.get(this.queueName, status);
|
|
3066
|
+
const result = stmt.get(this.queueName, status, ...prefixParams);
|
|
2545
3067
|
return result.count;
|
|
2546
3068
|
}
|
|
2547
3069
|
async complete(job) {
|
|
2548
3070
|
const now = new Date().toISOString();
|
|
3071
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3072
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2549
3073
|
let updateQuery;
|
|
2550
3074
|
let params;
|
|
2551
3075
|
if (job.status === "DISABLED" /* DISABLED */) {
|
|
2552
3076
|
updateQuery = `
|
|
2553
|
-
UPDATE
|
|
3077
|
+
UPDATE ${this.tableName}
|
|
2554
3078
|
SET
|
|
2555
3079
|
status = ?,
|
|
2556
3080
|
progress = 100,
|
|
2557
3081
|
progress_message = '',
|
|
2558
3082
|
progress_details = NULL,
|
|
2559
3083
|
completed_at = ?
|
|
2560
|
-
WHERE id = ? AND queue =
|
|
2561
|
-
params = [job.status, now, job.id, this.queueName];
|
|
3084
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
3085
|
+
params = [job.status, now, job.id, this.queueName, ...prefixParams];
|
|
2562
3086
|
} else {
|
|
2563
3087
|
updateQuery = `
|
|
2564
|
-
UPDATE
|
|
3088
|
+
UPDATE ${this.tableName}
|
|
2565
3089
|
SET
|
|
2566
3090
|
output = ?,
|
|
2567
3091
|
error = ?,
|
|
@@ -2573,7 +3097,7 @@ class SqliteQueueStorage {
|
|
|
2573
3097
|
last_ran_at = ?,
|
|
2574
3098
|
completed_at = ?,
|
|
2575
3099
|
run_attempts = run_attempts + 1
|
|
2576
|
-
WHERE id = ? AND queue =
|
|
3100
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2577
3101
|
params = [
|
|
2578
3102
|
job.output ? JSON.stringify(job.output) : null,
|
|
2579
3103
|
job.error ?? null,
|
|
@@ -2582,68 +3106,227 @@ class SqliteQueueStorage {
|
|
|
2582
3106
|
now,
|
|
2583
3107
|
now,
|
|
2584
3108
|
job.id,
|
|
2585
|
-
this.queueName
|
|
3109
|
+
this.queueName,
|
|
3110
|
+
...prefixParams
|
|
2586
3111
|
];
|
|
2587
3112
|
}
|
|
2588
3113
|
const stmt = this.db.prepare(updateQuery);
|
|
2589
3114
|
stmt.run(...params);
|
|
2590
3115
|
}
|
|
2591
3116
|
async deleteAll() {
|
|
3117
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3118
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2592
3119
|
const ClearQuery = `
|
|
2593
|
-
DELETE FROM
|
|
2594
|
-
WHERE queue =
|
|
3120
|
+
DELETE FROM ${this.tableName}
|
|
3121
|
+
WHERE queue = ?${prefixConditions}`;
|
|
2595
3122
|
const stmt = this.db.prepare(ClearQuery);
|
|
2596
|
-
stmt.run(this.queueName);
|
|
3123
|
+
stmt.run(this.queueName, ...prefixParams);
|
|
2597
3124
|
}
|
|
2598
3125
|
async outputForInput(input) {
|
|
2599
3126
|
const fingerprint = await makeFingerprint6(input);
|
|
3127
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3128
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2600
3129
|
const OutputQuery = `
|
|
2601
3130
|
SELECT output
|
|
2602
|
-
FROM
|
|
2603
|
-
WHERE queue = ? AND fingerprint = ? AND status =
|
|
3131
|
+
FROM ${this.tableName}
|
|
3132
|
+
WHERE queue = ? AND fingerprint = ? AND status = ?${prefixConditions}`;
|
|
2604
3133
|
const stmt = this.db.prepare(OutputQuery);
|
|
2605
|
-
const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED
|
|
3134
|
+
const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED */, ...prefixParams);
|
|
2606
3135
|
return result?.output ? JSON.parse(result.output) : null;
|
|
2607
3136
|
}
|
|
2608
3137
|
async saveProgress(jobId, progress, message, details) {
|
|
3138
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3139
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2609
3140
|
const UpdateProgressQuery = `
|
|
2610
|
-
UPDATE
|
|
3141
|
+
UPDATE ${this.tableName}
|
|
2611
3142
|
SET progress = ?,
|
|
2612
3143
|
progress_message = ?,
|
|
2613
3144
|
progress_details = ?
|
|
2614
|
-
WHERE id = ? AND queue =
|
|
3145
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2615
3146
|
const stmt = this.db.prepare(UpdateProgressQuery);
|
|
2616
|
-
stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName);
|
|
3147
|
+
stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName, ...prefixParams);
|
|
2617
3148
|
}
|
|
2618
3149
|
async delete(jobId) {
|
|
3150
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3151
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2619
3152
|
const DeleteQuery = `
|
|
2620
|
-
DELETE FROM
|
|
2621
|
-
WHERE id = ? AND queue =
|
|
3153
|
+
DELETE FROM ${this.tableName}
|
|
3154
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2622
3155
|
const stmt = this.db.prepare(DeleteQuery);
|
|
2623
|
-
stmt.run(String(jobId), this.queueName);
|
|
3156
|
+
stmt.run(String(jobId), this.queueName, ...prefixParams);
|
|
2624
3157
|
}
|
|
2625
3158
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2626
3159
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
3160
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3161
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2627
3162
|
const DeleteQuery = `
|
|
2628
|
-
DELETE FROM
|
|
3163
|
+
DELETE FROM ${this.tableName}
|
|
2629
3164
|
WHERE queue = ?
|
|
2630
3165
|
AND status = ?
|
|
2631
3166
|
AND completed_at IS NOT NULL
|
|
2632
|
-
AND completed_at <=
|
|
3167
|
+
AND completed_at <= ?${prefixConditions}`;
|
|
2633
3168
|
const stmt = this.db.prepare(DeleteQuery);
|
|
2634
|
-
stmt.run(this.queueName, status, cutoffDate);
|
|
3169
|
+
stmt.run(this.queueName, status, cutoffDate, ...prefixParams);
|
|
3170
|
+
}
|
|
3171
|
+
getAllJobs() {
|
|
3172
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3173
|
+
const prefixParams = this.getPrefixParamValues();
|
|
3174
|
+
const AllJobsQuery = `
|
|
3175
|
+
SELECT *
|
|
3176
|
+
FROM ${this.tableName}
|
|
3177
|
+
WHERE queue = ?${prefixConditions}`;
|
|
3178
|
+
const stmt = this.db.prepare(AllJobsQuery);
|
|
3179
|
+
const result = stmt.all(this.queueName, ...prefixParams);
|
|
3180
|
+
return (result || []).map((details) => {
|
|
3181
|
+
if (details.input)
|
|
3182
|
+
details.input = JSON.parse(details.input);
|
|
3183
|
+
if (details.output)
|
|
3184
|
+
details.output = JSON.parse(details.output);
|
|
3185
|
+
if (details.progress_details)
|
|
3186
|
+
details.progress_details = JSON.parse(details.progress_details);
|
|
3187
|
+
return details;
|
|
3188
|
+
});
|
|
3189
|
+
}
|
|
3190
|
+
getAllJobsWithFilter(prefixFilter) {
|
|
3191
|
+
const filterEntries = Object.entries(prefixFilter);
|
|
3192
|
+
let whereClause = "WHERE queue = ?";
|
|
3193
|
+
const params = [this.queueName];
|
|
3194
|
+
for (const [key, value] of filterEntries) {
|
|
3195
|
+
whereClause += ` AND ${key} = ?`;
|
|
3196
|
+
params.push(value);
|
|
3197
|
+
}
|
|
3198
|
+
const AllJobsQuery = `SELECT * FROM ${this.tableName} ${whereClause}`;
|
|
3199
|
+
const stmt = this.db.prepare(AllJobsQuery);
|
|
3200
|
+
const result = stmt.all(...params);
|
|
3201
|
+
return (result || []).map((details) => {
|
|
3202
|
+
if (details.input)
|
|
3203
|
+
details.input = JSON.parse(details.input);
|
|
3204
|
+
if (details.output)
|
|
3205
|
+
details.output = JSON.parse(details.output);
|
|
3206
|
+
if (details.progress_details)
|
|
3207
|
+
details.progress_details = JSON.parse(details.progress_details);
|
|
3208
|
+
return details;
|
|
3209
|
+
});
|
|
3210
|
+
}
|
|
3211
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
3212
|
+
if (prefixFilter === undefined) {
|
|
3213
|
+
return false;
|
|
3214
|
+
}
|
|
3215
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
3216
|
+
return true;
|
|
3217
|
+
}
|
|
3218
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
3219
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
3220
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
3221
|
+
return true;
|
|
3222
|
+
}
|
|
3223
|
+
for (const key of instanceKeys) {
|
|
3224
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
3225
|
+
return true;
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
3228
|
+
return false;
|
|
3229
|
+
}
|
|
3230
|
+
getPollingManager() {
|
|
3231
|
+
if (!this.pollingManager) {
|
|
3232
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
3233
|
+
const jobs = this.getAllJobs();
|
|
3234
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
3235
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
3236
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
3237
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
3238
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
3239
|
+
});
|
|
3240
|
+
}
|
|
3241
|
+
return this.pollingManager;
|
|
3242
|
+
}
|
|
3243
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
3244
|
+
let lastKnownJobs = new Map;
|
|
3245
|
+
const poll = () => {
|
|
3246
|
+
try {
|
|
3247
|
+
const currentJobs = this.getAllJobsWithFilter(prefixFilter);
|
|
3248
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
3249
|
+
for (const [id, job] of currentMap) {
|
|
3250
|
+
const old = lastKnownJobs.get(id);
|
|
3251
|
+
if (!old) {
|
|
3252
|
+
callback({ type: "INSERT", new: job });
|
|
3253
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
3254
|
+
callback({ type: "UPDATE", old, new: job });
|
|
3255
|
+
}
|
|
3256
|
+
}
|
|
3257
|
+
for (const [id, job] of lastKnownJobs) {
|
|
3258
|
+
if (!currentMap.has(id)) {
|
|
3259
|
+
callback({ type: "DELETE", old: job });
|
|
3260
|
+
}
|
|
3261
|
+
}
|
|
3262
|
+
lastKnownJobs = currentMap;
|
|
3263
|
+
} catch {}
|
|
3264
|
+
};
|
|
3265
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
3266
|
+
poll();
|
|
3267
|
+
return () => {
|
|
3268
|
+
clearInterval(intervalId);
|
|
3269
|
+
};
|
|
3270
|
+
}
|
|
3271
|
+
subscribeToChanges(callback, options) {
|
|
3272
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
3273
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
3274
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
3275
|
+
}
|
|
3276
|
+
const manager = this.getPollingManager();
|
|
3277
|
+
return manager.subscribe(callback, { intervalMs });
|
|
2635
3278
|
}
|
|
2636
3279
|
}
|
|
2637
3280
|
// src/queue/SupabaseQueueStorage.ts
|
|
2638
|
-
import { createServiceToken as
|
|
2639
|
-
var SUPABASE_QUEUE_STORAGE =
|
|
3281
|
+
import { createServiceToken as createServiceToken21, makeFingerprint as makeFingerprint7, uuid4 as uuid44 } from "@workglow/util";
|
|
3282
|
+
var SUPABASE_QUEUE_STORAGE = createServiceToken21("jobqueue.storage.supabase");
|
|
2640
3283
|
|
|
2641
3284
|
class SupabaseQueueStorage {
|
|
2642
3285
|
client;
|
|
2643
3286
|
queueName;
|
|
2644
|
-
|
|
3287
|
+
prefixes;
|
|
3288
|
+
prefixValues;
|
|
3289
|
+
tableName;
|
|
3290
|
+
realtimeChannel = null;
|
|
3291
|
+
pollingManager = null;
|
|
3292
|
+
constructor(client, queueName, options) {
|
|
2645
3293
|
this.client = client;
|
|
2646
3294
|
this.queueName = queueName;
|
|
3295
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3296
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3297
|
+
if (this.prefixes.length > 0) {
|
|
3298
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3299
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
3300
|
+
} else {
|
|
3301
|
+
this.tableName = "job_queue";
|
|
3302
|
+
}
|
|
3303
|
+
}
|
|
3304
|
+
getPrefixColumnType(type) {
|
|
3305
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
3306
|
+
}
|
|
3307
|
+
buildPrefixColumnsSql() {
|
|
3308
|
+
if (this.prefixes.length === 0)
|
|
3309
|
+
return "";
|
|
3310
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
3311
|
+
`) + `,
|
|
3312
|
+
`;
|
|
3313
|
+
}
|
|
3314
|
+
getPrefixColumnNames() {
|
|
3315
|
+
return this.prefixes.map((p) => p.name);
|
|
3316
|
+
}
|
|
3317
|
+
applyPrefixFilters(query) {
|
|
3318
|
+
let result = query;
|
|
3319
|
+
for (const prefix of this.prefixes) {
|
|
3320
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
3321
|
+
}
|
|
3322
|
+
return result;
|
|
3323
|
+
}
|
|
3324
|
+
getPrefixInsertValues() {
|
|
3325
|
+
const values = {};
|
|
3326
|
+
for (const prefix of this.prefixes) {
|
|
3327
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
3328
|
+
}
|
|
3329
|
+
return values;
|
|
2647
3330
|
}
|
|
2648
3331
|
async setupDatabase() {
|
|
2649
3332
|
const createTypeSql = `CREATE TYPE job_status AS ENUM (${Object.values(JobStatus).map((v) => `'${v}'`).join(",")})`;
|
|
@@ -2651,10 +3334,14 @@ class SupabaseQueueStorage {
|
|
|
2651
3334
|
if (typeError && typeError.code !== "42710") {
|
|
2652
3335
|
throw typeError;
|
|
2653
3336
|
}
|
|
3337
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
3338
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3339
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3340
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
2654
3341
|
const createTableSql = `
|
|
2655
|
-
CREATE TABLE IF NOT EXISTS
|
|
3342
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
2656
3343
|
id SERIAL NOT NULL,
|
|
2657
|
-
fingerprint text NOT NULL,
|
|
3344
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
2658
3345
|
queue text NOT NULL,
|
|
2659
3346
|
job_run_id text NOT NULL,
|
|
2660
3347
|
status job_status NOT NULL default 'PENDING',
|
|
@@ -2671,7 +3358,8 @@ class SupabaseQueueStorage {
|
|
|
2671
3358
|
error_code text,
|
|
2672
3359
|
progress real DEFAULT 0,
|
|
2673
3360
|
progress_message text DEFAULT '',
|
|
2674
|
-
progress_details jsonb
|
|
3361
|
+
progress_details jsonb,
|
|
3362
|
+
worker_id text
|
|
2675
3363
|
)`;
|
|
2676
3364
|
const { error: tableError } = await this.client.rpc("exec_sql", { query: createTableSql });
|
|
2677
3365
|
if (tableError) {
|
|
@@ -2680,12 +3368,12 @@ class SupabaseQueueStorage {
|
|
|
2680
3368
|
}
|
|
2681
3369
|
}
|
|
2682
3370
|
const indexes = [
|
|
2683
|
-
`CREATE INDEX IF NOT EXISTS
|
|
2684
|
-
`CREATE INDEX IF NOT EXISTS
|
|
2685
|
-
`CREATE INDEX IF NOT EXISTS
|
|
3371
|
+
`CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`,
|
|
3372
|
+
`CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`,
|
|
3373
|
+
`CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`
|
|
2686
3374
|
];
|
|
2687
3375
|
for (const indexSql of indexes) {
|
|
2688
|
-
|
|
3376
|
+
await this.client.rpc("exec_sql", { query: indexSql });
|
|
2689
3377
|
}
|
|
2690
3378
|
}
|
|
2691
3379
|
async add(job) {
|
|
@@ -2699,7 +3387,9 @@ class SupabaseQueueStorage {
|
|
|
2699
3387
|
job.progress_details = null;
|
|
2700
3388
|
job.created_at = now;
|
|
2701
3389
|
job.run_after = now;
|
|
2702
|
-
const
|
|
3390
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
3391
|
+
const { data, error } = await this.client.from(this.tableName).insert({
|
|
3392
|
+
...prefixInsertValues,
|
|
2703
3393
|
queue: job.queue,
|
|
2704
3394
|
fingerprint: job.fingerprint,
|
|
2705
3395
|
input: job.input,
|
|
@@ -2720,7 +3410,9 @@ class SupabaseQueueStorage {
|
|
|
2720
3410
|
return job.id;
|
|
2721
3411
|
}
|
|
2722
3412
|
async get(id) {
|
|
2723
|
-
|
|
3413
|
+
let query = this.client.from(this.tableName).select("*").eq("id", id).eq("queue", this.queueName);
|
|
3414
|
+
query = this.applyPrefixFilters(query);
|
|
3415
|
+
const { data, error } = await query.single();
|
|
2724
3416
|
if (error) {
|
|
2725
3417
|
if (error.code === "PGRST116")
|
|
2726
3418
|
return;
|
|
@@ -2730,36 +3422,53 @@ class SupabaseQueueStorage {
|
|
|
2730
3422
|
}
|
|
2731
3423
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2732
3424
|
num = Number(num) || 100;
|
|
2733
|
-
|
|
3425
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", status);
|
|
3426
|
+
query = this.applyPrefixFilters(query);
|
|
3427
|
+
const { data, error } = await query.order("run_after", { ascending: true }).limit(num);
|
|
2734
3428
|
if (error)
|
|
2735
3429
|
throw error;
|
|
2736
3430
|
return data ?? [];
|
|
2737
3431
|
}
|
|
2738
|
-
async next() {
|
|
2739
|
-
|
|
3432
|
+
async next(workerId) {
|
|
3433
|
+
let selectQuery = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", "PENDING" /* PENDING */).lte("run_after", new Date().toISOString());
|
|
3434
|
+
selectQuery = this.applyPrefixFilters(selectQuery);
|
|
3435
|
+
const { data: jobs, error: selectError } = await selectQuery.order("run_after", { ascending: true }).limit(1);
|
|
2740
3436
|
if (selectError)
|
|
2741
3437
|
throw selectError;
|
|
2742
3438
|
if (!jobs || jobs.length === 0)
|
|
2743
3439
|
return;
|
|
2744
3440
|
const job = jobs[0];
|
|
2745
|
-
|
|
3441
|
+
let updateQuery = this.client.from(this.tableName).update({
|
|
2746
3442
|
status: "PROCESSING" /* PROCESSING */,
|
|
2747
|
-
last_ran_at: new Date().toISOString()
|
|
2748
|
-
|
|
3443
|
+
last_ran_at: new Date().toISOString(),
|
|
3444
|
+
worker_id: workerId ?? null
|
|
3445
|
+
}).eq("id", job.id).eq("queue", this.queueName);
|
|
3446
|
+
updateQuery = this.applyPrefixFilters(updateQuery);
|
|
3447
|
+
const { data: updatedJob, error: updateError } = await updateQuery.select().single();
|
|
2749
3448
|
if (updateError)
|
|
2750
3449
|
throw updateError;
|
|
2751
3450
|
return updatedJob;
|
|
2752
3451
|
}
|
|
2753
3452
|
async size(status = "PENDING" /* PENDING */) {
|
|
2754
|
-
|
|
3453
|
+
let query = this.client.from(this.tableName).select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
|
|
3454
|
+
query = this.applyPrefixFilters(query);
|
|
3455
|
+
const { count, error } = await query;
|
|
2755
3456
|
if (error)
|
|
2756
3457
|
throw error;
|
|
2757
3458
|
return count ?? 0;
|
|
2758
3459
|
}
|
|
3460
|
+
async getAllJobs() {
|
|
3461
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
3462
|
+
query = this.applyPrefixFilters(query);
|
|
3463
|
+
const { data, error } = await query;
|
|
3464
|
+
if (error)
|
|
3465
|
+
throw error;
|
|
3466
|
+
return data ?? [];
|
|
3467
|
+
}
|
|
2759
3468
|
async complete(jobDetails) {
|
|
2760
3469
|
const now = new Date().toISOString();
|
|
2761
3470
|
if (jobDetails.status === "DISABLED" /* DISABLED */) {
|
|
2762
|
-
|
|
3471
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2763
3472
|
status: jobDetails.status,
|
|
2764
3473
|
progress: 100,
|
|
2765
3474
|
progress_message: "",
|
|
@@ -2767,16 +3476,20 @@ class SupabaseQueueStorage {
|
|
|
2767
3476
|
completed_at: now,
|
|
2768
3477
|
last_ran_at: now
|
|
2769
3478
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3479
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3480
|
+
const { error: error2 } = await query2;
|
|
2770
3481
|
if (error2)
|
|
2771
3482
|
throw error2;
|
|
2772
3483
|
return;
|
|
2773
3484
|
}
|
|
2774
|
-
|
|
3485
|
+
let getQuery = this.client.from(this.tableName).select("run_attempts").eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3486
|
+
getQuery = this.applyPrefixFilters(getQuery);
|
|
3487
|
+
const { data: current, error: getError } = await getQuery.single();
|
|
2775
3488
|
if (getError)
|
|
2776
3489
|
throw getError;
|
|
2777
3490
|
const nextAttempts = (current?.run_attempts ?? 0) + 1;
|
|
2778
3491
|
if (jobDetails.status === "PENDING" /* PENDING */) {
|
|
2779
|
-
|
|
3492
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2780
3493
|
error: jobDetails.error ?? null,
|
|
2781
3494
|
error_code: jobDetails.error_code ?? null,
|
|
2782
3495
|
status: jobDetails.status,
|
|
@@ -2787,12 +3500,14 @@ class SupabaseQueueStorage {
|
|
|
2787
3500
|
run_attempts: nextAttempts,
|
|
2788
3501
|
last_ran_at: now
|
|
2789
3502
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3503
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3504
|
+
const { error: error2 } = await query2;
|
|
2790
3505
|
if (error2)
|
|
2791
3506
|
throw error2;
|
|
2792
3507
|
return;
|
|
2793
3508
|
}
|
|
2794
3509
|
if (jobDetails.status === "COMPLETED" /* COMPLETED */ || jobDetails.status === "FAILED" /* FAILED */) {
|
|
2795
|
-
|
|
3510
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2796
3511
|
output: jobDetails.output ?? null,
|
|
2797
3512
|
error: jobDetails.error ?? null,
|
|
2798
3513
|
error_code: jobDetails.error_code ?? null,
|
|
@@ -2804,11 +3519,13 @@ class SupabaseQueueStorage {
|
|
|
2804
3519
|
completed_at: now,
|
|
2805
3520
|
last_ran_at: now
|
|
2806
3521
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3522
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3523
|
+
const { error: error2 } = await query2;
|
|
2807
3524
|
if (error2)
|
|
2808
3525
|
throw error2;
|
|
2809
3526
|
return;
|
|
2810
3527
|
}
|
|
2811
|
-
|
|
3528
|
+
let query = this.client.from(this.tableName).update({
|
|
2812
3529
|
status: jobDetails.status,
|
|
2813
3530
|
output: jobDetails.output ?? null,
|
|
2814
3531
|
error: jobDetails.error ?? null,
|
|
@@ -2817,17 +3534,23 @@ class SupabaseQueueStorage {
|
|
|
2817
3534
|
run_attempts: nextAttempts,
|
|
2818
3535
|
last_ran_at: now
|
|
2819
3536
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3537
|
+
query = this.applyPrefixFilters(query);
|
|
3538
|
+
const { error } = await query;
|
|
2820
3539
|
if (error)
|
|
2821
3540
|
throw error;
|
|
2822
3541
|
}
|
|
2823
3542
|
async deleteAll() {
|
|
2824
|
-
|
|
3543
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
|
|
3544
|
+
query = this.applyPrefixFilters(query);
|
|
3545
|
+
const { error } = await query;
|
|
2825
3546
|
if (error)
|
|
2826
3547
|
throw error;
|
|
2827
3548
|
}
|
|
2828
3549
|
async outputForInput(input) {
|
|
2829
3550
|
const fingerprint = await makeFingerprint7(input);
|
|
2830
|
-
|
|
3551
|
+
let query = this.client.from(this.tableName).select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", "COMPLETED" /* COMPLETED */);
|
|
3552
|
+
query = this.applyPrefixFilters(query);
|
|
3553
|
+
const { data, error } = await query.single();
|
|
2831
3554
|
if (error) {
|
|
2832
3555
|
if (error.code === "PGRST116")
|
|
2833
3556
|
return null;
|
|
@@ -2836,47 +3559,622 @@ class SupabaseQueueStorage {
|
|
|
2836
3559
|
return data?.output ?? null;
|
|
2837
3560
|
}
|
|
2838
3561
|
async abort(jobId) {
|
|
2839
|
-
|
|
3562
|
+
let query = this.client.from(this.tableName).update({ status: "ABORTING" /* ABORTING */ }).eq("id", jobId).eq("queue", this.queueName);
|
|
3563
|
+
query = this.applyPrefixFilters(query);
|
|
3564
|
+
const { error } = await query;
|
|
2840
3565
|
if (error)
|
|
2841
3566
|
throw error;
|
|
2842
3567
|
}
|
|
2843
3568
|
async getByRunId(job_run_id) {
|
|
2844
|
-
|
|
3569
|
+
let query = this.client.from(this.tableName).select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
|
|
3570
|
+
query = this.applyPrefixFilters(query);
|
|
3571
|
+
const { data, error } = await query;
|
|
2845
3572
|
if (error)
|
|
2846
3573
|
throw error;
|
|
2847
3574
|
return data ?? [];
|
|
2848
3575
|
}
|
|
2849
3576
|
async saveProgress(jobId, progress, message, details) {
|
|
2850
|
-
|
|
3577
|
+
let query = this.client.from(this.tableName).update({
|
|
2851
3578
|
progress,
|
|
2852
3579
|
progress_message: message,
|
|
2853
3580
|
progress_details: details
|
|
2854
3581
|
}).eq("id", jobId).eq("queue", this.queueName);
|
|
3582
|
+
query = this.applyPrefixFilters(query);
|
|
3583
|
+
const { error } = await query;
|
|
2855
3584
|
if (error)
|
|
2856
3585
|
throw error;
|
|
2857
3586
|
}
|
|
2858
3587
|
async delete(jobId) {
|
|
2859
|
-
|
|
3588
|
+
let query = this.client.from(this.tableName).delete().eq("id", jobId).eq("queue", this.queueName);
|
|
3589
|
+
query = this.applyPrefixFilters(query);
|
|
3590
|
+
const { error } = await query;
|
|
2860
3591
|
if (error)
|
|
2861
3592
|
throw error;
|
|
2862
3593
|
}
|
|
2863
3594
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2864
3595
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
2865
|
-
|
|
3596
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
|
|
3597
|
+
query = this.applyPrefixFilters(query);
|
|
3598
|
+
const { error } = await query;
|
|
2866
3599
|
if (error)
|
|
2867
3600
|
throw error;
|
|
2868
3601
|
}
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2879
|
-
|
|
3602
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
3603
|
+
if (!job)
|
|
3604
|
+
return false;
|
|
3605
|
+
if (job.queue !== this.queueName) {
|
|
3606
|
+
return false;
|
|
3607
|
+
}
|
|
3608
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
3609
|
+
return true;
|
|
3610
|
+
}
|
|
3611
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
3612
|
+
if (Object.keys(filterValues).length === 0) {
|
|
3613
|
+
return true;
|
|
3614
|
+
}
|
|
3615
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
3616
|
+
if (job[key] !== value) {
|
|
3617
|
+
return false;
|
|
3618
|
+
}
|
|
3619
|
+
}
|
|
3620
|
+
return true;
|
|
3621
|
+
}
|
|
3622
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
3623
|
+
if (prefixFilter === undefined) {
|
|
3624
|
+
return false;
|
|
3625
|
+
}
|
|
3626
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
3627
|
+
return true;
|
|
3628
|
+
}
|
|
3629
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
3630
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
3631
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
3632
|
+
return true;
|
|
3633
|
+
}
|
|
3634
|
+
for (const key of instanceKeys) {
|
|
3635
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
3636
|
+
return true;
|
|
3637
|
+
}
|
|
3638
|
+
}
|
|
3639
|
+
return false;
|
|
3640
|
+
}
|
|
3641
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
3642
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
3643
|
+
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
3644
|
+
query = query.eq(key, value);
|
|
3645
|
+
}
|
|
3646
|
+
const { data, error } = await query;
|
|
3647
|
+
if (error)
|
|
3648
|
+
throw error;
|
|
3649
|
+
return data ?? [];
|
|
3650
|
+
}
|
|
3651
|
+
subscribeToChanges(callback, options) {
|
|
3652
|
+
return this.subscribeToChangesWithRealtime(callback, options?.prefixFilter);
|
|
3653
|
+
}
|
|
3654
|
+
subscribeToChangesWithRealtime(callback, prefixFilter) {
|
|
3655
|
+
const channelName = `queue-${this.tableName}-${this.queueName}-${Date.now()}`;
|
|
3656
|
+
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
3657
|
+
event: "*",
|
|
3658
|
+
schema: "public",
|
|
3659
|
+
table: this.tableName,
|
|
3660
|
+
filter: `queue=eq.${this.queueName}`
|
|
3661
|
+
}, (payload) => {
|
|
3662
|
+
const newJob = payload.new;
|
|
3663
|
+
const oldJob = payload.old;
|
|
3664
|
+
const newMatches = this.matchesPrefixFilter(newJob, prefixFilter);
|
|
3665
|
+
const oldMatches = this.matchesPrefixFilter(oldJob, prefixFilter);
|
|
3666
|
+
if (!newMatches && !oldMatches) {
|
|
3667
|
+
return;
|
|
3668
|
+
}
|
|
3669
|
+
callback({
|
|
3670
|
+
type: payload.eventType.toUpperCase(),
|
|
3671
|
+
old: oldJob && Object.keys(oldJob).length > 0 ? oldJob : undefined,
|
|
3672
|
+
new: newJob && Object.keys(newJob).length > 0 ? newJob : undefined
|
|
3673
|
+
});
|
|
3674
|
+
}).subscribe();
|
|
3675
|
+
return () => {
|
|
3676
|
+
if (this.realtimeChannel) {
|
|
3677
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
3678
|
+
this.realtimeChannel = null;
|
|
3679
|
+
}
|
|
3680
|
+
};
|
|
3681
|
+
}
|
|
3682
|
+
getPollingManager() {
|
|
3683
|
+
if (!this.pollingManager) {
|
|
3684
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
3685
|
+
const jobs = await this.getAllJobs();
|
|
3686
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
3687
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
3688
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
3689
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
3690
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
3691
|
+
});
|
|
3692
|
+
}
|
|
3693
|
+
return this.pollingManager;
|
|
3694
|
+
}
|
|
3695
|
+
subscribeWithCustomPrefixFilterPolling(callback, prefixFilter, intervalMs) {
|
|
3696
|
+
let lastKnownJobs = new Map;
|
|
3697
|
+
let cancelled = false;
|
|
3698
|
+
const poll = async () => {
|
|
3699
|
+
if (cancelled)
|
|
3700
|
+
return;
|
|
3701
|
+
try {
|
|
3702
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
3703
|
+
if (cancelled)
|
|
3704
|
+
return;
|
|
3705
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
3706
|
+
for (const [id, job] of currentMap) {
|
|
3707
|
+
const old = lastKnownJobs.get(id);
|
|
3708
|
+
if (!old) {
|
|
3709
|
+
callback({ type: "INSERT", new: job });
|
|
3710
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
3711
|
+
callback({ type: "UPDATE", old, new: job });
|
|
3712
|
+
}
|
|
3713
|
+
}
|
|
3714
|
+
for (const [id, job] of lastKnownJobs) {
|
|
3715
|
+
if (!currentMap.has(id)) {
|
|
3716
|
+
callback({ type: "DELETE", old: job });
|
|
3717
|
+
}
|
|
3718
|
+
}
|
|
3719
|
+
lastKnownJobs = currentMap;
|
|
3720
|
+
} catch {}
|
|
3721
|
+
};
|
|
3722
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
3723
|
+
poll();
|
|
3724
|
+
return () => {
|
|
3725
|
+
cancelled = true;
|
|
3726
|
+
clearInterval(intervalId);
|
|
3727
|
+
};
|
|
3728
|
+
}
|
|
3729
|
+
subscribeToChangesWithPolling(callback, options) {
|
|
3730
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
3731
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
3732
|
+
return this.subscribeWithCustomPrefixFilterPolling(callback, options.prefixFilter, intervalMs);
|
|
3733
|
+
}
|
|
3734
|
+
const manager = this.getPollingManager();
|
|
3735
|
+
return manager.subscribe(callback, { intervalMs });
|
|
3736
|
+
}
|
|
3737
|
+
}
|
|
3738
|
+
// src/limiter/PostgresRateLimiterStorage.ts
|
|
3739
|
+
import { createServiceToken as createServiceToken22 } from "@workglow/util";
|
|
3740
|
+
var POSTGRES_RATE_LIMITER_STORAGE = createServiceToken22("ratelimiter.storage.postgres");
|
|
3741
|
+
|
|
3742
|
+
class PostgresRateLimiterStorage {
|
|
3743
|
+
db;
|
|
3744
|
+
prefixes;
|
|
3745
|
+
prefixValues;
|
|
3746
|
+
executionTableName;
|
|
3747
|
+
nextAvailableTableName;
|
|
3748
|
+
constructor(db, options) {
|
|
3749
|
+
this.db = db;
|
|
3750
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3751
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3752
|
+
if (this.prefixes.length > 0) {
|
|
3753
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3754
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
3755
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
3756
|
+
} else {
|
|
3757
|
+
this.executionTableName = "rate_limit_executions";
|
|
3758
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
3759
|
+
}
|
|
3760
|
+
}
|
|
3761
|
+
getPrefixColumnType(type) {
|
|
3762
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
3763
|
+
}
|
|
3764
|
+
buildPrefixColumnsSql() {
|
|
3765
|
+
if (this.prefixes.length === 0)
|
|
3766
|
+
return "";
|
|
3767
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
3768
|
+
`) + `,
|
|
3769
|
+
`;
|
|
3770
|
+
}
|
|
3771
|
+
getPrefixColumnNames() {
|
|
3772
|
+
return this.prefixes.map((p) => p.name);
|
|
3773
|
+
}
|
|
3774
|
+
buildPrefixWhereClause(startParam) {
|
|
3775
|
+
if (this.prefixes.length === 0) {
|
|
3776
|
+
return { conditions: "", params: [] };
|
|
3777
|
+
}
|
|
3778
|
+
const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
|
|
3779
|
+
const params = this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3780
|
+
return { conditions: " AND " + conditions, params };
|
|
3781
|
+
}
|
|
3782
|
+
getPrefixParamValues() {
|
|
3783
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3784
|
+
}
|
|
3785
|
+
async setupDatabase() {
|
|
3786
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
3787
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3788
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3789
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
3790
|
+
await this.db.query(`
|
|
3791
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
3792
|
+
id SERIAL PRIMARY KEY,
|
|
3793
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3794
|
+
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
3795
|
+
)
|
|
3796
|
+
`);
|
|
3797
|
+
await this.db.query(`
|
|
3798
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
3799
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
3800
|
+
`);
|
|
3801
|
+
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
3802
|
+
await this.db.query(`
|
|
3803
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
3804
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3805
|
+
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
3806
|
+
PRIMARY KEY (${primaryKeyColumns})
|
|
3807
|
+
)
|
|
3808
|
+
`);
|
|
3809
|
+
}
|
|
3810
|
+
async recordExecution(queueName) {
|
|
3811
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3812
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3813
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
3814
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
|
|
3815
|
+
const queueParamNum = prefixColumnNames.length + 1;
|
|
3816
|
+
await this.db.query(`
|
|
3817
|
+
INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
|
|
3818
|
+
VALUES (${prefixParamPlaceholders}$${queueParamNum})
|
|
3819
|
+
`, [...prefixParamValues, queueName]);
|
|
3820
|
+
}
|
|
3821
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
3822
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
3823
|
+
const result = await this.db.query(`
|
|
3824
|
+
SELECT COUNT(*) AS count
|
|
3825
|
+
FROM ${this.executionTableName}
|
|
3826
|
+
WHERE queue_name = $1 AND executed_at > $2${prefixConditions}
|
|
3827
|
+
`, [queueName, windowStartTime, ...prefixParams]);
|
|
3828
|
+
return parseInt(result.rows[0]?.count ?? "0", 10);
|
|
3829
|
+
}
|
|
3830
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
3831
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
3832
|
+
const result = await this.db.query(`
|
|
3833
|
+
SELECT executed_at
|
|
3834
|
+
FROM ${this.executionTableName}
|
|
3835
|
+
WHERE queue_name = $1${prefixConditions}
|
|
3836
|
+
ORDER BY executed_at ASC
|
|
3837
|
+
LIMIT 1 OFFSET $2
|
|
3838
|
+
`, [queueName, offset, ...prefixParams]);
|
|
3839
|
+
const executedAt = result.rows[0]?.executed_at;
|
|
3840
|
+
if (!executedAt)
|
|
3841
|
+
return;
|
|
3842
|
+
return new Date(executedAt).toISOString();
|
|
3843
|
+
}
|
|
3844
|
+
async getNextAvailableTime(queueName) {
|
|
3845
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
3846
|
+
const result = await this.db.query(`
|
|
3847
|
+
SELECT next_available_at
|
|
3848
|
+
FROM ${this.nextAvailableTableName}
|
|
3849
|
+
WHERE queue_name = $1${prefixConditions}
|
|
3850
|
+
`, [queueName, ...prefixParams]);
|
|
3851
|
+
const nextAvailableAt = result.rows[0]?.next_available_at;
|
|
3852
|
+
if (!nextAvailableAt)
|
|
3853
|
+
return;
|
|
3854
|
+
return new Date(nextAvailableAt).toISOString();
|
|
3855
|
+
}
|
|
3856
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
3857
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3858
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3859
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
3860
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
|
|
3861
|
+
const baseParamStart = prefixColumnNames.length + 1;
|
|
3862
|
+
const conflictColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
3863
|
+
await this.db.query(`
|
|
3864
|
+
INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
|
|
3865
|
+
VALUES (${prefixParamPlaceholders}$${baseParamStart}, $${baseParamStart + 1})
|
|
3866
|
+
ON CONFLICT (${conflictColumns})
|
|
3867
|
+
DO UPDATE SET next_available_at = EXCLUDED.next_available_at
|
|
3868
|
+
`, [...prefixParamValues, queueName, nextAvailableAt]);
|
|
3869
|
+
}
|
|
3870
|
+
async clear(queueName) {
|
|
3871
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
3872
|
+
await this.db.query(`DELETE FROM ${this.executionTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
|
|
3873
|
+
await this.db.query(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
|
|
3874
|
+
}
|
|
3875
|
+
}
|
|
3876
|
+
// src/limiter/SqliteRateLimiterStorage.ts
|
|
3877
|
+
import { createServiceToken as createServiceToken23, sleep as sleep5, toSQLiteTimestamp } from "@workglow/util";
|
|
3878
|
+
var SQLITE_RATE_LIMITER_STORAGE = createServiceToken23("ratelimiter.storage.sqlite");
|
|
3879
|
+
|
|
3880
|
+
class SqliteRateLimiterStorage {
|
|
3881
|
+
db;
|
|
3882
|
+
prefixes;
|
|
3883
|
+
prefixValues;
|
|
3884
|
+
executionTableName;
|
|
3885
|
+
nextAvailableTableName;
|
|
3886
|
+
constructor(db, options) {
|
|
3887
|
+
this.db = db;
|
|
3888
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3889
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3890
|
+
if (this.prefixes.length > 0) {
|
|
3891
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3892
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
3893
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
3894
|
+
} else {
|
|
3895
|
+
this.executionTableName = "rate_limit_executions";
|
|
3896
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
3897
|
+
}
|
|
3898
|
+
}
|
|
3899
|
+
getPrefixColumnType(type) {
|
|
3900
|
+
return type === "uuid" ? "TEXT" : "INTEGER";
|
|
3901
|
+
}
|
|
3902
|
+
buildPrefixColumnsSql() {
|
|
3903
|
+
if (this.prefixes.length === 0)
|
|
3904
|
+
return "";
|
|
3905
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
3906
|
+
`) + `,
|
|
3907
|
+
`;
|
|
3908
|
+
}
|
|
3909
|
+
getPrefixColumnNames() {
|
|
3910
|
+
return this.prefixes.map((p) => p.name);
|
|
3911
|
+
}
|
|
3912
|
+
buildPrefixWhereClause() {
|
|
3913
|
+
if (this.prefixes.length === 0) {
|
|
3914
|
+
return "";
|
|
3915
|
+
}
|
|
3916
|
+
const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
|
|
3917
|
+
return " AND " + conditions;
|
|
3918
|
+
}
|
|
3919
|
+
getPrefixParamValues() {
|
|
3920
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3921
|
+
}
|
|
3922
|
+
async setupDatabase() {
|
|
3923
|
+
await sleep5(0);
|
|
3924
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
3925
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3926
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3927
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
3928
|
+
this.db.exec(`
|
|
3929
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
3930
|
+
id INTEGER PRIMARY KEY,
|
|
3931
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3932
|
+
executed_at TEXT DEFAULT CURRENT_TIMESTAMP
|
|
3933
|
+
);
|
|
3934
|
+
|
|
3935
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
3936
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at);
|
|
3937
|
+
`);
|
|
3938
|
+
this.db.exec(`
|
|
3939
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
3940
|
+
${prefixColumnsSql}queue_name TEXT PRIMARY KEY,
|
|
3941
|
+
next_available_at TEXT
|
|
3942
|
+
);
|
|
3943
|
+
`);
|
|
3944
|
+
}
|
|
3945
|
+
async recordExecution(queueName) {
|
|
3946
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3947
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3948
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
3949
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
3950
|
+
const stmt = this.db.prepare(`
|
|
3951
|
+
INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
|
|
3952
|
+
VALUES (${prefixPlaceholders}?)
|
|
3953
|
+
`);
|
|
3954
|
+
stmt.run(...prefixParamValues, queueName);
|
|
3955
|
+
}
|
|
3956
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
3957
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3958
|
+
const prefixParams = this.getPrefixParamValues();
|
|
3959
|
+
const thresholdTime = toSQLiteTimestamp(new Date(windowStartTime));
|
|
3960
|
+
const stmt = this.db.prepare(`
|
|
3961
|
+
SELECT COUNT(*) AS count
|
|
3962
|
+
FROM ${this.executionTableName}
|
|
3963
|
+
WHERE queue_name = ? AND executed_at > ?${prefixConditions}
|
|
3964
|
+
`);
|
|
3965
|
+
const result = stmt.get(queueName, thresholdTime, ...prefixParams);
|
|
3966
|
+
return result?.count ?? 0;
|
|
3967
|
+
}
|
|
3968
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
3969
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3970
|
+
const prefixParams = this.getPrefixParamValues();
|
|
3971
|
+
const stmt = this.db.prepare(`
|
|
3972
|
+
SELECT executed_at
|
|
3973
|
+
FROM ${this.executionTableName}
|
|
3974
|
+
WHERE queue_name = ?${prefixConditions}
|
|
3975
|
+
ORDER BY executed_at ASC
|
|
3976
|
+
LIMIT 1 OFFSET ?
|
|
3977
|
+
`);
|
|
3978
|
+
const result = stmt.get(queueName, ...prefixParams, offset);
|
|
3979
|
+
if (!result)
|
|
3980
|
+
return;
|
|
3981
|
+
return result.executed_at + "Z";
|
|
3982
|
+
}
|
|
3983
|
+
async getNextAvailableTime(queueName) {
|
|
3984
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3985
|
+
const prefixParams = this.getPrefixParamValues();
|
|
3986
|
+
const stmt = this.db.prepare(`
|
|
3987
|
+
SELECT next_available_at
|
|
3988
|
+
FROM ${this.nextAvailableTableName}
|
|
3989
|
+
WHERE queue_name = ?${prefixConditions}
|
|
3990
|
+
`);
|
|
3991
|
+
const result = stmt.get(queueName, ...prefixParams);
|
|
3992
|
+
if (!result?.next_available_at)
|
|
3993
|
+
return;
|
|
3994
|
+
return result.next_available_at + "Z";
|
|
3995
|
+
}
|
|
3996
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
3997
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3998
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3999
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
4000
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
4001
|
+
const stmt = this.db.prepare(`
|
|
4002
|
+
INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
|
|
4003
|
+
VALUES (${prefixPlaceholders}?, ?)
|
|
4004
|
+
ON CONFLICT(queue_name) DO UPDATE SET next_available_at = excluded.next_available_at
|
|
4005
|
+
`);
|
|
4006
|
+
stmt.run(...prefixParamValues, queueName, nextAvailableAt);
|
|
4007
|
+
}
|
|
4008
|
+
async clear(queueName) {
|
|
4009
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
4010
|
+
const prefixParams = this.getPrefixParamValues();
|
|
4011
|
+
this.db.prepare(`DELETE FROM ${this.executionTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
|
|
4012
|
+
this.db.prepare(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
|
|
4013
|
+
}
|
|
4014
|
+
}
|
|
4015
|
+
// src/limiter/SupabaseRateLimiterStorage.ts
|
|
4016
|
+
import { createServiceToken as createServiceToken24 } from "@workglow/util";
|
|
4017
|
+
var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken24("ratelimiter.storage.supabase");
|
|
4018
|
+
|
|
4019
|
+
class SupabaseRateLimiterStorage {
|
|
4020
|
+
client;
|
|
4021
|
+
prefixes;
|
|
4022
|
+
prefixValues;
|
|
4023
|
+
executionTableName;
|
|
4024
|
+
nextAvailableTableName;
|
|
4025
|
+
constructor(client, options) {
|
|
4026
|
+
this.client = client;
|
|
4027
|
+
this.prefixes = options?.prefixes ?? [];
|
|
4028
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
4029
|
+
if (this.prefixes.length > 0) {
|
|
4030
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
4031
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
4032
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
4033
|
+
} else {
|
|
4034
|
+
this.executionTableName = "rate_limit_executions";
|
|
4035
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
4036
|
+
}
|
|
4037
|
+
}
|
|
4038
|
+
getPrefixColumnType(type) {
|
|
4039
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
4040
|
+
}
|
|
4041
|
+
buildPrefixColumnsSql() {
|
|
4042
|
+
if (this.prefixes.length === 0)
|
|
4043
|
+
return "";
|
|
4044
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
4045
|
+
`) + `,
|
|
4046
|
+
`;
|
|
4047
|
+
}
|
|
4048
|
+
getPrefixColumnNames() {
|
|
4049
|
+
return this.prefixes.map((p) => p.name);
|
|
4050
|
+
}
|
|
4051
|
+
applyPrefixFilters(query) {
|
|
4052
|
+
let result = query;
|
|
4053
|
+
for (const prefix of this.prefixes) {
|
|
4054
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
4055
|
+
}
|
|
4056
|
+
return result;
|
|
4057
|
+
}
|
|
4058
|
+
getPrefixInsertValues() {
|
|
4059
|
+
const values = {};
|
|
4060
|
+
for (const prefix of this.prefixes) {
|
|
4061
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
4062
|
+
}
|
|
4063
|
+
return values;
|
|
4064
|
+
}
|
|
4065
|
+
async setupDatabase() {
|
|
4066
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
4067
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4068
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
4069
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
4070
|
+
const createExecTableSql = `
|
|
4071
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
4072
|
+
id SERIAL PRIMARY KEY,
|
|
4073
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
4074
|
+
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
4075
|
+
)
|
|
4076
|
+
`;
|
|
4077
|
+
const { error: execTableError } = await this.client.rpc("exec_sql", {
|
|
4078
|
+
query: createExecTableSql
|
|
4079
|
+
});
|
|
4080
|
+
if (execTableError && execTableError.code !== "42P07") {
|
|
4081
|
+
throw execTableError;
|
|
4082
|
+
}
|
|
4083
|
+
const createExecIndexSql = `
|
|
4084
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
4085
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
4086
|
+
`;
|
|
4087
|
+
await this.client.rpc("exec_sql", { query: createExecIndexSql });
|
|
4088
|
+
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
4089
|
+
const createNextTableSql = `
|
|
4090
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
4091
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
4092
|
+
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
4093
|
+
PRIMARY KEY (${primaryKeyColumns})
|
|
4094
|
+
)
|
|
4095
|
+
`;
|
|
4096
|
+
const { error: nextTableError } = await this.client.rpc("exec_sql", {
|
|
4097
|
+
query: createNextTableSql
|
|
4098
|
+
});
|
|
4099
|
+
if (nextTableError && nextTableError.code !== "42P07") {
|
|
4100
|
+
throw nextTableError;
|
|
4101
|
+
}
|
|
4102
|
+
}
|
|
4103
|
+
async recordExecution(queueName) {
|
|
4104
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
4105
|
+
const { error } = await this.client.from(this.executionTableName).insert({
|
|
4106
|
+
...prefixInsertValues,
|
|
4107
|
+
queue_name: queueName
|
|
4108
|
+
});
|
|
4109
|
+
if (error)
|
|
4110
|
+
throw error;
|
|
4111
|
+
}
|
|
4112
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
4113
|
+
let query = this.client.from(this.executionTableName).select("*", { count: "exact", head: true }).eq("queue_name", queueName).gt("executed_at", windowStartTime);
|
|
4114
|
+
query = this.applyPrefixFilters(query);
|
|
4115
|
+
const { count, error } = await query;
|
|
4116
|
+
if (error)
|
|
4117
|
+
throw error;
|
|
4118
|
+
return count ?? 0;
|
|
4119
|
+
}
|
|
4120
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
4121
|
+
let query = this.client.from(this.executionTableName).select("executed_at").eq("queue_name", queueName);
|
|
4122
|
+
query = this.applyPrefixFilters(query);
|
|
4123
|
+
const { data, error } = await query.order("executed_at", { ascending: true }).range(offset, offset);
|
|
4124
|
+
if (error)
|
|
4125
|
+
throw error;
|
|
4126
|
+
if (!data || data.length === 0)
|
|
4127
|
+
return;
|
|
4128
|
+
return new Date(data[0].executed_at).toISOString();
|
|
4129
|
+
}
|
|
4130
|
+
async getNextAvailableTime(queueName) {
|
|
4131
|
+
let query = this.client.from(this.nextAvailableTableName).select("next_available_at").eq("queue_name", queueName);
|
|
4132
|
+
query = this.applyPrefixFilters(query);
|
|
4133
|
+
const { data, error } = await query.single();
|
|
4134
|
+
if (error) {
|
|
4135
|
+
if (error.code === "PGRST116")
|
|
4136
|
+
return;
|
|
4137
|
+
throw error;
|
|
4138
|
+
}
|
|
4139
|
+
if (!data?.next_available_at)
|
|
4140
|
+
return;
|
|
4141
|
+
return new Date(data.next_available_at).toISOString();
|
|
4142
|
+
}
|
|
4143
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
4144
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
4145
|
+
const { error } = await this.client.from(this.nextAvailableTableName).upsert({
|
|
4146
|
+
...prefixInsertValues,
|
|
4147
|
+
queue_name: queueName,
|
|
4148
|
+
next_available_at: nextAvailableAt
|
|
4149
|
+
}, {
|
|
4150
|
+
onConflict: this.prefixes.length > 0 ? `${this.getPrefixColumnNames().join(",")},queue_name` : "queue_name"
|
|
4151
|
+
});
|
|
4152
|
+
if (error)
|
|
4153
|
+
throw error;
|
|
4154
|
+
}
|
|
4155
|
+
async clear(queueName) {
|
|
4156
|
+
let execQuery = this.client.from(this.executionTableName).delete().eq("queue_name", queueName);
|
|
4157
|
+
execQuery = this.applyPrefixFilters(execQuery);
|
|
4158
|
+
const { error: execError } = await execQuery;
|
|
4159
|
+
if (execError)
|
|
4160
|
+
throw execError;
|
|
4161
|
+
let nextQuery = this.client.from(this.nextAvailableTableName).delete().eq("queue_name", queueName);
|
|
4162
|
+
nextQuery = this.applyPrefixFilters(nextQuery);
|
|
4163
|
+
const { error: nextError } = await nextQuery;
|
|
4164
|
+
if (nextError)
|
|
4165
|
+
throw nextError;
|
|
4166
|
+
}
|
|
4167
|
+
}
|
|
4168
|
+
// src/kv/IndexedDbKvRepository.ts
|
|
4169
|
+
import { createServiceToken as createServiceToken26 } from "@workglow/util";
|
|
4170
|
+
|
|
4171
|
+
// src/tabular/IndexedDbTabularRepository.ts
|
|
4172
|
+
import { createServiceToken as createServiceToken25 } from "@workglow/util";
|
|
4173
|
+
|
|
4174
|
+
// src/util/IndexedDbTable.ts
|
|
4175
|
+
var METADATA_STORE_NAME = "__schema_metadata__";
|
|
4176
|
+
async function saveSchemaMetadata(db, tableName, snapshot) {
|
|
4177
|
+
return new Promise((resolve, reject) => {
|
|
2880
4178
|
try {
|
|
2881
4179
|
const transaction = db.transaction(METADATA_STORE_NAME, "readwrite");
|
|
2882
4180
|
const store = transaction.objectStore(METADATA_STORE_NAME);
|
|
@@ -3211,7 +4509,7 @@ async function dropIndexedDbTable(tableName) {
|
|
|
3211
4509
|
}
|
|
3212
4510
|
|
|
3213
4511
|
// src/tabular/IndexedDbTabularRepository.ts
|
|
3214
|
-
var IDB_TABULAR_REPOSITORY =
|
|
4512
|
+
var IDB_TABULAR_REPOSITORY = createServiceToken25("storage.tabularRepository.indexedDb");
|
|
3215
4513
|
|
|
3216
4514
|
class IndexedDbTabularRepository extends TabularRepository {
|
|
3217
4515
|
table;
|
|
@@ -3547,7 +4845,7 @@ class IndexedDbTabularRepository extends TabularRepository {
|
|
|
3547
4845
|
}
|
|
3548
4846
|
|
|
3549
4847
|
// src/kv/IndexedDbKvRepository.ts
|
|
3550
|
-
var IDB_KV_REPOSITORY =
|
|
4848
|
+
var IDB_KV_REPOSITORY = createServiceToken26("storage.kvRepository.indexedDb");
|
|
3551
4849
|
|
|
3552
4850
|
class IndexedDbKvRepository extends KvViaTabularRepository {
|
|
3553
4851
|
dbName;
|
|
@@ -3559,18 +4857,42 @@ class IndexedDbKvRepository extends KvViaTabularRepository {
|
|
|
3559
4857
|
}
|
|
3560
4858
|
}
|
|
3561
4859
|
// src/queue/IndexedDbQueueStorage.ts
|
|
3562
|
-
import { createServiceToken as
|
|
3563
|
-
var INDEXED_DB_QUEUE_STORAGE =
|
|
4860
|
+
import { createServiceToken as createServiceToken27, makeFingerprint as makeFingerprint8, uuid4 as uuid45 } from "@workglow/util";
|
|
4861
|
+
var INDEXED_DB_QUEUE_STORAGE = createServiceToken27("jobqueue.storage.indexedDb");
|
|
3564
4862
|
|
|
3565
4863
|
class IndexedDbQueueStorage {
|
|
3566
4864
|
queueName;
|
|
3567
4865
|
db;
|
|
3568
4866
|
tableName;
|
|
3569
4867
|
migrationOptions;
|
|
3570
|
-
|
|
4868
|
+
prefixes;
|
|
4869
|
+
prefixValues;
|
|
4870
|
+
pollingManager = null;
|
|
4871
|
+
constructor(queueName, options = {}) {
|
|
3571
4872
|
this.queueName = queueName;
|
|
3572
|
-
this.
|
|
3573
|
-
this.
|
|
4873
|
+
this.migrationOptions = options;
|
|
4874
|
+
this.prefixes = options.prefixes ?? [];
|
|
4875
|
+
this.prefixValues = options.prefixValues ?? {};
|
|
4876
|
+
if (this.prefixes.length > 0) {
|
|
4877
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
4878
|
+
this.tableName = `jobs_${prefixNames}`;
|
|
4879
|
+
} else {
|
|
4880
|
+
this.tableName = "jobs";
|
|
4881
|
+
}
|
|
4882
|
+
}
|
|
4883
|
+
getPrefixColumnNames() {
|
|
4884
|
+
return this.prefixes.map((p) => p.name);
|
|
4885
|
+
}
|
|
4886
|
+
matchesPrefixes(job) {
|
|
4887
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
4888
|
+
if (job[key] !== value) {
|
|
4889
|
+
return false;
|
|
4890
|
+
}
|
|
4891
|
+
}
|
|
4892
|
+
return true;
|
|
4893
|
+
}
|
|
4894
|
+
getPrefixKeyValues() {
|
|
4895
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3574
4896
|
}
|
|
3575
4897
|
async getDb() {
|
|
3576
4898
|
if (this.db)
|
|
@@ -3579,25 +4901,29 @@ class IndexedDbQueueStorage {
|
|
|
3579
4901
|
return this.db;
|
|
3580
4902
|
}
|
|
3581
4903
|
async setupDatabase() {
|
|
4904
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4905
|
+
const buildKeyPath = (basePath) => {
|
|
4906
|
+
return [...prefixColumnNames, ...basePath];
|
|
4907
|
+
};
|
|
3582
4908
|
const expectedIndexes = [
|
|
3583
4909
|
{
|
|
3584
|
-
name: "
|
|
3585
|
-
keyPath:
|
|
4910
|
+
name: "queue_status",
|
|
4911
|
+
keyPath: buildKeyPath(["queue", "status"]),
|
|
3586
4912
|
options: { unique: false }
|
|
3587
4913
|
},
|
|
3588
4914
|
{
|
|
3589
|
-
name: "
|
|
3590
|
-
keyPath: ["status", "run_after"],
|
|
4915
|
+
name: "queue_status_run_after",
|
|
4916
|
+
keyPath: buildKeyPath(["queue", "status", "run_after"]),
|
|
3591
4917
|
options: { unique: false }
|
|
3592
4918
|
},
|
|
3593
4919
|
{
|
|
3594
|
-
name: "
|
|
3595
|
-
keyPath:
|
|
4920
|
+
name: "queue_job_run_id",
|
|
4921
|
+
keyPath: buildKeyPath(["queue", "job_run_id"]),
|
|
3596
4922
|
options: { unique: false }
|
|
3597
4923
|
},
|
|
3598
4924
|
{
|
|
3599
|
-
name: "
|
|
3600
|
-
keyPath: ["fingerprint", "status"],
|
|
4925
|
+
name: "queue_fingerprint_status",
|
|
4926
|
+
keyPath: buildKeyPath(["queue", "fingerprint", "status"]),
|
|
3601
4927
|
options: { unique: false }
|
|
3602
4928
|
}
|
|
3603
4929
|
];
|
|
@@ -3606,21 +4932,25 @@ class IndexedDbQueueStorage {
|
|
|
3606
4932
|
async add(job) {
|
|
3607
4933
|
const db = await this.getDb();
|
|
3608
4934
|
const now = new Date().toISOString();
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
4935
|
+
const jobWithPrefixes = job;
|
|
4936
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid45();
|
|
4937
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid45();
|
|
4938
|
+
jobWithPrefixes.queue = this.queueName;
|
|
4939
|
+
jobWithPrefixes.fingerprint = await makeFingerprint8(jobWithPrefixes.input);
|
|
4940
|
+
jobWithPrefixes.status = "PENDING" /* PENDING */;
|
|
4941
|
+
jobWithPrefixes.progress = 0;
|
|
4942
|
+
jobWithPrefixes.progress_message = "";
|
|
4943
|
+
jobWithPrefixes.progress_details = null;
|
|
4944
|
+
jobWithPrefixes.created_at = now;
|
|
4945
|
+
jobWithPrefixes.run_after = now;
|
|
4946
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
4947
|
+
jobWithPrefixes[key] = value;
|
|
4948
|
+
}
|
|
3619
4949
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3620
4950
|
const store = tx.objectStore(this.tableName);
|
|
3621
4951
|
return new Promise((resolve, reject) => {
|
|
3622
|
-
const request = store.add(
|
|
3623
|
-
tx.oncomplete = () => resolve(
|
|
4952
|
+
const request = store.add(jobWithPrefixes);
|
|
4953
|
+
tx.oncomplete = () => resolve(jobWithPrefixes.id);
|
|
3624
4954
|
tx.onerror = () => reject(tx.error);
|
|
3625
4955
|
request.onerror = () => reject(request.error);
|
|
3626
4956
|
});
|
|
@@ -3631,7 +4961,14 @@ class IndexedDbQueueStorage {
|
|
|
3631
4961
|
const store = tx.objectStore(this.tableName);
|
|
3632
4962
|
const request = store.get(id);
|
|
3633
4963
|
return new Promise((resolve, reject) => {
|
|
3634
|
-
request.onsuccess = () =>
|
|
4964
|
+
request.onsuccess = () => {
|
|
4965
|
+
const job = request.result;
|
|
4966
|
+
if (job && job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
4967
|
+
resolve(job);
|
|
4968
|
+
} else {
|
|
4969
|
+
resolve(undefined);
|
|
4970
|
+
}
|
|
4971
|
+
};
|
|
3635
4972
|
request.onerror = () => reject(request.error);
|
|
3636
4973
|
tx.onerror = () => reject(tx.error);
|
|
3637
4974
|
});
|
|
@@ -3640,10 +4977,11 @@ class IndexedDbQueueStorage {
|
|
|
3640
4977
|
const db = await this.getDb();
|
|
3641
4978
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3642
4979
|
const store = tx.objectStore(this.tableName);
|
|
3643
|
-
const index = store.index("
|
|
4980
|
+
const index = store.index("queue_status_run_after");
|
|
4981
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3644
4982
|
return new Promise((resolve, reject) => {
|
|
3645
4983
|
const ret = new Map;
|
|
3646
|
-
const keyRange = IDBKeyRange.bound([status, ""], [status, ""]);
|
|
4984
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, status, ""], [...prefixKeyValues, this.queueName, status, ""]);
|
|
3647
4985
|
const cursorRequest = index.openCursor(keyRange);
|
|
3648
4986
|
const handleCursor = (e) => {
|
|
3649
4987
|
const cursor = e.target.result;
|
|
@@ -3651,7 +4989,10 @@ class IndexedDbQueueStorage {
|
|
|
3651
4989
|
resolve(Array.from(ret.values()));
|
|
3652
4990
|
return;
|
|
3653
4991
|
}
|
|
3654
|
-
|
|
4992
|
+
const job = cursor.value;
|
|
4993
|
+
if (this.matchesPrefixes(job)) {
|
|
4994
|
+
ret.set(cursor.value.id, cursor.value);
|
|
4995
|
+
}
|
|
3655
4996
|
cursor.continue();
|
|
3656
4997
|
};
|
|
3657
4998
|
cursorRequest.onsuccess = handleCursor;
|
|
@@ -3659,14 +5000,15 @@ class IndexedDbQueueStorage {
|
|
|
3659
5000
|
tx.onerror = () => reject(tx.error);
|
|
3660
5001
|
});
|
|
3661
5002
|
}
|
|
3662
|
-
async next() {
|
|
5003
|
+
async next(workerId) {
|
|
3663
5004
|
const db = await this.getDb();
|
|
3664
5005
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3665
5006
|
const store = tx.objectStore(this.tableName);
|
|
3666
|
-
const index = store.index("
|
|
5007
|
+
const index = store.index("queue_status_run_after");
|
|
3667
5008
|
const now = new Date().toISOString();
|
|
5009
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3668
5010
|
return new Promise((resolve, reject) => {
|
|
3669
|
-
const cursorRequest = index.openCursor(IDBKeyRange.bound(["PENDING" /* PENDING */, ""], ["PENDING" /* PENDING */, now], false, true));
|
|
5011
|
+
const cursorRequest = index.openCursor(IDBKeyRange.bound([...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, ""], [...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, now], false, true));
|
|
3670
5012
|
let jobToReturn;
|
|
3671
5013
|
cursorRequest.onsuccess = (e) => {
|
|
3672
5014
|
const cursor = e.target.result;
|
|
@@ -3679,12 +5021,13 @@ class IndexedDbQueueStorage {
|
|
|
3679
5021
|
return;
|
|
3680
5022
|
}
|
|
3681
5023
|
const job = cursor.value;
|
|
3682
|
-
if (job.status !== "PENDING" /* PENDING */) {
|
|
5024
|
+
if (job.queue !== this.queueName || job.status !== "PENDING" /* PENDING */ || !this.matchesPrefixes(job)) {
|
|
3683
5025
|
cursor.continue();
|
|
3684
5026
|
return;
|
|
3685
5027
|
}
|
|
3686
5028
|
job.status = "PROCESSING" /* PROCESSING */;
|
|
3687
5029
|
job.last_ran_at = now;
|
|
5030
|
+
job.worker_id = workerId ?? null;
|
|
3688
5031
|
try {
|
|
3689
5032
|
const updateRequest = store.put(job);
|
|
3690
5033
|
updateRequest.onsuccess = () => {
|
|
@@ -3708,11 +5051,13 @@ class IndexedDbQueueStorage {
|
|
|
3708
5051
|
}
|
|
3709
5052
|
async size(status = "PENDING" /* PENDING */) {
|
|
3710
5053
|
const db = await this.getDb();
|
|
5054
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3711
5055
|
return new Promise((resolve, reject) => {
|
|
3712
5056
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3713
5057
|
const store = tx.objectStore(this.tableName);
|
|
3714
|
-
const index = store.index("
|
|
3715
|
-
const
|
|
5058
|
+
const index = store.index("queue_status");
|
|
5059
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
5060
|
+
const request = index.count(keyRange);
|
|
3716
5061
|
request.onsuccess = () => resolve(request.result);
|
|
3717
5062
|
request.onerror = () => reject(request.error);
|
|
3718
5063
|
tx.onerror = () => reject(tx.error);
|
|
@@ -3726,9 +5071,18 @@ class IndexedDbQueueStorage {
|
|
|
3726
5071
|
const getReq = store.get(job.id);
|
|
3727
5072
|
getReq.onsuccess = () => {
|
|
3728
5073
|
const existing = getReq.result;
|
|
3729
|
-
|
|
5074
|
+
if (!existing || existing.queue !== this.queueName || !this.matchesPrefixes(existing)) {
|
|
5075
|
+
reject(new Error(`Job ${job.id} not found or does not belong to queue ${this.queueName}`));
|
|
5076
|
+
return;
|
|
5077
|
+
}
|
|
5078
|
+
const currentAttempts = existing.run_attempts ?? 0;
|
|
3730
5079
|
job.run_attempts = currentAttempts + 1;
|
|
3731
|
-
|
|
5080
|
+
job.queue = this.queueName;
|
|
5081
|
+
const jobWithPrefixes = job;
|
|
5082
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5083
|
+
jobWithPrefixes[key] = value;
|
|
5084
|
+
}
|
|
5085
|
+
const putReq = store.put(jobWithPrefixes);
|
|
3732
5086
|
putReq.onsuccess = () => {};
|
|
3733
5087
|
putReq.onerror = () => reject(putReq.error);
|
|
3734
5088
|
};
|
|
@@ -3748,10 +5102,15 @@ class IndexedDbQueueStorage {
|
|
|
3748
5102
|
const db = await this.getDb();
|
|
3749
5103
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3750
5104
|
const store = tx.objectStore(this.tableName);
|
|
3751
|
-
const index = store.index("
|
|
3752
|
-
const
|
|
5105
|
+
const index = store.index("queue_job_run_id");
|
|
5106
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5107
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, job_run_id]);
|
|
5108
|
+
const request = index.getAll(keyRange);
|
|
3753
5109
|
return new Promise((resolve, reject) => {
|
|
3754
|
-
request.onsuccess = () =>
|
|
5110
|
+
request.onsuccess = () => {
|
|
5111
|
+
const results = (request.result || []).filter((job) => this.matchesPrefixes(job));
|
|
5112
|
+
resolve(results);
|
|
5113
|
+
};
|
|
3755
5114
|
request.onerror = () => reject(request.error);
|
|
3756
5115
|
tx.onerror = () => reject(tx.error);
|
|
3757
5116
|
});
|
|
@@ -3760,11 +5119,31 @@ class IndexedDbQueueStorage {
|
|
|
3760
5119
|
const db = await this.getDb();
|
|
3761
5120
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3762
5121
|
const store = tx.objectStore(this.tableName);
|
|
3763
|
-
const
|
|
5122
|
+
const index = store.index("queue_status");
|
|
5123
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3764
5124
|
return new Promise((resolve, reject) => {
|
|
3765
|
-
|
|
3766
|
-
request
|
|
5125
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, ""]);
|
|
5126
|
+
const request = index.openCursor(keyRange);
|
|
5127
|
+
request.onsuccess = (event) => {
|
|
5128
|
+
const cursor = event.target.result;
|
|
5129
|
+
if (cursor) {
|
|
5130
|
+
const job = cursor.value;
|
|
5131
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
5132
|
+
const deleteRequest = cursor.delete();
|
|
5133
|
+
deleteRequest.onsuccess = () => {
|
|
5134
|
+
cursor.continue();
|
|
5135
|
+
};
|
|
5136
|
+
deleteRequest.onerror = () => {
|
|
5137
|
+
cursor.continue();
|
|
5138
|
+
};
|
|
5139
|
+
} else {
|
|
5140
|
+
cursor.continue();
|
|
5141
|
+
}
|
|
5142
|
+
}
|
|
5143
|
+
};
|
|
5144
|
+
tx.oncomplete = () => resolve();
|
|
3767
5145
|
tx.onerror = () => reject(tx.error);
|
|
5146
|
+
request.onerror = () => reject(request.error);
|
|
3768
5147
|
});
|
|
3769
5148
|
}
|
|
3770
5149
|
async outputForInput(input) {
|
|
@@ -3772,10 +5151,23 @@ class IndexedDbQueueStorage {
|
|
|
3772
5151
|
const db = await this.getDb();
|
|
3773
5152
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3774
5153
|
const store = tx.objectStore(this.tableName);
|
|
3775
|
-
const index = store.index("
|
|
3776
|
-
const
|
|
5154
|
+
const index = store.index("queue_fingerprint_status");
|
|
5155
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5156
|
+
const request = index.get([
|
|
5157
|
+
...prefixKeyValues,
|
|
5158
|
+
this.queueName,
|
|
5159
|
+
fingerprint,
|
|
5160
|
+
"COMPLETED" /* COMPLETED */
|
|
5161
|
+
]);
|
|
3777
5162
|
return new Promise((resolve, reject) => {
|
|
3778
|
-
request.onsuccess = () =>
|
|
5163
|
+
request.onsuccess = () => {
|
|
5164
|
+
const job = request.result;
|
|
5165
|
+
if (job && this.matchesPrefixes(job)) {
|
|
5166
|
+
resolve(job.output ?? null);
|
|
5167
|
+
} else {
|
|
5168
|
+
resolve(null);
|
|
5169
|
+
}
|
|
5170
|
+
};
|
|
3779
5171
|
request.onerror = () => reject(request.error);
|
|
3780
5172
|
tx.onerror = () => reject(tx.error);
|
|
3781
5173
|
});
|
|
@@ -3790,6 +5182,9 @@ class IndexedDbQueueStorage {
|
|
|
3790
5182
|
await this.complete(job);
|
|
3791
5183
|
}
|
|
3792
5184
|
async delete(id) {
|
|
5185
|
+
const job = await this.get(id);
|
|
5186
|
+
if (!job)
|
|
5187
|
+
return;
|
|
3793
5188
|
const db = await this.getDb();
|
|
3794
5189
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3795
5190
|
const store = tx.objectStore(this.tableName);
|
|
@@ -3804,15 +5199,17 @@ class IndexedDbQueueStorage {
|
|
|
3804
5199
|
const db = await this.getDb();
|
|
3805
5200
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3806
5201
|
const store = tx.objectStore(this.tableName);
|
|
3807
|
-
const index = store.index("
|
|
5202
|
+
const index = store.index("queue_status");
|
|
3808
5203
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
5204
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5205
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
3809
5206
|
return new Promise((resolve, reject) => {
|
|
3810
|
-
const request = index.openCursor();
|
|
5207
|
+
const request = index.openCursor(keyRange);
|
|
3811
5208
|
request.onsuccess = (event) => {
|
|
3812
5209
|
const cursor = event.target.result;
|
|
3813
5210
|
if (cursor) {
|
|
3814
5211
|
const job = cursor.value;
|
|
3815
|
-
if (job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
|
|
5212
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
|
|
3816
5213
|
cursor.delete();
|
|
3817
5214
|
}
|
|
3818
5215
|
cursor.continue();
|
|
@@ -3823,6 +5220,363 @@ class IndexedDbQueueStorage {
|
|
|
3823
5220
|
request.onerror = () => reject(request.error);
|
|
3824
5221
|
});
|
|
3825
5222
|
}
|
|
5223
|
+
async getAllJobs() {
|
|
5224
|
+
const db = await this.getDb();
|
|
5225
|
+
const tx = db.transaction(this.tableName, "readonly");
|
|
5226
|
+
const store = tx.objectStore(this.tableName);
|
|
5227
|
+
const index = store.index("queue_status");
|
|
5228
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5229
|
+
return new Promise((resolve, reject) => {
|
|
5230
|
+
const jobs = [];
|
|
5231
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, ""]);
|
|
5232
|
+
const request = index.openCursor(keyRange);
|
|
5233
|
+
request.onsuccess = (event) => {
|
|
5234
|
+
const cursor = event.target.result;
|
|
5235
|
+
if (cursor) {
|
|
5236
|
+
const job = cursor.value;
|
|
5237
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
5238
|
+
jobs.push(job);
|
|
5239
|
+
}
|
|
5240
|
+
cursor.continue();
|
|
5241
|
+
}
|
|
5242
|
+
};
|
|
5243
|
+
tx.oncomplete = () => resolve(jobs);
|
|
5244
|
+
tx.onerror = () => reject(tx.error);
|
|
5245
|
+
request.onerror = () => reject(request.error);
|
|
5246
|
+
});
|
|
5247
|
+
}
|
|
5248
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
5249
|
+
const db = await this.getDb();
|
|
5250
|
+
const tx = db.transaction(this.tableName, "readonly");
|
|
5251
|
+
const store = tx.objectStore(this.tableName);
|
|
5252
|
+
return new Promise((resolve, reject) => {
|
|
5253
|
+
const jobs = [];
|
|
5254
|
+
const request = store.openCursor();
|
|
5255
|
+
request.onsuccess = (event) => {
|
|
5256
|
+
const cursor = event.target.result;
|
|
5257
|
+
if (cursor) {
|
|
5258
|
+
const job = cursor.value;
|
|
5259
|
+
if (job.queue !== this.queueName) {
|
|
5260
|
+
cursor.continue();
|
|
5261
|
+
return;
|
|
5262
|
+
}
|
|
5263
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
5264
|
+
jobs.push(job);
|
|
5265
|
+
} else {
|
|
5266
|
+
let matches = true;
|
|
5267
|
+
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
5268
|
+
if (job[key] !== value) {
|
|
5269
|
+
matches = false;
|
|
5270
|
+
break;
|
|
5271
|
+
}
|
|
5272
|
+
}
|
|
5273
|
+
if (matches) {
|
|
5274
|
+
jobs.push(job);
|
|
5275
|
+
}
|
|
5276
|
+
}
|
|
5277
|
+
cursor.continue();
|
|
5278
|
+
}
|
|
5279
|
+
};
|
|
5280
|
+
tx.oncomplete = () => resolve(jobs);
|
|
5281
|
+
tx.onerror = () => reject(tx.error);
|
|
5282
|
+
request.onerror = () => reject(request.error);
|
|
5283
|
+
});
|
|
5284
|
+
}
|
|
5285
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
5286
|
+
if (prefixFilter === undefined) {
|
|
5287
|
+
return false;
|
|
5288
|
+
}
|
|
5289
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
5290
|
+
return true;
|
|
5291
|
+
}
|
|
5292
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
5293
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
5294
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
5295
|
+
return true;
|
|
5296
|
+
}
|
|
5297
|
+
for (const key of instanceKeys) {
|
|
5298
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
5299
|
+
return true;
|
|
5300
|
+
}
|
|
5301
|
+
}
|
|
5302
|
+
return false;
|
|
5303
|
+
}
|
|
5304
|
+
getPollingManager() {
|
|
5305
|
+
if (!this.pollingManager) {
|
|
5306
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
5307
|
+
const jobs = await this.getAllJobs();
|
|
5308
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
5309
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
5310
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
5311
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
5312
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
5313
|
+
});
|
|
5314
|
+
}
|
|
5315
|
+
return this.pollingManager;
|
|
5316
|
+
}
|
|
5317
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
5318
|
+
let lastKnownJobs = new Map;
|
|
5319
|
+
let cancelled = false;
|
|
5320
|
+
const poll = async () => {
|
|
5321
|
+
if (cancelled)
|
|
5322
|
+
return;
|
|
5323
|
+
try {
|
|
5324
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
5325
|
+
if (cancelled)
|
|
5326
|
+
return;
|
|
5327
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
5328
|
+
for (const [id, job] of currentMap) {
|
|
5329
|
+
const old = lastKnownJobs.get(id);
|
|
5330
|
+
if (!old) {
|
|
5331
|
+
callback({ type: "INSERT", new: job });
|
|
5332
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
5333
|
+
callback({ type: "UPDATE", old, new: job });
|
|
5334
|
+
}
|
|
5335
|
+
}
|
|
5336
|
+
for (const [id, job] of lastKnownJobs) {
|
|
5337
|
+
if (!currentMap.has(id)) {
|
|
5338
|
+
callback({ type: "DELETE", old: job });
|
|
5339
|
+
}
|
|
5340
|
+
}
|
|
5341
|
+
lastKnownJobs = currentMap;
|
|
5342
|
+
} catch {}
|
|
5343
|
+
};
|
|
5344
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
5345
|
+
poll();
|
|
5346
|
+
return () => {
|
|
5347
|
+
cancelled = true;
|
|
5348
|
+
clearInterval(intervalId);
|
|
5349
|
+
};
|
|
5350
|
+
}
|
|
5351
|
+
subscribeToChanges(callback, options) {
|
|
5352
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
5353
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
5354
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
5355
|
+
}
|
|
5356
|
+
const manager = this.getPollingManager();
|
|
5357
|
+
return manager.subscribe(callback, { intervalMs });
|
|
5358
|
+
}
|
|
5359
|
+
}
|
|
5360
|
+
// src/limiter/IndexedDbRateLimiterStorage.ts
|
|
5361
|
+
import { createServiceToken as createServiceToken28 } from "@workglow/util";
|
|
5362
|
+
var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken28("ratelimiter.storage.indexedDb");
|
|
5363
|
+
|
|
5364
|
+
class IndexedDbRateLimiterStorage {
|
|
5365
|
+
executionDb;
|
|
5366
|
+
nextAvailableDb;
|
|
5367
|
+
executionTableName;
|
|
5368
|
+
nextAvailableTableName;
|
|
5369
|
+
migrationOptions;
|
|
5370
|
+
prefixes;
|
|
5371
|
+
prefixValues;
|
|
5372
|
+
constructor(options = {}) {
|
|
5373
|
+
this.migrationOptions = options;
|
|
5374
|
+
this.prefixes = options.prefixes ?? [];
|
|
5375
|
+
this.prefixValues = options.prefixValues ?? {};
|
|
5376
|
+
if (this.prefixes.length > 0) {
|
|
5377
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
5378
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
5379
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
5380
|
+
} else {
|
|
5381
|
+
this.executionTableName = "rate_limit_executions";
|
|
5382
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
5383
|
+
}
|
|
5384
|
+
}
|
|
5385
|
+
getPrefixColumnNames() {
|
|
5386
|
+
return this.prefixes.map((p) => p.name);
|
|
5387
|
+
}
|
|
5388
|
+
matchesPrefixes(record) {
|
|
5389
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5390
|
+
if (record[key] !== value) {
|
|
5391
|
+
return false;
|
|
5392
|
+
}
|
|
5393
|
+
}
|
|
5394
|
+
return true;
|
|
5395
|
+
}
|
|
5396
|
+
getPrefixKeyValues() {
|
|
5397
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
5398
|
+
}
|
|
5399
|
+
async getExecutionDb() {
|
|
5400
|
+
if (this.executionDb)
|
|
5401
|
+
return this.executionDb;
|
|
5402
|
+
await this.setupDatabase();
|
|
5403
|
+
return this.executionDb;
|
|
5404
|
+
}
|
|
5405
|
+
async getNextAvailableDb() {
|
|
5406
|
+
if (this.nextAvailableDb)
|
|
5407
|
+
return this.nextAvailableDb;
|
|
5408
|
+
await this.setupDatabase();
|
|
5409
|
+
return this.nextAvailableDb;
|
|
5410
|
+
}
|
|
5411
|
+
async setupDatabase() {
|
|
5412
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
5413
|
+
const buildKeyPath = (basePath) => {
|
|
5414
|
+
return [...prefixColumnNames, ...basePath];
|
|
5415
|
+
};
|
|
5416
|
+
const executionIndexes = [
|
|
5417
|
+
{
|
|
5418
|
+
name: "queue_executed_at",
|
|
5419
|
+
keyPath: buildKeyPath(["queue_name", "executed_at"]),
|
|
5420
|
+
options: { unique: false }
|
|
5421
|
+
}
|
|
5422
|
+
];
|
|
5423
|
+
this.executionDb = await ensureIndexedDbTable(this.executionTableName, "id", executionIndexes, this.migrationOptions);
|
|
5424
|
+
const nextAvailableIndexes = [
|
|
5425
|
+
{
|
|
5426
|
+
name: "queue_name",
|
|
5427
|
+
keyPath: buildKeyPath(["queue_name"]),
|
|
5428
|
+
options: { unique: true }
|
|
5429
|
+
}
|
|
5430
|
+
];
|
|
5431
|
+
this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
|
|
5432
|
+
}
|
|
5433
|
+
async recordExecution(queueName) {
|
|
5434
|
+
const db = await this.getExecutionDb();
|
|
5435
|
+
const tx = db.transaction(this.executionTableName, "readwrite");
|
|
5436
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5437
|
+
const record = {
|
|
5438
|
+
id: crypto.randomUUID(),
|
|
5439
|
+
queue_name: queueName,
|
|
5440
|
+
executed_at: new Date().toISOString()
|
|
5441
|
+
};
|
|
5442
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5443
|
+
record[key] = value;
|
|
5444
|
+
}
|
|
5445
|
+
return new Promise((resolve, reject) => {
|
|
5446
|
+
const request = store.add(record);
|
|
5447
|
+
tx.oncomplete = () => resolve();
|
|
5448
|
+
tx.onerror = () => reject(tx.error);
|
|
5449
|
+
request.onerror = () => reject(request.error);
|
|
5450
|
+
});
|
|
5451
|
+
}
|
|
5452
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
5453
|
+
const db = await this.getExecutionDb();
|
|
5454
|
+
const tx = db.transaction(this.executionTableName, "readonly");
|
|
5455
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5456
|
+
const index = store.index("queue_executed_at");
|
|
5457
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5458
|
+
return new Promise((resolve, reject) => {
|
|
5459
|
+
let count = 0;
|
|
5460
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartTime], [...prefixKeyValues, queueName, ""], true, false);
|
|
5461
|
+
const request = index.openCursor(keyRange);
|
|
5462
|
+
request.onsuccess = (event) => {
|
|
5463
|
+
const cursor = event.target.result;
|
|
5464
|
+
if (cursor) {
|
|
5465
|
+
const record = cursor.value;
|
|
5466
|
+
if (this.matchesPrefixes(record)) {
|
|
5467
|
+
count++;
|
|
5468
|
+
}
|
|
5469
|
+
cursor.continue();
|
|
5470
|
+
}
|
|
5471
|
+
};
|
|
5472
|
+
tx.oncomplete = () => resolve(count);
|
|
5473
|
+
tx.onerror = () => reject(tx.error);
|
|
5474
|
+
request.onerror = () => reject(request.error);
|
|
5475
|
+
});
|
|
5476
|
+
}
|
|
5477
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
5478
|
+
const db = await this.getExecutionDb();
|
|
5479
|
+
const tx = db.transaction(this.executionTableName, "readonly");
|
|
5480
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5481
|
+
const index = store.index("queue_executed_at");
|
|
5482
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5483
|
+
return new Promise((resolve, reject) => {
|
|
5484
|
+
const executions = [];
|
|
5485
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, ""]);
|
|
5486
|
+
const request = index.openCursor(keyRange);
|
|
5487
|
+
request.onsuccess = (event) => {
|
|
5488
|
+
const cursor = event.target.result;
|
|
5489
|
+
if (cursor) {
|
|
5490
|
+
const record = cursor.value;
|
|
5491
|
+
if (this.matchesPrefixes(record)) {
|
|
5492
|
+
executions.push(record.executed_at);
|
|
5493
|
+
}
|
|
5494
|
+
cursor.continue();
|
|
5495
|
+
}
|
|
5496
|
+
};
|
|
5497
|
+
tx.oncomplete = () => {
|
|
5498
|
+
executions.sort();
|
|
5499
|
+
resolve(executions[offset]);
|
|
5500
|
+
};
|
|
5501
|
+
tx.onerror = () => reject(tx.error);
|
|
5502
|
+
request.onerror = () => reject(request.error);
|
|
5503
|
+
});
|
|
5504
|
+
}
|
|
5505
|
+
async getNextAvailableTime(queueName) {
|
|
5506
|
+
const db = await this.getNextAvailableDb();
|
|
5507
|
+
const tx = db.transaction(this.nextAvailableTableName, "readonly");
|
|
5508
|
+
const store = tx.objectStore(this.nextAvailableTableName);
|
|
5509
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5510
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5511
|
+
return new Promise((resolve, reject) => {
|
|
5512
|
+
const request = store.get(key);
|
|
5513
|
+
request.onsuccess = () => {
|
|
5514
|
+
const record = request.result;
|
|
5515
|
+
if (record && this.matchesPrefixes(record)) {
|
|
5516
|
+
resolve(record.next_available_at);
|
|
5517
|
+
} else {
|
|
5518
|
+
resolve(undefined);
|
|
5519
|
+
}
|
|
5520
|
+
};
|
|
5521
|
+
request.onerror = () => reject(request.error);
|
|
5522
|
+
tx.onerror = () => reject(tx.error);
|
|
5523
|
+
});
|
|
5524
|
+
}
|
|
5525
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
5526
|
+
const db = await this.getNextAvailableDb();
|
|
5527
|
+
const tx = db.transaction(this.nextAvailableTableName, "readwrite");
|
|
5528
|
+
const store = tx.objectStore(this.nextAvailableTableName);
|
|
5529
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5530
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5531
|
+
const record = {
|
|
5532
|
+
queue_name: queueName,
|
|
5533
|
+
next_available_at: nextAvailableAt
|
|
5534
|
+
};
|
|
5535
|
+
for (const [k, value] of Object.entries(this.prefixValues)) {
|
|
5536
|
+
record[k] = value;
|
|
5537
|
+
}
|
|
5538
|
+
record[this.getPrefixColumnNames().concat(["queue_name"]).join("_")] = key;
|
|
5539
|
+
return new Promise((resolve, reject) => {
|
|
5540
|
+
const request = store.put(record);
|
|
5541
|
+
tx.oncomplete = () => resolve();
|
|
5542
|
+
tx.onerror = () => reject(tx.error);
|
|
5543
|
+
request.onerror = () => reject(request.error);
|
|
5544
|
+
});
|
|
5545
|
+
}
|
|
5546
|
+
async clear(queueName) {
|
|
5547
|
+
const execDb = await this.getExecutionDb();
|
|
5548
|
+
const execTx = execDb.transaction(this.executionTableName, "readwrite");
|
|
5549
|
+
const execStore = execTx.objectStore(this.executionTableName);
|
|
5550
|
+
const execIndex = execStore.index("queue_executed_at");
|
|
5551
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5552
|
+
await new Promise((resolve, reject) => {
|
|
5553
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, ""]);
|
|
5554
|
+
const request = execIndex.openCursor(keyRange);
|
|
5555
|
+
request.onsuccess = (event) => {
|
|
5556
|
+
const cursor = event.target.result;
|
|
5557
|
+
if (cursor) {
|
|
5558
|
+
const record = cursor.value;
|
|
5559
|
+
if (this.matchesPrefixes(record)) {
|
|
5560
|
+
cursor.delete();
|
|
5561
|
+
}
|
|
5562
|
+
cursor.continue();
|
|
5563
|
+
}
|
|
5564
|
+
};
|
|
5565
|
+
execTx.oncomplete = () => resolve();
|
|
5566
|
+
execTx.onerror = () => reject(execTx.error);
|
|
5567
|
+
request.onerror = () => reject(request.error);
|
|
5568
|
+
});
|
|
5569
|
+
const nextDb = await this.getNextAvailableDb();
|
|
5570
|
+
const nextTx = nextDb.transaction(this.nextAvailableTableName, "readwrite");
|
|
5571
|
+
const nextStore = nextTx.objectStore(this.nextAvailableTableName);
|
|
5572
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5573
|
+
await new Promise((resolve, reject) => {
|
|
5574
|
+
const request = nextStore.delete(key);
|
|
5575
|
+
nextTx.oncomplete = () => resolve();
|
|
5576
|
+
nextTx.onerror = () => reject(nextTx.error);
|
|
5577
|
+
request.onerror = () => reject(request.error);
|
|
5578
|
+
});
|
|
5579
|
+
}
|
|
3826
5580
|
}
|
|
3827
5581
|
export {
|
|
3828
5582
|
ensureIndexedDbTable,
|
|
@@ -3830,22 +5584,29 @@ export {
|
|
|
3830
5584
|
TabularRepository,
|
|
3831
5585
|
TABULAR_REPOSITORY,
|
|
3832
5586
|
SupabaseTabularRepository,
|
|
5587
|
+
SupabaseRateLimiterStorage,
|
|
3833
5588
|
SupabaseQueueStorage,
|
|
3834
5589
|
SupabaseKvRepository,
|
|
3835
5590
|
SqliteTabularRepository,
|
|
5591
|
+
SqliteRateLimiterStorage,
|
|
3836
5592
|
SqliteQueueStorage,
|
|
3837
5593
|
SqliteKvRepository,
|
|
3838
5594
|
SUPABASE_TABULAR_REPOSITORY,
|
|
5595
|
+
SUPABASE_RATE_LIMITER_STORAGE,
|
|
3839
5596
|
SUPABASE_QUEUE_STORAGE,
|
|
3840
5597
|
SUPABASE_KV_REPOSITORY,
|
|
3841
5598
|
SQLITE_TABULAR_REPOSITORY,
|
|
5599
|
+
SQLITE_RATE_LIMITER_STORAGE,
|
|
3842
5600
|
SQLITE_QUEUE_STORAGE,
|
|
3843
5601
|
SQLITE_KV_REPOSITORY,
|
|
5602
|
+
RATE_LIMITER_STORAGE,
|
|
3844
5603
|
QUEUE_STORAGE,
|
|
3845
5604
|
PostgresTabularRepository,
|
|
5605
|
+
PostgresRateLimiterStorage,
|
|
3846
5606
|
PostgresQueueStorage,
|
|
3847
5607
|
PostgresKvRepository,
|
|
3848
5608
|
POSTGRES_TABULAR_REPOSITORY,
|
|
5609
|
+
POSTGRES_RATE_LIMITER_STORAGE,
|
|
3849
5610
|
POSTGRES_QUEUE_STORAGE,
|
|
3850
5611
|
POSTGRES_KV_REPOSITORY,
|
|
3851
5612
|
MEMORY_TABULAR_REPOSITORY,
|
|
@@ -3855,12 +5616,16 @@ export {
|
|
|
3855
5616
|
KV_REPOSITORY,
|
|
3856
5617
|
JobStatus,
|
|
3857
5618
|
IndexedDbTabularRepository,
|
|
5619
|
+
IndexedDbRateLimiterStorage,
|
|
3858
5620
|
IndexedDbQueueStorage,
|
|
3859
5621
|
IndexedDbKvRepository,
|
|
3860
5622
|
InMemoryTabularRepository,
|
|
5623
|
+
InMemoryRateLimiterStorage,
|
|
3861
5624
|
InMemoryQueueStorage,
|
|
3862
5625
|
InMemoryKvRepository,
|
|
5626
|
+
IN_MEMORY_RATE_LIMITER_STORAGE,
|
|
3863
5627
|
IN_MEMORY_QUEUE_STORAGE,
|
|
5628
|
+
INDEXED_DB_RATE_LIMITER_STORAGE,
|
|
3864
5629
|
INDEXED_DB_QUEUE_STORAGE,
|
|
3865
5630
|
IDB_TABULAR_REPOSITORY,
|
|
3866
5631
|
IDB_KV_REPOSITORY,
|
|
@@ -3876,4 +5641,4 @@ export {
|
|
|
3876
5641
|
CACHED_TABULAR_REPOSITORY
|
|
3877
5642
|
};
|
|
3878
5643
|
|
|
3879
|
-
//# debugId=
|
|
5644
|
+
//# debugId=A8A981E6F7A25E7F64756E2164756E21
|