@workglow/storage 0.0.57 → 0.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +1266 -121
- package/dist/browser.js.map +15 -10
- package/dist/bun.js +2017 -252
- package/dist/bun.js.map +19 -12
- package/dist/common-server.d.ts +4 -0
- package/dist/common-server.d.ts.map +1 -1
- package/dist/common.d.ts +2 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/limiter/IRateLimiterStorage.d.ts +81 -0
- package/dist/limiter/IRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/InMemoryRateLimiterStorage.d.ts +32 -0
- package/dist/limiter/InMemoryRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/IndexedDbRateLimiterStorage.d.ts +52 -0
- package/dist/limiter/IndexedDbRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/PostgresRateLimiterStorage.d.ts +54 -0
- package/dist/limiter/PostgresRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/SqliteRateLimiterStorage.d.ts +53 -0
- package/dist/limiter/SqliteRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/SupabaseRateLimiterStorage.d.ts +53 -0
- package/dist/limiter/SupabaseRateLimiterStorage.d.ts.map +1 -0
- package/dist/node.js +2017 -252
- package/dist/node.js.map +19 -12
- package/dist/queue/IQueueStorage.d.ts +72 -1
- package/dist/queue/IQueueStorage.d.ts.map +1 -1
- package/dist/queue/InMemoryQueueStorage.d.ts +44 -11
- package/dist/queue/InMemoryQueueStorage.d.ts.map +1 -1
- package/dist/queue/IndexedDbQueueStorage.d.ts +70 -5
- package/dist/queue/IndexedDbQueueStorage.d.ts.map +1 -1
- package/dist/queue/PostgresQueueStorage.d.ts +80 -8
- package/dist/queue/PostgresQueueStorage.d.ts.map +1 -1
- package/dist/queue/SqliteQueueStorage.d.ts +90 -34
- package/dist/queue/SqliteQueueStorage.d.ts.map +1 -1
- package/dist/queue/SupabaseQueueStorage.d.ts +98 -4
- package/dist/queue/SupabaseQueueStorage.d.ts.map +1 -1
- package/dist/tabular/ITabularRepository.d.ts +18 -0
- package/dist/tabular/ITabularRepository.d.ts.map +1 -1
- package/dist/tabular/InMemoryTabularRepository.d.ts +9 -1
- package/dist/tabular/InMemoryTabularRepository.d.ts.map +1 -1
- package/dist/tabular/SupabaseTabularRepository.d.ts +21 -1
- package/dist/tabular/SupabaseTabularRepository.d.ts.map +1 -1
- package/dist/tabular/TabularRepository.d.ts +10 -1
- package/dist/tabular/TabularRepository.d.ts.map +1 -1
- package/dist/util/PollingSubscriptionManager.d.ts +112 -0
- package/dist/util/PollingSubscriptionManager.d.ts.map +1 -0
- package/package.json +5 -5
package/dist/bun.js
CHANGED
|
@@ -107,6 +107,9 @@ class TabularRepository {
|
|
|
107
107
|
waitOn(name) {
|
|
108
108
|
return this.events.waitOn(name);
|
|
109
109
|
}
|
|
110
|
+
subscribeToChanges(_callback) {
|
|
111
|
+
throw new Error(`subscribeToChanges is not implemented for ${this.constructor.name}. ` + `Use InMemoryTabularRepository or SupabaseTabularRepository for subscription support.`);
|
|
112
|
+
}
|
|
110
113
|
primaryKeyColumns() {
|
|
111
114
|
const columns = [];
|
|
112
115
|
for (const key of Object.keys(this.primaryKeySchema.properties)) {
|
|
@@ -274,12 +277,30 @@ class InMemoryTabularRepository extends TabularRepository {
|
|
|
274
277
|
return false;
|
|
275
278
|
}
|
|
276
279
|
});
|
|
277
|
-
for (const [id,
|
|
280
|
+
for (const [id, entity] of entriesToDelete) {
|
|
278
281
|
this.values.delete(id);
|
|
282
|
+
const { key } = this.separateKeyValueFromCombined(entity);
|
|
283
|
+
this.events.emit("delete", key);
|
|
279
284
|
}
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
285
|
+
}
|
|
286
|
+
subscribeToChanges(callback) {
|
|
287
|
+
const handlePut = (entity) => {
|
|
288
|
+
callback({ type: "UPDATE", new: entity });
|
|
289
|
+
};
|
|
290
|
+
const handleDelete = (_key) => {
|
|
291
|
+
callback({ type: "DELETE" });
|
|
292
|
+
};
|
|
293
|
+
const handleClearAll = () => {
|
|
294
|
+
callback({ type: "DELETE" });
|
|
295
|
+
};
|
|
296
|
+
this.events.on("put", handlePut);
|
|
297
|
+
this.events.on("delete", handleDelete);
|
|
298
|
+
this.events.on("clearall", handleClearAll);
|
|
299
|
+
return () => {
|
|
300
|
+
this.events.off("put", handlePut);
|
|
301
|
+
this.events.off("delete", handleDelete);
|
|
302
|
+
this.events.off("clearall", handleClearAll);
|
|
303
|
+
};
|
|
283
304
|
}
|
|
284
305
|
destroy() {
|
|
285
306
|
this.values.clear();
|
|
@@ -541,7 +562,7 @@ class InMemoryKvRepository extends KvViaTabularRepository {
|
|
|
541
562
|
}
|
|
542
563
|
}
|
|
543
564
|
// src/queue/InMemoryQueueStorage.ts
|
|
544
|
-
import { createServiceToken as createServiceToken7, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
|
|
565
|
+
import { createServiceToken as createServiceToken7, EventEmitter as EventEmitter3, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
|
|
545
566
|
|
|
546
567
|
// src/queue/IQueueStorage.ts
|
|
547
568
|
import { createServiceToken as createServiceToken6 } from "@workglow/util";
|
|
@@ -561,63 +582,88 @@ var IN_MEMORY_QUEUE_STORAGE = createServiceToken7("jobqueue.storage.inMemory");
|
|
|
561
582
|
|
|
562
583
|
class InMemoryQueueStorage {
|
|
563
584
|
queueName;
|
|
564
|
-
|
|
585
|
+
prefixValues;
|
|
586
|
+
events = new EventEmitter3;
|
|
587
|
+
constructor(queueName, options) {
|
|
565
588
|
this.queueName = queueName;
|
|
566
589
|
this.jobQueue = [];
|
|
590
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
567
591
|
}
|
|
568
592
|
jobQueue;
|
|
593
|
+
matchesPrefixes(job) {
|
|
594
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
595
|
+
if (job[key] !== value) {
|
|
596
|
+
return false;
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
return true;
|
|
600
|
+
}
|
|
569
601
|
pendingQueue() {
|
|
570
602
|
const now = new Date().toISOString();
|
|
571
|
-
return this.jobQueue.filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
603
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
572
604
|
}
|
|
573
605
|
async add(job) {
|
|
574
606
|
await sleep(0);
|
|
575
607
|
const now = new Date().toISOString();
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
608
|
+
const jobWithPrefixes = job;
|
|
609
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid4();
|
|
610
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid4();
|
|
611
|
+
jobWithPrefixes.queue = this.queueName;
|
|
612
|
+
jobWithPrefixes.fingerprint = await makeFingerprint4(jobWithPrefixes.input);
|
|
613
|
+
jobWithPrefixes.status = "PENDING" /* PENDING */;
|
|
614
|
+
jobWithPrefixes.progress = 0;
|
|
615
|
+
jobWithPrefixes.progress_message = "";
|
|
616
|
+
jobWithPrefixes.progress_details = null;
|
|
617
|
+
jobWithPrefixes.created_at = now;
|
|
618
|
+
jobWithPrefixes.run_after = now;
|
|
619
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
620
|
+
jobWithPrefixes[key] = value;
|
|
621
|
+
}
|
|
622
|
+
this.jobQueue.push(jobWithPrefixes);
|
|
623
|
+
this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
|
|
624
|
+
return jobWithPrefixes.id;
|
|
588
625
|
}
|
|
589
626
|
async get(id) {
|
|
590
627
|
await sleep(0);
|
|
591
|
-
|
|
628
|
+
const job = this.jobQueue.find((j) => j.id === id);
|
|
629
|
+
if (job && this.matchesPrefixes(job)) {
|
|
630
|
+
return job;
|
|
631
|
+
}
|
|
632
|
+
return;
|
|
592
633
|
}
|
|
593
634
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
594
635
|
await sleep(0);
|
|
595
636
|
num = Number(num) || 100;
|
|
596
|
-
return this.jobQueue.sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
637
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
597
638
|
}
|
|
598
|
-
async next() {
|
|
639
|
+
async next(workerId) {
|
|
599
640
|
await sleep(0);
|
|
600
641
|
const top = this.pendingQueue();
|
|
601
642
|
const job = top[0];
|
|
602
643
|
if (job) {
|
|
644
|
+
const oldJob = { ...job };
|
|
603
645
|
job.status = "PROCESSING" /* PROCESSING */;
|
|
604
646
|
job.last_ran_at = new Date().toISOString();
|
|
647
|
+
job.worker_id = workerId ?? null;
|
|
648
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
605
649
|
return job;
|
|
606
650
|
}
|
|
607
651
|
}
|
|
608
652
|
async size(status = "PENDING" /* PENDING */) {
|
|
609
653
|
await sleep(0);
|
|
610
|
-
return this.jobQueue.filter((j) => j.status === status).length;
|
|
654
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
|
|
611
655
|
}
|
|
612
656
|
async saveProgress(id, progress, message, details) {
|
|
613
657
|
await sleep(0);
|
|
614
|
-
const job = this.jobQueue.find((j) => j.id === id);
|
|
658
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
615
659
|
if (!job) {
|
|
616
660
|
throw new Error(`Job ${id} not found`);
|
|
617
661
|
}
|
|
662
|
+
const oldJob = { ...job };
|
|
618
663
|
job.progress = progress;
|
|
619
664
|
job.progress_message = message;
|
|
620
665
|
job.progress_details = details;
|
|
666
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
621
667
|
}
|
|
622
668
|
async complete(job) {
|
|
623
669
|
await sleep(0);
|
|
@@ -627,44 +673,149 @@ class InMemoryQueueStorage {
|
|
|
627
673
|
const currentAttempts = existing?.run_attempts ?? 0;
|
|
628
674
|
job.run_attempts = currentAttempts + 1;
|
|
629
675
|
this.jobQueue[index] = job;
|
|
676
|
+
this.events.emit("change", { type: "UPDATE", old: existing, new: job });
|
|
630
677
|
}
|
|
631
678
|
}
|
|
632
679
|
async abort(id) {
|
|
633
680
|
await sleep(0);
|
|
634
|
-
const job = this.jobQueue.find((j) => j.id === id);
|
|
681
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
635
682
|
if (job) {
|
|
683
|
+
const oldJob = { ...job };
|
|
636
684
|
job.status = "ABORTING" /* ABORTING */;
|
|
685
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
637
686
|
}
|
|
638
687
|
}
|
|
639
688
|
async getByRunId(runId) {
|
|
640
689
|
await sleep(0);
|
|
641
|
-
return this.jobQueue.filter((job) => job.job_run_id === runId);
|
|
690
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
|
|
642
691
|
}
|
|
643
692
|
async deleteAll() {
|
|
644
693
|
await sleep(0);
|
|
645
|
-
this.jobQueue
|
|
694
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
|
|
695
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
|
|
696
|
+
for (const job of deletedJobs) {
|
|
697
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
698
|
+
}
|
|
646
699
|
}
|
|
647
700
|
async outputForInput(input) {
|
|
648
701
|
await sleep(0);
|
|
649
702
|
const fingerprint = await makeFingerprint4(input);
|
|
650
|
-
return this.jobQueue.find((j) => j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
|
|
703
|
+
return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
|
|
651
704
|
}
|
|
652
705
|
async delete(id) {
|
|
653
706
|
await sleep(0);
|
|
654
|
-
|
|
707
|
+
const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
|
|
708
|
+
this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
|
|
709
|
+
if (deletedJob) {
|
|
710
|
+
this.events.emit("change", { type: "DELETE", old: deletedJob });
|
|
711
|
+
}
|
|
655
712
|
}
|
|
656
713
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
657
714
|
await sleep(0);
|
|
658
715
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
659
|
-
|
|
716
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
|
|
717
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
|
|
718
|
+
for (const job of deletedJobs) {
|
|
719
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
async setupDatabase() {}
|
|
723
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
724
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
725
|
+
return true;
|
|
726
|
+
}
|
|
727
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
728
|
+
if (Object.keys(filterValues).length === 0) {
|
|
729
|
+
return true;
|
|
730
|
+
}
|
|
731
|
+
const jobWithPrefixes = job;
|
|
732
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
733
|
+
if (jobWithPrefixes[key] !== value) {
|
|
734
|
+
return false;
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
return true;
|
|
738
|
+
}
|
|
739
|
+
subscribeToChanges(callback, options) {
|
|
740
|
+
const prefixFilter = options?.prefixFilter;
|
|
741
|
+
const filteredCallback = (change) => {
|
|
742
|
+
const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
|
|
743
|
+
const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
|
|
744
|
+
if (!newMatches && !oldMatches) {
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
747
|
+
callback(change);
|
|
748
|
+
};
|
|
749
|
+
return this.events.subscribe("change", filteredCallback);
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
// src/limiter/IRateLimiterStorage.ts
|
|
753
|
+
import { createServiceToken as createServiceToken8 } from "@workglow/util";
|
|
754
|
+
var RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage");
|
|
755
|
+
// src/limiter/InMemoryRateLimiterStorage.ts
|
|
756
|
+
import { createServiceToken as createServiceToken9, sleep as sleep2 } from "@workglow/util";
|
|
757
|
+
var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken9("ratelimiter.storage.inMemory");
|
|
758
|
+
|
|
759
|
+
class InMemoryRateLimiterStorage {
|
|
760
|
+
prefixValues;
|
|
761
|
+
executions = new Map;
|
|
762
|
+
nextAvailableTimes = new Map;
|
|
763
|
+
constructor(options) {
|
|
764
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
765
|
+
}
|
|
766
|
+
makeKey(queueName) {
|
|
767
|
+
const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
|
|
768
|
+
return prefixPart ? `${prefixPart}|${queueName}` : queueName;
|
|
660
769
|
}
|
|
661
770
|
async setupDatabase() {}
|
|
771
|
+
async recordExecution(queueName) {
|
|
772
|
+
await sleep2(0);
|
|
773
|
+
const key = this.makeKey(queueName);
|
|
774
|
+
const executions = this.executions.get(key) ?? [];
|
|
775
|
+
executions.push({
|
|
776
|
+
queueName,
|
|
777
|
+
executedAt: new Date
|
|
778
|
+
});
|
|
779
|
+
this.executions.set(key, executions);
|
|
780
|
+
}
|
|
781
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
782
|
+
await sleep2(0);
|
|
783
|
+
const key = this.makeKey(queueName);
|
|
784
|
+
const executions = this.executions.get(key) ?? [];
|
|
785
|
+
const windowStart = new Date(windowStartTime);
|
|
786
|
+
return executions.filter((e) => e.executedAt > windowStart).length;
|
|
787
|
+
}
|
|
788
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
789
|
+
await sleep2(0);
|
|
790
|
+
const key = this.makeKey(queueName);
|
|
791
|
+
const executions = this.executions.get(key) ?? [];
|
|
792
|
+
const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
|
|
793
|
+
const execution = sorted[offset];
|
|
794
|
+
return execution?.executedAt.toISOString();
|
|
795
|
+
}
|
|
796
|
+
async getNextAvailableTime(queueName) {
|
|
797
|
+
await sleep2(0);
|
|
798
|
+
const key = this.makeKey(queueName);
|
|
799
|
+
const time = this.nextAvailableTimes.get(key);
|
|
800
|
+
return time?.toISOString();
|
|
801
|
+
}
|
|
802
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
803
|
+
await sleep2(0);
|
|
804
|
+
const key = this.makeKey(queueName);
|
|
805
|
+
this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
|
|
806
|
+
}
|
|
807
|
+
async clear(queueName) {
|
|
808
|
+
await sleep2(0);
|
|
809
|
+
const key = this.makeKey(queueName);
|
|
810
|
+
this.executions.delete(key);
|
|
811
|
+
this.nextAvailableTimes.delete(key);
|
|
812
|
+
}
|
|
662
813
|
}
|
|
663
814
|
// src/tabular/FsFolderTabularRepository.ts
|
|
664
|
-
import { createServiceToken as
|
|
815
|
+
import { createServiceToken as createServiceToken10, sleep as sleep3 } from "@workglow/util";
|
|
665
816
|
import { mkdir, readdir, readFile, rm, writeFile } from "fs/promises";
|
|
666
817
|
import path from "path";
|
|
667
|
-
var FS_FOLDER_TABULAR_REPOSITORY =
|
|
818
|
+
var FS_FOLDER_TABULAR_REPOSITORY = createServiceToken10("storage.tabularRepository.fsFolder");
|
|
668
819
|
|
|
669
820
|
class FsFolderTabularRepository extends TabularRepository {
|
|
670
821
|
folderPath;
|
|
@@ -689,7 +840,7 @@ class FsFolderTabularRepository extends TabularRepository {
|
|
|
689
840
|
await writeFile(filePath, JSON.stringify(entity));
|
|
690
841
|
} catch (error) {
|
|
691
842
|
try {
|
|
692
|
-
await
|
|
843
|
+
await sleep3(1);
|
|
693
844
|
await writeFile(filePath, JSON.stringify(entity));
|
|
694
845
|
} catch (error2) {
|
|
695
846
|
console.error("Error writing file", filePath, error2);
|
|
@@ -773,7 +924,7 @@ class FsFolderTabularRepository extends TabularRepository {
|
|
|
773
924
|
}
|
|
774
925
|
}
|
|
775
926
|
// src/tabular/PostgresTabularRepository.ts
|
|
776
|
-
import { createServiceToken as
|
|
927
|
+
import { createServiceToken as createServiceToken11 } from "@workglow/util";
|
|
777
928
|
|
|
778
929
|
// src/tabular/BaseSqlTabularRepository.ts
|
|
779
930
|
class BaseSqlTabularRepository extends TabularRepository {
|
|
@@ -959,7 +1110,7 @@ class BaseSqlTabularRepository extends TabularRepository {
|
|
|
959
1110
|
}
|
|
960
1111
|
|
|
961
1112
|
// src/tabular/PostgresTabularRepository.ts
|
|
962
|
-
var POSTGRES_TABULAR_REPOSITORY =
|
|
1113
|
+
var POSTGRES_TABULAR_REPOSITORY = createServiceToken11("storage.tabularRepository.postgres");
|
|
963
1114
|
|
|
964
1115
|
class PostgresTabularRepository extends BaseSqlTabularRepository {
|
|
965
1116
|
db;
|
|
@@ -1317,8 +1468,8 @@ class PostgresTabularRepository extends BaseSqlTabularRepository {
|
|
|
1317
1468
|
}
|
|
1318
1469
|
// src/tabular/SqliteTabularRepository.ts
|
|
1319
1470
|
import { Sqlite } from "@workglow/sqlite";
|
|
1320
|
-
import { createServiceToken as
|
|
1321
|
-
var SQLITE_TABULAR_REPOSITORY =
|
|
1471
|
+
import { createServiceToken as createServiceToken12 } from "@workglow/util";
|
|
1472
|
+
var SQLITE_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.sqlite");
|
|
1322
1473
|
var Database = Sqlite.Database;
|
|
1323
1474
|
|
|
1324
1475
|
class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
@@ -1595,11 +1746,12 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1595
1746
|
}
|
|
1596
1747
|
}
|
|
1597
1748
|
// src/tabular/SupabaseTabularRepository.ts
|
|
1598
|
-
import { createServiceToken as
|
|
1599
|
-
var SUPABASE_TABULAR_REPOSITORY =
|
|
1749
|
+
import { createServiceToken as createServiceToken13 } from "@workglow/util";
|
|
1750
|
+
var SUPABASE_TABULAR_REPOSITORY = createServiceToken13("storage.tabularRepository.supabase");
|
|
1600
1751
|
|
|
1601
1752
|
class SupabaseTabularRepository extends BaseSqlTabularRepository {
|
|
1602
1753
|
client;
|
|
1754
|
+
realtimeChannel = null;
|
|
1603
1755
|
constructor(client, table = "tabular_store", schema, primaryKeyNames, indexes = []) {
|
|
1604
1756
|
super(table, schema, primaryKeyNames, indexes);
|
|
1605
1757
|
this.client = client;
|
|
@@ -1972,10 +2124,44 @@ class SupabaseTabularRepository extends BaseSqlTabularRepository {
|
|
|
1972
2124
|
throw error;
|
|
1973
2125
|
this.events.emit("delete", column);
|
|
1974
2126
|
}
|
|
2127
|
+
convertRealtimeRow(row) {
|
|
2128
|
+
const entity = { ...row };
|
|
2129
|
+
for (const key in this.schema.properties) {
|
|
2130
|
+
entity[key] = this.sqlToJsValue(key, row[key]);
|
|
2131
|
+
}
|
|
2132
|
+
return entity;
|
|
2133
|
+
}
|
|
2134
|
+
subscribeToChanges(callback) {
|
|
2135
|
+
const channelName = `tabular-${this.table}-${Date.now()}`;
|
|
2136
|
+
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
2137
|
+
event: "*",
|
|
2138
|
+
schema: "public",
|
|
2139
|
+
table: this.table
|
|
2140
|
+
}, (payload) => {
|
|
2141
|
+
const change = {
|
|
2142
|
+
type: payload.eventType.toUpperCase(),
|
|
2143
|
+
old: payload.old && Object.keys(payload.old).length > 0 ? this.convertRealtimeRow(payload.old) : undefined,
|
|
2144
|
+
new: payload.new && Object.keys(payload.new).length > 0 ? this.convertRealtimeRow(payload.new) : undefined
|
|
2145
|
+
};
|
|
2146
|
+
callback(change);
|
|
2147
|
+
}).subscribe();
|
|
2148
|
+
return () => {
|
|
2149
|
+
if (this.realtimeChannel) {
|
|
2150
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
2151
|
+
this.realtimeChannel = null;
|
|
2152
|
+
}
|
|
2153
|
+
};
|
|
2154
|
+
}
|
|
2155
|
+
destroy() {
|
|
2156
|
+
if (this.realtimeChannel) {
|
|
2157
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
2158
|
+
this.realtimeChannel = null;
|
|
2159
|
+
}
|
|
2160
|
+
}
|
|
1975
2161
|
}
|
|
1976
2162
|
// src/kv/FsFolderJsonKvRepository.ts
|
|
1977
|
-
import { createServiceToken as
|
|
1978
|
-
var FS_FOLDER_JSON_KV_REPOSITORY =
|
|
2163
|
+
import { createServiceToken as createServiceToken14 } from "@workglow/util";
|
|
2164
|
+
var FS_FOLDER_JSON_KV_REPOSITORY = createServiceToken14("storage.kvRepository.fsFolderJson");
|
|
1979
2165
|
|
|
1980
2166
|
class FsFolderJsonKvRepository extends KvViaTabularRepository {
|
|
1981
2167
|
folderPath;
|
|
@@ -1987,10 +2173,10 @@ class FsFolderJsonKvRepository extends KvViaTabularRepository {
|
|
|
1987
2173
|
}
|
|
1988
2174
|
}
|
|
1989
2175
|
// src/kv/FsFolderKvRepository.ts
|
|
1990
|
-
import { createServiceToken as
|
|
2176
|
+
import { createServiceToken as createServiceToken15 } from "@workglow/util";
|
|
1991
2177
|
import { mkdir as mkdir2, readFile as readFile2, rm as rm2, unlink, writeFile as writeFile2 } from "fs/promises";
|
|
1992
2178
|
import path2 from "path";
|
|
1993
|
-
var FS_FOLDER_KV_REPOSITORY =
|
|
2179
|
+
var FS_FOLDER_KV_REPOSITORY = createServiceToken15("storage.kvRepository.fsFolder");
|
|
1994
2180
|
|
|
1995
2181
|
class FsFolderKvRepository extends KvRepository {
|
|
1996
2182
|
folderPath;
|
|
@@ -2067,8 +2253,8 @@ class FsFolderKvRepository extends KvRepository {
|
|
|
2067
2253
|
}
|
|
2068
2254
|
}
|
|
2069
2255
|
// src/kv/PostgresKvRepository.ts
|
|
2070
|
-
import { createServiceToken as
|
|
2071
|
-
var POSTGRES_KV_REPOSITORY =
|
|
2256
|
+
import { createServiceToken as createServiceToken16 } from "@workglow/util";
|
|
2257
|
+
var POSTGRES_KV_REPOSITORY = createServiceToken16("storage.kvRepository.postgres");
|
|
2072
2258
|
|
|
2073
2259
|
class PostgresKvRepository extends KvViaTabularRepository {
|
|
2074
2260
|
db;
|
|
@@ -2082,8 +2268,8 @@ class PostgresKvRepository extends KvViaTabularRepository {
|
|
|
2082
2268
|
}
|
|
2083
2269
|
}
|
|
2084
2270
|
// src/kv/SqliteKvRepository.ts
|
|
2085
|
-
import { createServiceToken as
|
|
2086
|
-
var SQLITE_KV_REPOSITORY =
|
|
2271
|
+
import { createServiceToken as createServiceToken17 } from "@workglow/util";
|
|
2272
|
+
var SQLITE_KV_REPOSITORY = createServiceToken17("storage.kvRepository.sqlite");
|
|
2087
2273
|
|
|
2088
2274
|
class SqliteKvRepository extends KvViaTabularRepository {
|
|
2089
2275
|
db;
|
|
@@ -2097,8 +2283,8 @@ class SqliteKvRepository extends KvViaTabularRepository {
|
|
|
2097
2283
|
}
|
|
2098
2284
|
}
|
|
2099
2285
|
// src/kv/SupabaseKvRepository.ts
|
|
2100
|
-
import { createServiceToken as
|
|
2101
|
-
var SUPABASE_KV_REPOSITORY =
|
|
2286
|
+
import { createServiceToken as createServiceToken18 } from "@workglow/util";
|
|
2287
|
+
var SUPABASE_KV_REPOSITORY = createServiceToken18("storage.kvRepository.supabase");
|
|
2102
2288
|
|
|
2103
2289
|
class SupabaseKvRepository extends KvViaTabularRepository {
|
|
2104
2290
|
client;
|
|
@@ -2112,15 +2298,169 @@ class SupabaseKvRepository extends KvViaTabularRepository {
|
|
|
2112
2298
|
}
|
|
2113
2299
|
}
|
|
2114
2300
|
// src/queue/PostgresQueueStorage.ts
|
|
2115
|
-
import { createServiceToken as
|
|
2116
|
-
|
|
2301
|
+
import { createServiceToken as createServiceToken19, makeFingerprint as makeFingerprint5, uuid4 as uuid42 } from "@workglow/util";
|
|
2302
|
+
|
|
2303
|
+
// src/util/PollingSubscriptionManager.ts
|
|
2304
|
+
class PollingSubscriptionManager {
|
|
2305
|
+
intervals = new Map;
|
|
2306
|
+
lastKnownState = new Map;
|
|
2307
|
+
initialized = false;
|
|
2308
|
+
fetchState;
|
|
2309
|
+
compareItems;
|
|
2310
|
+
payloadFactory;
|
|
2311
|
+
defaultIntervalMs;
|
|
2312
|
+
constructor(fetchState, compareItems, payloadFactory, options) {
|
|
2313
|
+
this.fetchState = fetchState;
|
|
2314
|
+
this.compareItems = compareItems;
|
|
2315
|
+
this.payloadFactory = payloadFactory;
|
|
2316
|
+
this.defaultIntervalMs = options?.defaultIntervalMs ?? 1000;
|
|
2317
|
+
}
|
|
2318
|
+
subscribe(callback, options) {
|
|
2319
|
+
const interval = options?.intervalMs ?? this.defaultIntervalMs;
|
|
2320
|
+
const subscription = {
|
|
2321
|
+
callback,
|
|
2322
|
+
intervalMs: interval
|
|
2323
|
+
};
|
|
2324
|
+
let intervalGroup = this.intervals.get(interval);
|
|
2325
|
+
if (!intervalGroup) {
|
|
2326
|
+
const subscribers = new Set;
|
|
2327
|
+
const intervalId = setInterval(() => this.poll(subscribers), interval);
|
|
2328
|
+
intervalGroup = { intervalId, subscribers };
|
|
2329
|
+
this.intervals.set(interval, intervalGroup);
|
|
2330
|
+
if (!this.initialized) {
|
|
2331
|
+
this.initialized = true;
|
|
2332
|
+
this.initAndPoll(subscribers, subscription);
|
|
2333
|
+
} else {
|
|
2334
|
+
this.pollForNewSubscriber(subscription);
|
|
2335
|
+
}
|
|
2336
|
+
} else {
|
|
2337
|
+
this.pollForNewSubscriber(subscription);
|
|
2338
|
+
}
|
|
2339
|
+
intervalGroup.subscribers.add(subscription);
|
|
2340
|
+
return () => {
|
|
2341
|
+
const group = this.intervals.get(interval);
|
|
2342
|
+
if (group) {
|
|
2343
|
+
group.subscribers.delete(subscription);
|
|
2344
|
+
if (group.subscribers.size === 0) {
|
|
2345
|
+
clearInterval(group.intervalId);
|
|
2346
|
+
this.intervals.delete(interval);
|
|
2347
|
+
}
|
|
2348
|
+
}
|
|
2349
|
+
};
|
|
2350
|
+
}
|
|
2351
|
+
async initAndPoll(subscribers, newSubscription) {
|
|
2352
|
+
try {
|
|
2353
|
+
this.lastKnownState = await this.fetchState();
|
|
2354
|
+
for (const [, item] of this.lastKnownState) {
|
|
2355
|
+
const payload = this.payloadFactory.insert(item);
|
|
2356
|
+
try {
|
|
2357
|
+
newSubscription.callback(payload);
|
|
2358
|
+
} catch {}
|
|
2359
|
+
}
|
|
2360
|
+
} catch {}
|
|
2361
|
+
}
|
|
2362
|
+
pollForNewSubscriber(subscription) {
|
|
2363
|
+
for (const [, item] of this.lastKnownState) {
|
|
2364
|
+
const payload = this.payloadFactory.insert(item);
|
|
2365
|
+
try {
|
|
2366
|
+
subscription.callback(payload);
|
|
2367
|
+
} catch {}
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
async poll(subscribers) {
|
|
2371
|
+
if (subscribers.size === 0)
|
|
2372
|
+
return;
|
|
2373
|
+
try {
|
|
2374
|
+
const currentState = await this.fetchState();
|
|
2375
|
+
const changes = [];
|
|
2376
|
+
for (const [key, item] of currentState) {
|
|
2377
|
+
const oldItem = this.lastKnownState.get(key);
|
|
2378
|
+
if (!oldItem) {
|
|
2379
|
+
changes.push(this.payloadFactory.insert(item));
|
|
2380
|
+
} else if (!this.compareItems(oldItem, item)) {
|
|
2381
|
+
changes.push(this.payloadFactory.update(oldItem, item));
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
for (const [key, item] of this.lastKnownState) {
|
|
2385
|
+
if (!currentState.has(key)) {
|
|
2386
|
+
changes.push(this.payloadFactory.delete(item));
|
|
2387
|
+
}
|
|
2388
|
+
}
|
|
2389
|
+
this.lastKnownState = currentState;
|
|
2390
|
+
for (const change of changes) {
|
|
2391
|
+
for (const sub of subscribers) {
|
|
2392
|
+
try {
|
|
2393
|
+
sub.callback(change);
|
|
2394
|
+
} catch {}
|
|
2395
|
+
}
|
|
2396
|
+
}
|
|
2397
|
+
} catch {}
|
|
2398
|
+
}
|
|
2399
|
+
get subscriptionCount() {
|
|
2400
|
+
let count = 0;
|
|
2401
|
+
for (const group of this.intervals.values()) {
|
|
2402
|
+
count += group.subscribers.size;
|
|
2403
|
+
}
|
|
2404
|
+
return count;
|
|
2405
|
+
}
|
|
2406
|
+
get hasSubscriptions() {
|
|
2407
|
+
return this.intervals.size > 0;
|
|
2408
|
+
}
|
|
2409
|
+
destroy() {
|
|
2410
|
+
for (const group of this.intervals.values()) {
|
|
2411
|
+
clearInterval(group.intervalId);
|
|
2412
|
+
}
|
|
2413
|
+
this.intervals.clear();
|
|
2414
|
+
this.lastKnownState.clear();
|
|
2415
|
+
this.initialized = false;
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
|
|
2419
|
+
// src/queue/PostgresQueueStorage.ts
|
|
2420
|
+
var POSTGRES_QUEUE_STORAGE = createServiceToken19("jobqueue.storage.postgres");
|
|
2117
2421
|
|
|
2118
2422
|
class PostgresQueueStorage {
|
|
2119
2423
|
db;
|
|
2120
2424
|
queueName;
|
|
2121
|
-
|
|
2425
|
+
prefixes;
|
|
2426
|
+
prefixValues;
|
|
2427
|
+
tableName;
|
|
2428
|
+
pollingManager = null;
|
|
2429
|
+
constructor(db, queueName, options) {
|
|
2122
2430
|
this.db = db;
|
|
2123
2431
|
this.queueName = queueName;
|
|
2432
|
+
this.prefixes = options?.prefixes ?? [];
|
|
2433
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
2434
|
+
if (this.prefixes.length > 0) {
|
|
2435
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
2436
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
2437
|
+
} else {
|
|
2438
|
+
this.tableName = "job_queue";
|
|
2439
|
+
}
|
|
2440
|
+
}
|
|
2441
|
+
getPrefixColumnType(type) {
|
|
2442
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
2443
|
+
}
|
|
2444
|
+
buildPrefixColumnsSql() {
|
|
2445
|
+
if (this.prefixes.length === 0)
|
|
2446
|
+
return "";
|
|
2447
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
2448
|
+
`) + `,
|
|
2449
|
+
`;
|
|
2450
|
+
}
|
|
2451
|
+
getPrefixColumnNames() {
|
|
2452
|
+
return this.prefixes.map((p) => p.name);
|
|
2453
|
+
}
|
|
2454
|
+
buildPrefixWhereClause(startParam) {
|
|
2455
|
+
if (this.prefixes.length === 0) {
|
|
2456
|
+
return { conditions: "", params: [] };
|
|
2457
|
+
}
|
|
2458
|
+
const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
|
|
2459
|
+
const params = this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2460
|
+
return { conditions: " AND " + conditions, params };
|
|
2461
|
+
}
|
|
2462
|
+
getPrefixParamValues() {
|
|
2463
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2124
2464
|
}
|
|
2125
2465
|
async setupDatabase() {
|
|
2126
2466
|
let sql;
|
|
@@ -2131,10 +2471,13 @@ class PostgresQueueStorage {
|
|
|
2131
2471
|
if (e.code !== "42710")
|
|
2132
2472
|
throw e;
|
|
2133
2473
|
}
|
|
2474
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
2475
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2476
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2134
2477
|
sql = `
|
|
2135
|
-
CREATE TABLE IF NOT EXISTS
|
|
2478
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
2136
2479
|
id SERIAL NOT NULL,
|
|
2137
|
-
fingerprint text NOT NULL,
|
|
2480
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
2138
2481
|
queue text NOT NULL,
|
|
2139
2482
|
job_run_id text NOT NULL,
|
|
2140
2483
|
status job_status NOT NULL default 'PENDING',
|
|
@@ -2151,20 +2494,22 @@ class PostgresQueueStorage {
|
|
|
2151
2494
|
error_code text,
|
|
2152
2495
|
progress real DEFAULT 0,
|
|
2153
2496
|
progress_message text DEFAULT '',
|
|
2154
|
-
progress_details jsonb
|
|
2497
|
+
progress_details jsonb,
|
|
2498
|
+
worker_id text
|
|
2155
2499
|
)`;
|
|
2156
2500
|
await this.db.query(sql);
|
|
2501
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
2157
2502
|
sql = `
|
|
2158
|
-
CREATE INDEX IF NOT EXISTS
|
|
2159
|
-
ON
|
|
2503
|
+
CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx
|
|
2504
|
+
ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`;
|
|
2160
2505
|
await this.db.query(sql);
|
|
2161
2506
|
sql = `
|
|
2162
|
-
CREATE INDEX IF NOT EXISTS
|
|
2163
|
-
ON
|
|
2507
|
+
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx
|
|
2508
|
+
ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`;
|
|
2164
2509
|
await this.db.query(sql);
|
|
2165
2510
|
sql = `
|
|
2166
|
-
CREATE INDEX IF NOT EXISTS
|
|
2167
|
-
ON
|
|
2511
|
+
CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx
|
|
2512
|
+
ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`;
|
|
2168
2513
|
await this.db.query(sql);
|
|
2169
2514
|
}
|
|
2170
2515
|
async add(job) {
|
|
@@ -2178,9 +2523,14 @@ class PostgresQueueStorage {
|
|
|
2178
2523
|
job.progress_details = null;
|
|
2179
2524
|
job.created_at = now;
|
|
2180
2525
|
job.run_after = now;
|
|
2526
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2527
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2528
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
2529
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(",") + "," : "";
|
|
2530
|
+
const baseParamStart = prefixColumnNames.length + 1;
|
|
2181
2531
|
const sql = `
|
|
2182
|
-
INSERT INTO
|
|
2183
|
-
queue,
|
|
2532
|
+
INSERT INTO ${this.tableName}(
|
|
2533
|
+
${prefixColumnsInsert}queue,
|
|
2184
2534
|
fingerprint,
|
|
2185
2535
|
input,
|
|
2186
2536
|
run_after,
|
|
@@ -2193,9 +2543,10 @@ class PostgresQueueStorage {
|
|
|
2193
2543
|
progress_details
|
|
2194
2544
|
)
|
|
2195
2545
|
VALUES
|
|
2196
|
-
($1
|
|
2546
|
+
(${prefixParamPlaceholders}$${baseParamStart},$${baseParamStart + 1},$${baseParamStart + 2},$${baseParamStart + 3},$${baseParamStart + 4},$${baseParamStart + 5},$${baseParamStart + 6},$${baseParamStart + 7},$${baseParamStart + 8},$${baseParamStart + 9},$${baseParamStart + 10})
|
|
2197
2547
|
RETURNING id`;
|
|
2198
2548
|
const params = [
|
|
2549
|
+
...prefixParamValues,
|
|
2199
2550
|
job.queue,
|
|
2200
2551
|
job.fingerprint,
|
|
2201
2552
|
JSON.stringify(job.input),
|
|
@@ -2215,68 +2566,75 @@ class PostgresQueueStorage {
|
|
|
2215
2566
|
return job.id;
|
|
2216
2567
|
}
|
|
2217
2568
|
async get(id) {
|
|
2569
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2218
2570
|
const result = await this.db.query(`SELECT *
|
|
2219
|
-
FROM
|
|
2220
|
-
WHERE id = $1 AND queue = $2
|
|
2571
|
+
FROM ${this.tableName}
|
|
2572
|
+
WHERE id = $1 AND queue = $2${prefixConditions}
|
|
2221
2573
|
FOR UPDATE SKIP LOCKED
|
|
2222
|
-
LIMIT 1`, [id, this.queueName]);
|
|
2574
|
+
LIMIT 1`, [id, this.queueName, ...prefixParams]);
|
|
2223
2575
|
if (!result || result.rows.length === 0)
|
|
2224
2576
|
return;
|
|
2225
2577
|
return result.rows[0];
|
|
2226
2578
|
}
|
|
2227
2579
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2228
2580
|
num = Number(num) || 100;
|
|
2581
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
2229
2582
|
const result = await this.db.query(`
|
|
2230
2583
|
SELECT *
|
|
2231
|
-
FROM
|
|
2584
|
+
FROM ${this.tableName}
|
|
2232
2585
|
WHERE queue = $1
|
|
2233
|
-
AND status = $2
|
|
2586
|
+
AND status = $2${prefixConditions}
|
|
2234
2587
|
ORDER BY run_after ASC
|
|
2235
2588
|
LIMIT $3
|
|
2236
|
-
FOR UPDATE SKIP LOCKED`, [this.queueName, status, num]);
|
|
2589
|
+
FOR UPDATE SKIP LOCKED`, [this.queueName, status, num, ...prefixParams]);
|
|
2237
2590
|
if (!result)
|
|
2238
2591
|
return [];
|
|
2239
2592
|
return result.rows;
|
|
2240
2593
|
}
|
|
2241
|
-
async next() {
|
|
2594
|
+
async next(workerId) {
|
|
2595
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
|
|
2242
2596
|
const result = await this.db.query(`
|
|
2243
|
-
UPDATE
|
|
2244
|
-
SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2597
|
+
UPDATE ${this.tableName}
|
|
2598
|
+
SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = $4
|
|
2245
2599
|
WHERE id = (
|
|
2246
2600
|
SELECT id
|
|
2247
|
-
FROM
|
|
2601
|
+
FROM ${this.tableName}
|
|
2248
2602
|
WHERE queue = $2
|
|
2249
|
-
AND status = $3
|
|
2603
|
+
AND status = $3${prefixConditions}
|
|
2250
2604
|
AND run_after <= NOW() AT TIME ZONE 'UTC'
|
|
2251
2605
|
ORDER BY run_after ASC
|
|
2252
2606
|
FOR UPDATE SKIP LOCKED
|
|
2253
2607
|
LIMIT 1
|
|
2254
2608
|
)
|
|
2255
|
-
RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING
|
|
2609
|
+
RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING */, workerId ?? null, ...prefixParams]);
|
|
2256
2610
|
return result?.rows?.[0] ?? undefined;
|
|
2257
2611
|
}
|
|
2258
2612
|
async size(status = "PENDING" /* PENDING */) {
|
|
2613
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2259
2614
|
const result = await this.db.query(`
|
|
2260
2615
|
SELECT COUNT(*) as count
|
|
2261
|
-
FROM
|
|
2616
|
+
FROM ${this.tableName}
|
|
2262
2617
|
WHERE queue = $1
|
|
2263
|
-
AND status = $2`, [this.queueName, status]);
|
|
2618
|
+
AND status = $2${prefixConditions}`, [this.queueName, status, ...prefixParams]);
|
|
2264
2619
|
if (!result)
|
|
2265
2620
|
return 0;
|
|
2266
2621
|
return parseInt(result.rows[0].count, 10);
|
|
2267
2622
|
}
|
|
2268
2623
|
async complete(jobDetails) {
|
|
2624
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2269
2625
|
if (jobDetails.status === "DISABLED" /* DISABLED */) {
|
|
2270
|
-
|
|
2626
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(4);
|
|
2627
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
2271
2628
|
SET
|
|
2272
2629
|
status = $1,
|
|
2273
2630
|
progress = 100,
|
|
2274
2631
|
progress_message = '',
|
|
2275
2632
|
progress_details = NULL,
|
|
2276
2633
|
completed_at = NOW() AT TIME ZONE 'UTC'
|
|
2277
|
-
WHERE id = $2 AND queue = $3`, [jobDetails.status, jobDetails.id, this.queueName]);
|
|
2634
|
+
WHERE id = $2 AND queue = $3${prefixConditions}`, [jobDetails.status, jobDetails.id, this.queueName, ...prefixParams]);
|
|
2278
2635
|
} else if (jobDetails.status === "PENDING" /* PENDING */) {
|
|
2279
|
-
|
|
2636
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
|
|
2637
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
2280
2638
|
SET
|
|
2281
2639
|
error = $1,
|
|
2282
2640
|
error_code = $2,
|
|
@@ -2287,17 +2645,19 @@ class PostgresQueueStorage {
|
|
|
2287
2645
|
progress_details = NULL,
|
|
2288
2646
|
run_attempts = run_attempts + 1,
|
|
2289
2647
|
last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2290
|
-
WHERE id = $5 AND queue = $6`, [
|
|
2648
|
+
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
2291
2649
|
jobDetails.error,
|
|
2292
2650
|
jobDetails.error_code,
|
|
2293
2651
|
jobDetails.status,
|
|
2294
2652
|
jobDetails.run_after,
|
|
2295
2653
|
jobDetails.id,
|
|
2296
|
-
this.queueName
|
|
2654
|
+
this.queueName,
|
|
2655
|
+
...prefixParams
|
|
2297
2656
|
]);
|
|
2298
2657
|
} else {
|
|
2658
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
|
|
2299
2659
|
await this.db.query(`
|
|
2300
|
-
UPDATE
|
|
2660
|
+
UPDATE ${this.tableName}
|
|
2301
2661
|
SET
|
|
2302
2662
|
output = $1,
|
|
2303
2663
|
error = $2,
|
|
@@ -2309,86 +2669,231 @@ class PostgresQueueStorage {
|
|
|
2309
2669
|
run_attempts = run_attempts + 1,
|
|
2310
2670
|
completed_at = NOW() AT TIME ZONE 'UTC',
|
|
2311
2671
|
last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2312
|
-
WHERE id = $5 AND queue = $6`, [
|
|
2672
|
+
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
2313
2673
|
jobDetails.output ? JSON.stringify(jobDetails.output) : null,
|
|
2314
2674
|
jobDetails.error ?? null,
|
|
2315
2675
|
jobDetails.error_code ?? null,
|
|
2316
2676
|
jobDetails.status,
|
|
2317
2677
|
jobDetails.id,
|
|
2318
|
-
this.queueName
|
|
2678
|
+
this.queueName,
|
|
2679
|
+
...prefixParams
|
|
2319
2680
|
]);
|
|
2320
2681
|
}
|
|
2321
2682
|
}
|
|
2322
2683
|
async deleteAll() {
|
|
2684
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
2323
2685
|
await this.db.query(`
|
|
2324
|
-
DELETE FROM
|
|
2325
|
-
WHERE queue = $1`, [this.queueName]);
|
|
2686
|
+
DELETE FROM ${this.tableName}
|
|
2687
|
+
WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
|
|
2326
2688
|
}
|
|
2327
2689
|
async outputForInput(input) {
|
|
2328
2690
|
const fingerprint = await makeFingerprint5(input);
|
|
2691
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2329
2692
|
const result = await this.db.query(`
|
|
2330
2693
|
SELECT output
|
|
2331
|
-
FROM
|
|
2332
|
-
WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'`, [fingerprint, this.queueName]);
|
|
2333
|
-
if (!result)
|
|
2694
|
+
FROM ${this.tableName}
|
|
2695
|
+
WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'${prefixConditions}`, [fingerprint, this.queueName, ...prefixParams]);
|
|
2696
|
+
if (!result || result.rows.length === 0)
|
|
2334
2697
|
return null;
|
|
2335
2698
|
return result.rows[0].output;
|
|
2336
2699
|
}
|
|
2337
2700
|
async abort(jobId) {
|
|
2338
|
-
const
|
|
2339
|
-
|
|
2701
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2702
|
+
await this.db.query(`
|
|
2703
|
+
UPDATE ${this.tableName}
|
|
2340
2704
|
SET status = 'ABORTING'
|
|
2341
|
-
WHERE id = $1 AND queue = $2`, [jobId, this.queueName]);
|
|
2705
|
+
WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
|
|
2342
2706
|
}
|
|
2343
2707
|
async getByRunId(job_run_id) {
|
|
2708
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2344
2709
|
const result = await this.db.query(`
|
|
2345
|
-
SELECT * FROM
|
|
2710
|
+
SELECT * FROM ${this.tableName} WHERE job_run_id = $1 AND queue = $2${prefixConditions}`, [job_run_id, this.queueName, ...prefixParams]);
|
|
2346
2711
|
if (!result)
|
|
2347
2712
|
return [];
|
|
2348
2713
|
return result.rows;
|
|
2349
2714
|
}
|
|
2350
2715
|
async saveProgress(jobId, progress, message, details) {
|
|
2716
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(6);
|
|
2351
2717
|
await this.db.query(`
|
|
2352
|
-
UPDATE
|
|
2718
|
+
UPDATE ${this.tableName}
|
|
2353
2719
|
SET progress = $1,
|
|
2354
2720
|
progress_message = $2,
|
|
2355
2721
|
progress_details = $3
|
|
2356
|
-
WHERE id = $4 AND queue = $5`, [
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2722
|
+
WHERE id = $4 AND queue = $5${prefixConditions}`, [
|
|
2723
|
+
progress,
|
|
2724
|
+
message,
|
|
2725
|
+
details ? JSON.stringify(details) : null,
|
|
2360
2726
|
jobId,
|
|
2361
|
-
this.queueName
|
|
2727
|
+
this.queueName,
|
|
2728
|
+
...prefixParams
|
|
2362
2729
|
]);
|
|
2363
2730
|
}
|
|
2731
|
+
async delete(jobId) {
|
|
2732
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2733
|
+
await this.db.query(`DELETE FROM ${this.tableName} WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
|
|
2734
|
+
}
|
|
2364
2735
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2365
2736
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
2366
|
-
|
|
2737
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
2738
|
+
await this.db.query(`DELETE FROM ${this.tableName}
|
|
2367
2739
|
WHERE queue = $1
|
|
2368
2740
|
AND status = $2
|
|
2369
2741
|
AND completed_at IS NOT NULL
|
|
2370
|
-
AND completed_at <= $3`, [this.queueName, status, cutoffDate]);
|
|
2742
|
+
AND completed_at <= $3${prefixConditions}`, [this.queueName, status, cutoffDate, ...prefixParams]);
|
|
2743
|
+
}
|
|
2744
|
+
async getAllJobs() {
|
|
2745
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
2746
|
+
const result = await this.db.query(`SELECT * FROM ${this.tableName} WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
|
|
2747
|
+
if (!result)
|
|
2748
|
+
return [];
|
|
2749
|
+
return result.rows;
|
|
2750
|
+
}
|
|
2751
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
2752
|
+
const filterEntries = Object.entries(prefixFilter);
|
|
2753
|
+
let query = `SELECT * FROM ${this.tableName} WHERE queue = $1`;
|
|
2754
|
+
const params = [this.queueName];
|
|
2755
|
+
filterEntries.forEach(([key, value], index) => {
|
|
2756
|
+
query += ` AND ${key} = $${index + 2}`;
|
|
2757
|
+
params.push(value);
|
|
2758
|
+
});
|
|
2759
|
+
const result = await this.db.query(query, params);
|
|
2760
|
+
if (!result)
|
|
2761
|
+
return [];
|
|
2762
|
+
return result.rows;
|
|
2763
|
+
}
|
|
2764
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
2765
|
+
if (prefixFilter === undefined) {
|
|
2766
|
+
return false;
|
|
2767
|
+
}
|
|
2768
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
2769
|
+
return true;
|
|
2770
|
+
}
|
|
2771
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
2772
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
2773
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
2774
|
+
return true;
|
|
2775
|
+
}
|
|
2776
|
+
for (const key of instanceKeys) {
|
|
2777
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
2778
|
+
return true;
|
|
2779
|
+
}
|
|
2780
|
+
}
|
|
2781
|
+
return false;
|
|
2782
|
+
}
|
|
2783
|
+
getPollingManager() {
|
|
2784
|
+
if (!this.pollingManager) {
|
|
2785
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
2786
|
+
const jobs = await this.getAllJobs();
|
|
2787
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
2788
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
2789
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
2790
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
2791
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
2792
|
+
});
|
|
2793
|
+
}
|
|
2794
|
+
return this.pollingManager;
|
|
2795
|
+
}
|
|
2796
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
2797
|
+
let lastKnownJobs = new Map;
|
|
2798
|
+
let cancelled = false;
|
|
2799
|
+
const poll = async () => {
|
|
2800
|
+
if (cancelled)
|
|
2801
|
+
return;
|
|
2802
|
+
try {
|
|
2803
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
2804
|
+
if (cancelled)
|
|
2805
|
+
return;
|
|
2806
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
2807
|
+
for (const [id, job] of currentMap) {
|
|
2808
|
+
const old = lastKnownJobs.get(id);
|
|
2809
|
+
if (!old) {
|
|
2810
|
+
callback({ type: "INSERT", new: job });
|
|
2811
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
2812
|
+
callback({ type: "UPDATE", old, new: job });
|
|
2813
|
+
}
|
|
2814
|
+
}
|
|
2815
|
+
for (const [id, job] of lastKnownJobs) {
|
|
2816
|
+
if (!currentMap.has(id)) {
|
|
2817
|
+
callback({ type: "DELETE", old: job });
|
|
2818
|
+
}
|
|
2819
|
+
}
|
|
2820
|
+
lastKnownJobs = currentMap;
|
|
2821
|
+
} catch {}
|
|
2822
|
+
};
|
|
2823
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
2824
|
+
poll();
|
|
2825
|
+
return () => {
|
|
2826
|
+
cancelled = true;
|
|
2827
|
+
clearInterval(intervalId);
|
|
2828
|
+
};
|
|
2829
|
+
}
|
|
2830
|
+
subscribeToChanges(callback, options) {
|
|
2831
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
2832
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
2833
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
2834
|
+
}
|
|
2835
|
+
const manager = this.getPollingManager();
|
|
2836
|
+
return manager.subscribe(callback, { intervalMs });
|
|
2371
2837
|
}
|
|
2372
2838
|
}
|
|
2373
2839
|
// src/queue/SqliteQueueStorage.ts
|
|
2374
|
-
import { createServiceToken as
|
|
2375
|
-
var SQLITE_QUEUE_STORAGE =
|
|
2840
|
+
import { createServiceToken as createServiceToken20, makeFingerprint as makeFingerprint6, sleep as sleep4, uuid4 as uuid43 } from "@workglow/util";
|
|
2841
|
+
var SQLITE_QUEUE_STORAGE = createServiceToken20("jobqueue.storage.sqlite");
|
|
2376
2842
|
|
|
2377
2843
|
class SqliteQueueStorage {
|
|
2378
2844
|
db;
|
|
2379
2845
|
queueName;
|
|
2380
2846
|
options;
|
|
2847
|
+
prefixes;
|
|
2848
|
+
prefixValues;
|
|
2849
|
+
tableName;
|
|
2850
|
+
pollingManager = null;
|
|
2381
2851
|
constructor(db, queueName, options) {
|
|
2382
2852
|
this.db = db;
|
|
2383
2853
|
this.queueName = queueName;
|
|
2384
2854
|
this.options = options;
|
|
2855
|
+
this.prefixes = options?.prefixes ?? [];
|
|
2856
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
2857
|
+
if (this.prefixes.length > 0) {
|
|
2858
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
2859
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
2860
|
+
} else {
|
|
2861
|
+
this.tableName = "job_queue";
|
|
2862
|
+
}
|
|
2863
|
+
}
|
|
2864
|
+
getPrefixColumnType(type) {
|
|
2865
|
+
return type === "uuid" ? "TEXT" : "INTEGER";
|
|
2866
|
+
}
|
|
2867
|
+
buildPrefixColumnsSql() {
|
|
2868
|
+
if (this.prefixes.length === 0)
|
|
2869
|
+
return "";
|
|
2870
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
2871
|
+
`) + `,
|
|
2872
|
+
`;
|
|
2873
|
+
}
|
|
2874
|
+
getPrefixColumnNames() {
|
|
2875
|
+
return this.prefixes.map((p) => p.name);
|
|
2876
|
+
}
|
|
2877
|
+
buildPrefixWhereClause() {
|
|
2878
|
+
if (this.prefixes.length === 0) {
|
|
2879
|
+
return "";
|
|
2880
|
+
}
|
|
2881
|
+
const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
|
|
2882
|
+
return " AND " + conditions;
|
|
2883
|
+
}
|
|
2884
|
+
getPrefixParamValues() {
|
|
2885
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2385
2886
|
}
|
|
2386
2887
|
async setupDatabase() {
|
|
2387
|
-
await
|
|
2888
|
+
await sleep4(0);
|
|
2889
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
2890
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2891
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2892
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
2388
2893
|
this.db.exec(`
|
|
2389
|
-
CREATE TABLE IF NOT EXISTS
|
|
2894
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
2390
2895
|
id INTEGER PRIMARY KEY,
|
|
2391
|
-
fingerprint text NOT NULL,
|
|
2896
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
2392
2897
|
queue text NOT NULL,
|
|
2393
2898
|
job_run_id text NOT NULL,
|
|
2394
2899
|
status TEXT NOT NULL default 'PENDING',
|
|
@@ -2405,12 +2910,13 @@ class SqliteQueueStorage {
|
|
|
2405
2910
|
error_code TEXT,
|
|
2406
2911
|
progress REAL DEFAULT 0,
|
|
2407
2912
|
progress_message TEXT DEFAULT '',
|
|
2408
|
-
progress_details TEXT NULL
|
|
2913
|
+
progress_details TEXT NULL,
|
|
2914
|
+
worker_id TEXT
|
|
2409
2915
|
);
|
|
2410
2916
|
|
|
2411
|
-
CREATE INDEX IF NOT EXISTS
|
|
2412
|
-
CREATE INDEX IF NOT EXISTS
|
|
2413
|
-
CREATE INDEX IF NOT EXISTS
|
|
2917
|
+
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after);
|
|
2918
|
+
CREATE INDEX IF NOT EXISTS job_queue_fingerprint${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status);
|
|
2919
|
+
CREATE INDEX IF NOT EXISTS job_queue_job_run_id${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, job_run_id);
|
|
2414
2920
|
`);
|
|
2415
2921
|
}
|
|
2416
2922
|
async add(job) {
|
|
@@ -2424,9 +2930,13 @@ class SqliteQueueStorage {
|
|
|
2424
2930
|
job.progress_details = null;
|
|
2425
2931
|
job.created_at = now;
|
|
2426
2932
|
job.run_after = now;
|
|
2933
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2934
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2935
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
2936
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
2427
2937
|
const AddQuery = `
|
|
2428
|
-
INSERT INTO
|
|
2429
|
-
queue,
|
|
2938
|
+
INSERT INTO ${this.tableName}(
|
|
2939
|
+
${prefixColumnsInsert}queue,
|
|
2430
2940
|
fingerprint,
|
|
2431
2941
|
input,
|
|
2432
2942
|
run_after,
|
|
@@ -2438,21 +2948,23 @@ class SqliteQueueStorage {
|
|
|
2438
2948
|
progress_details,
|
|
2439
2949
|
created_at
|
|
2440
2950
|
)
|
|
2441
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2951
|
+
VALUES (${prefixPlaceholders}?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2442
2952
|
RETURNING id`;
|
|
2443
2953
|
const stmt = this.db.prepare(AddQuery);
|
|
2444
|
-
const result = stmt.get(job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
|
|
2954
|
+
const result = stmt.get(...prefixParamValues, job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
|
|
2445
2955
|
job.id = result?.id;
|
|
2446
2956
|
return result?.id;
|
|
2447
2957
|
}
|
|
2448
2958
|
async get(id) {
|
|
2959
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
2960
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2449
2961
|
const JobQuery = `
|
|
2450
2962
|
SELECT *
|
|
2451
|
-
FROM
|
|
2452
|
-
WHERE id = ? AND queue =
|
|
2963
|
+
FROM ${this.tableName}
|
|
2964
|
+
WHERE id = ? AND queue = ?${prefixConditions}
|
|
2453
2965
|
LIMIT 1`;
|
|
2454
2966
|
const stmt = this.db.prepare(JobQuery);
|
|
2455
|
-
const result = stmt.get(id, this.queueName);
|
|
2967
|
+
const result = stmt.get(String(id), this.queueName, ...prefixParams);
|
|
2456
2968
|
if (!result)
|
|
2457
2969
|
return;
|
|
2458
2970
|
if (result.input)
|
|
@@ -2465,15 +2977,17 @@ class SqliteQueueStorage {
|
|
|
2465
2977
|
}
|
|
2466
2978
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2467
2979
|
num = Number(num) || 100;
|
|
2980
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
2981
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2468
2982
|
const FutureJobQuery = `
|
|
2469
2983
|
SELECT *
|
|
2470
|
-
FROM
|
|
2984
|
+
FROM ${this.tableName}
|
|
2471
2985
|
WHERE queue = ?
|
|
2472
|
-
AND status =
|
|
2986
|
+
AND status = ?${prefixConditions}
|
|
2473
2987
|
ORDER BY run_after ASC
|
|
2474
2988
|
LIMIT ${num}`;
|
|
2475
2989
|
const stmt = this.db.prepare(FutureJobQuery);
|
|
2476
|
-
const result = stmt.all(this.queueName, status);
|
|
2990
|
+
const result = stmt.all(this.queueName, status, ...prefixParams);
|
|
2477
2991
|
return (result || []).map((details) => {
|
|
2478
2992
|
if (details.input)
|
|
2479
2993
|
details.input = JSON.parse(details.input);
|
|
@@ -2485,20 +2999,24 @@ class SqliteQueueStorage {
|
|
|
2485
2999
|
});
|
|
2486
3000
|
}
|
|
2487
3001
|
async abort(jobId) {
|
|
3002
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3003
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2488
3004
|
const AbortQuery = `
|
|
2489
|
-
UPDATE
|
|
3005
|
+
UPDATE ${this.tableName}
|
|
2490
3006
|
SET status = ?
|
|
2491
|
-
WHERE id = ? AND queue =
|
|
3007
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2492
3008
|
const stmt = this.db.prepare(AbortQuery);
|
|
2493
|
-
stmt.run("ABORTING" /* ABORTING */, jobId, this.queueName);
|
|
3009
|
+
stmt.run("ABORTING" /* ABORTING */, String(jobId), this.queueName, ...prefixParams);
|
|
2494
3010
|
}
|
|
2495
3011
|
async getByRunId(job_run_id) {
|
|
3012
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3013
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2496
3014
|
const JobsByRunIdQuery = `
|
|
2497
3015
|
SELECT *
|
|
2498
|
-
FROM
|
|
2499
|
-
WHERE job_run_id = ? AND queue =
|
|
3016
|
+
FROM ${this.tableName}
|
|
3017
|
+
WHERE job_run_id = ? AND queue = ?${prefixConditions}`;
|
|
2500
3018
|
const stmt = this.db.prepare(JobsByRunIdQuery);
|
|
2501
|
-
const result = stmt.all(job_run_id, this.queueName);
|
|
3019
|
+
const result = stmt.all(job_run_id, this.queueName, ...prefixParams);
|
|
2502
3020
|
return (result || []).map((details) => {
|
|
2503
3021
|
if (details.input)
|
|
2504
3022
|
details.input = JSON.parse(details.input);
|
|
@@ -2509,22 +3027,24 @@ class SqliteQueueStorage {
|
|
|
2509
3027
|
return details;
|
|
2510
3028
|
});
|
|
2511
3029
|
}
|
|
2512
|
-
async next() {
|
|
3030
|
+
async next(workerId) {
|
|
2513
3031
|
const now = new Date().toISOString();
|
|
3032
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3033
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2514
3034
|
const stmt = this.db.prepare(`
|
|
2515
|
-
UPDATE
|
|
2516
|
-
SET status = ?, last_ran_at = ?
|
|
3035
|
+
UPDATE ${this.tableName}
|
|
3036
|
+
SET status = ?, last_ran_at = ?, worker_id = ?
|
|
2517
3037
|
WHERE id = (
|
|
2518
3038
|
SELECT id
|
|
2519
|
-
FROM
|
|
3039
|
+
FROM ${this.tableName}
|
|
2520
3040
|
WHERE queue = ?
|
|
2521
|
-
AND status =
|
|
3041
|
+
AND status = ?${prefixConditions}
|
|
2522
3042
|
AND run_after <= ?
|
|
2523
3043
|
ORDER BY run_after ASC
|
|
2524
3044
|
LIMIT 1
|
|
2525
3045
|
)
|
|
2526
3046
|
RETURNING *`);
|
|
2527
|
-
const result = stmt.get("PROCESSING" /* PROCESSING */, now, this.queueName, "PENDING" /* PENDING */, now);
|
|
3047
|
+
const result = stmt.get("PROCESSING" /* PROCESSING */, now, workerId ?? null, this.queueName, "PENDING" /* PENDING */, ...prefixParams, now);
|
|
2528
3048
|
if (!result)
|
|
2529
3049
|
return;
|
|
2530
3050
|
if (result.input)
|
|
@@ -2536,33 +3056,37 @@ class SqliteQueueStorage {
|
|
|
2536
3056
|
return result;
|
|
2537
3057
|
}
|
|
2538
3058
|
async size(status = "PENDING" /* PENDING */) {
|
|
3059
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3060
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2539
3061
|
const sizeQuery = `
|
|
2540
3062
|
SELECT COUNT(*) as count
|
|
2541
|
-
FROM
|
|
3063
|
+
FROM ${this.tableName}
|
|
2542
3064
|
WHERE queue = ?
|
|
2543
|
-
AND status =
|
|
3065
|
+
AND status = ?${prefixConditions}`;
|
|
2544
3066
|
const stmt = this.db.prepare(sizeQuery);
|
|
2545
|
-
const result = stmt.get(this.queueName, status);
|
|
3067
|
+
const result = stmt.get(this.queueName, status, ...prefixParams);
|
|
2546
3068
|
return result.count;
|
|
2547
3069
|
}
|
|
2548
3070
|
async complete(job) {
|
|
2549
3071
|
const now = new Date().toISOString();
|
|
3072
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3073
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2550
3074
|
let updateQuery;
|
|
2551
3075
|
let params;
|
|
2552
3076
|
if (job.status === "DISABLED" /* DISABLED */) {
|
|
2553
3077
|
updateQuery = `
|
|
2554
|
-
UPDATE
|
|
3078
|
+
UPDATE ${this.tableName}
|
|
2555
3079
|
SET
|
|
2556
3080
|
status = ?,
|
|
2557
3081
|
progress = 100,
|
|
2558
3082
|
progress_message = '',
|
|
2559
3083
|
progress_details = NULL,
|
|
2560
3084
|
completed_at = ?
|
|
2561
|
-
WHERE id = ? AND queue =
|
|
2562
|
-
params = [job.status, now, job.id, this.queueName];
|
|
3085
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
3086
|
+
params = [job.status, now, job.id, this.queueName, ...prefixParams];
|
|
2563
3087
|
} else {
|
|
2564
3088
|
updateQuery = `
|
|
2565
|
-
UPDATE
|
|
3089
|
+
UPDATE ${this.tableName}
|
|
2566
3090
|
SET
|
|
2567
3091
|
output = ?,
|
|
2568
3092
|
error = ?,
|
|
@@ -2574,7 +3098,7 @@ class SqliteQueueStorage {
|
|
|
2574
3098
|
last_ran_at = ?,
|
|
2575
3099
|
completed_at = ?,
|
|
2576
3100
|
run_attempts = run_attempts + 1
|
|
2577
|
-
WHERE id = ? AND queue =
|
|
3101
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2578
3102
|
params = [
|
|
2579
3103
|
job.output ? JSON.stringify(job.output) : null,
|
|
2580
3104
|
job.error ?? null,
|
|
@@ -2583,68 +3107,227 @@ class SqliteQueueStorage {
|
|
|
2583
3107
|
now,
|
|
2584
3108
|
now,
|
|
2585
3109
|
job.id,
|
|
2586
|
-
this.queueName
|
|
3110
|
+
this.queueName,
|
|
3111
|
+
...prefixParams
|
|
2587
3112
|
];
|
|
2588
3113
|
}
|
|
2589
3114
|
const stmt = this.db.prepare(updateQuery);
|
|
2590
3115
|
stmt.run(...params);
|
|
2591
3116
|
}
|
|
2592
3117
|
async deleteAll() {
|
|
3118
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3119
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2593
3120
|
const ClearQuery = `
|
|
2594
|
-
DELETE FROM
|
|
2595
|
-
WHERE queue =
|
|
3121
|
+
DELETE FROM ${this.tableName}
|
|
3122
|
+
WHERE queue = ?${prefixConditions}`;
|
|
2596
3123
|
const stmt = this.db.prepare(ClearQuery);
|
|
2597
|
-
stmt.run(this.queueName);
|
|
3124
|
+
stmt.run(this.queueName, ...prefixParams);
|
|
2598
3125
|
}
|
|
2599
3126
|
async outputForInput(input) {
|
|
2600
3127
|
const fingerprint = await makeFingerprint6(input);
|
|
3128
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3129
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2601
3130
|
const OutputQuery = `
|
|
2602
3131
|
SELECT output
|
|
2603
|
-
FROM
|
|
2604
|
-
WHERE queue = ? AND fingerprint = ? AND status =
|
|
3132
|
+
FROM ${this.tableName}
|
|
3133
|
+
WHERE queue = ? AND fingerprint = ? AND status = ?${prefixConditions}`;
|
|
2605
3134
|
const stmt = this.db.prepare(OutputQuery);
|
|
2606
|
-
const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED
|
|
3135
|
+
const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED */, ...prefixParams);
|
|
2607
3136
|
return result?.output ? JSON.parse(result.output) : null;
|
|
2608
3137
|
}
|
|
2609
3138
|
async saveProgress(jobId, progress, message, details) {
|
|
3139
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3140
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2610
3141
|
const UpdateProgressQuery = `
|
|
2611
|
-
UPDATE
|
|
3142
|
+
UPDATE ${this.tableName}
|
|
2612
3143
|
SET progress = ?,
|
|
2613
3144
|
progress_message = ?,
|
|
2614
3145
|
progress_details = ?
|
|
2615
|
-
WHERE id = ? AND queue =
|
|
3146
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2616
3147
|
const stmt = this.db.prepare(UpdateProgressQuery);
|
|
2617
|
-
stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName);
|
|
3148
|
+
stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName, ...prefixParams);
|
|
2618
3149
|
}
|
|
2619
3150
|
async delete(jobId) {
|
|
3151
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3152
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2620
3153
|
const DeleteQuery = `
|
|
2621
|
-
DELETE FROM
|
|
2622
|
-
WHERE id = ? AND queue =
|
|
3154
|
+
DELETE FROM ${this.tableName}
|
|
3155
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2623
3156
|
const stmt = this.db.prepare(DeleteQuery);
|
|
2624
|
-
stmt.run(String(jobId), this.queueName);
|
|
3157
|
+
stmt.run(String(jobId), this.queueName, ...prefixParams);
|
|
2625
3158
|
}
|
|
2626
3159
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2627
3160
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
3161
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3162
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2628
3163
|
const DeleteQuery = `
|
|
2629
|
-
DELETE FROM
|
|
3164
|
+
DELETE FROM ${this.tableName}
|
|
2630
3165
|
WHERE queue = ?
|
|
2631
3166
|
AND status = ?
|
|
2632
3167
|
AND completed_at IS NOT NULL
|
|
2633
|
-
AND completed_at <=
|
|
3168
|
+
AND completed_at <= ?${prefixConditions}`;
|
|
2634
3169
|
const stmt = this.db.prepare(DeleteQuery);
|
|
2635
|
-
stmt.run(this.queueName, status, cutoffDate);
|
|
3170
|
+
stmt.run(this.queueName, status, cutoffDate, ...prefixParams);
|
|
3171
|
+
}
|
|
3172
|
+
getAllJobs() {
|
|
3173
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3174
|
+
const prefixParams = this.getPrefixParamValues();
|
|
3175
|
+
const AllJobsQuery = `
|
|
3176
|
+
SELECT *
|
|
3177
|
+
FROM ${this.tableName}
|
|
3178
|
+
WHERE queue = ?${prefixConditions}`;
|
|
3179
|
+
const stmt = this.db.prepare(AllJobsQuery);
|
|
3180
|
+
const result = stmt.all(this.queueName, ...prefixParams);
|
|
3181
|
+
return (result || []).map((details) => {
|
|
3182
|
+
if (details.input)
|
|
3183
|
+
details.input = JSON.parse(details.input);
|
|
3184
|
+
if (details.output)
|
|
3185
|
+
details.output = JSON.parse(details.output);
|
|
3186
|
+
if (details.progress_details)
|
|
3187
|
+
details.progress_details = JSON.parse(details.progress_details);
|
|
3188
|
+
return details;
|
|
3189
|
+
});
|
|
3190
|
+
}
|
|
3191
|
+
getAllJobsWithFilter(prefixFilter) {
|
|
3192
|
+
const filterEntries = Object.entries(prefixFilter);
|
|
3193
|
+
let whereClause = "WHERE queue = ?";
|
|
3194
|
+
const params = [this.queueName];
|
|
3195
|
+
for (const [key, value] of filterEntries) {
|
|
3196
|
+
whereClause += ` AND ${key} = ?`;
|
|
3197
|
+
params.push(value);
|
|
3198
|
+
}
|
|
3199
|
+
const AllJobsQuery = `SELECT * FROM ${this.tableName} ${whereClause}`;
|
|
3200
|
+
const stmt = this.db.prepare(AllJobsQuery);
|
|
3201
|
+
const result = stmt.all(...params);
|
|
3202
|
+
return (result || []).map((details) => {
|
|
3203
|
+
if (details.input)
|
|
3204
|
+
details.input = JSON.parse(details.input);
|
|
3205
|
+
if (details.output)
|
|
3206
|
+
details.output = JSON.parse(details.output);
|
|
3207
|
+
if (details.progress_details)
|
|
3208
|
+
details.progress_details = JSON.parse(details.progress_details);
|
|
3209
|
+
return details;
|
|
3210
|
+
});
|
|
3211
|
+
}
|
|
3212
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
3213
|
+
if (prefixFilter === undefined) {
|
|
3214
|
+
return false;
|
|
3215
|
+
}
|
|
3216
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
3217
|
+
return true;
|
|
3218
|
+
}
|
|
3219
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
3220
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
3221
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
3222
|
+
return true;
|
|
3223
|
+
}
|
|
3224
|
+
for (const key of instanceKeys) {
|
|
3225
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
3226
|
+
return true;
|
|
3227
|
+
}
|
|
3228
|
+
}
|
|
3229
|
+
return false;
|
|
3230
|
+
}
|
|
3231
|
+
getPollingManager() {
|
|
3232
|
+
if (!this.pollingManager) {
|
|
3233
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
3234
|
+
const jobs = this.getAllJobs();
|
|
3235
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
3236
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
3237
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
3238
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
3239
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
3240
|
+
});
|
|
3241
|
+
}
|
|
3242
|
+
return this.pollingManager;
|
|
3243
|
+
}
|
|
3244
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
3245
|
+
let lastKnownJobs = new Map;
|
|
3246
|
+
const poll = () => {
|
|
3247
|
+
try {
|
|
3248
|
+
const currentJobs = this.getAllJobsWithFilter(prefixFilter);
|
|
3249
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
3250
|
+
for (const [id, job] of currentMap) {
|
|
3251
|
+
const old = lastKnownJobs.get(id);
|
|
3252
|
+
if (!old) {
|
|
3253
|
+
callback({ type: "INSERT", new: job });
|
|
3254
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
3255
|
+
callback({ type: "UPDATE", old, new: job });
|
|
3256
|
+
}
|
|
3257
|
+
}
|
|
3258
|
+
for (const [id, job] of lastKnownJobs) {
|
|
3259
|
+
if (!currentMap.has(id)) {
|
|
3260
|
+
callback({ type: "DELETE", old: job });
|
|
3261
|
+
}
|
|
3262
|
+
}
|
|
3263
|
+
lastKnownJobs = currentMap;
|
|
3264
|
+
} catch {}
|
|
3265
|
+
};
|
|
3266
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
3267
|
+
poll();
|
|
3268
|
+
return () => {
|
|
3269
|
+
clearInterval(intervalId);
|
|
3270
|
+
};
|
|
3271
|
+
}
|
|
3272
|
+
subscribeToChanges(callback, options) {
|
|
3273
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
3274
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
3275
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
3276
|
+
}
|
|
3277
|
+
const manager = this.getPollingManager();
|
|
3278
|
+
return manager.subscribe(callback, { intervalMs });
|
|
2636
3279
|
}
|
|
2637
3280
|
}
|
|
2638
3281
|
// src/queue/SupabaseQueueStorage.ts
|
|
2639
|
-
import { createServiceToken as
|
|
2640
|
-
var SUPABASE_QUEUE_STORAGE =
|
|
3282
|
+
import { createServiceToken as createServiceToken21, makeFingerprint as makeFingerprint7, uuid4 as uuid44 } from "@workglow/util";
|
|
3283
|
+
var SUPABASE_QUEUE_STORAGE = createServiceToken21("jobqueue.storage.supabase");
|
|
2641
3284
|
|
|
2642
3285
|
class SupabaseQueueStorage {
|
|
2643
3286
|
client;
|
|
2644
3287
|
queueName;
|
|
2645
|
-
|
|
3288
|
+
prefixes;
|
|
3289
|
+
prefixValues;
|
|
3290
|
+
tableName;
|
|
3291
|
+
realtimeChannel = null;
|
|
3292
|
+
pollingManager = null;
|
|
3293
|
+
constructor(client, queueName, options) {
|
|
2646
3294
|
this.client = client;
|
|
2647
3295
|
this.queueName = queueName;
|
|
3296
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3297
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3298
|
+
if (this.prefixes.length > 0) {
|
|
3299
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3300
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
3301
|
+
} else {
|
|
3302
|
+
this.tableName = "job_queue";
|
|
3303
|
+
}
|
|
3304
|
+
}
|
|
3305
|
+
getPrefixColumnType(type) {
|
|
3306
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
3307
|
+
}
|
|
3308
|
+
buildPrefixColumnsSql() {
|
|
3309
|
+
if (this.prefixes.length === 0)
|
|
3310
|
+
return "";
|
|
3311
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
3312
|
+
`) + `,
|
|
3313
|
+
`;
|
|
3314
|
+
}
|
|
3315
|
+
getPrefixColumnNames() {
|
|
3316
|
+
return this.prefixes.map((p) => p.name);
|
|
3317
|
+
}
|
|
3318
|
+
applyPrefixFilters(query) {
|
|
3319
|
+
let result = query;
|
|
3320
|
+
for (const prefix of this.prefixes) {
|
|
3321
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
3322
|
+
}
|
|
3323
|
+
return result;
|
|
3324
|
+
}
|
|
3325
|
+
getPrefixInsertValues() {
|
|
3326
|
+
const values = {};
|
|
3327
|
+
for (const prefix of this.prefixes) {
|
|
3328
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
3329
|
+
}
|
|
3330
|
+
return values;
|
|
2648
3331
|
}
|
|
2649
3332
|
async setupDatabase() {
|
|
2650
3333
|
const createTypeSql = `CREATE TYPE job_status AS ENUM (${Object.values(JobStatus).map((v) => `'${v}'`).join(",")})`;
|
|
@@ -2652,10 +3335,14 @@ class SupabaseQueueStorage {
|
|
|
2652
3335
|
if (typeError && typeError.code !== "42710") {
|
|
2653
3336
|
throw typeError;
|
|
2654
3337
|
}
|
|
3338
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
3339
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3340
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3341
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
2655
3342
|
const createTableSql = `
|
|
2656
|
-
CREATE TABLE IF NOT EXISTS
|
|
3343
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
2657
3344
|
id SERIAL NOT NULL,
|
|
2658
|
-
fingerprint text NOT NULL,
|
|
3345
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
2659
3346
|
queue text NOT NULL,
|
|
2660
3347
|
job_run_id text NOT NULL,
|
|
2661
3348
|
status job_status NOT NULL default 'PENDING',
|
|
@@ -2672,7 +3359,8 @@ class SupabaseQueueStorage {
|
|
|
2672
3359
|
error_code text,
|
|
2673
3360
|
progress real DEFAULT 0,
|
|
2674
3361
|
progress_message text DEFAULT '',
|
|
2675
|
-
progress_details jsonb
|
|
3362
|
+
progress_details jsonb,
|
|
3363
|
+
worker_id text
|
|
2676
3364
|
)`;
|
|
2677
3365
|
const { error: tableError } = await this.client.rpc("exec_sql", { query: createTableSql });
|
|
2678
3366
|
if (tableError) {
|
|
@@ -2681,12 +3369,12 @@ class SupabaseQueueStorage {
|
|
|
2681
3369
|
}
|
|
2682
3370
|
}
|
|
2683
3371
|
const indexes = [
|
|
2684
|
-
`CREATE INDEX IF NOT EXISTS
|
|
2685
|
-
`CREATE INDEX IF NOT EXISTS
|
|
2686
|
-
`CREATE INDEX IF NOT EXISTS
|
|
3372
|
+
`CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`,
|
|
3373
|
+
`CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`,
|
|
3374
|
+
`CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`
|
|
2687
3375
|
];
|
|
2688
3376
|
for (const indexSql of indexes) {
|
|
2689
|
-
|
|
3377
|
+
await this.client.rpc("exec_sql", { query: indexSql });
|
|
2690
3378
|
}
|
|
2691
3379
|
}
|
|
2692
3380
|
async add(job) {
|
|
@@ -2700,7 +3388,9 @@ class SupabaseQueueStorage {
|
|
|
2700
3388
|
job.progress_details = null;
|
|
2701
3389
|
job.created_at = now;
|
|
2702
3390
|
job.run_after = now;
|
|
2703
|
-
const
|
|
3391
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
3392
|
+
const { data, error } = await this.client.from(this.tableName).insert({
|
|
3393
|
+
...prefixInsertValues,
|
|
2704
3394
|
queue: job.queue,
|
|
2705
3395
|
fingerprint: job.fingerprint,
|
|
2706
3396
|
input: job.input,
|
|
@@ -2721,7 +3411,9 @@ class SupabaseQueueStorage {
|
|
|
2721
3411
|
return job.id;
|
|
2722
3412
|
}
|
|
2723
3413
|
async get(id) {
|
|
2724
|
-
|
|
3414
|
+
let query = this.client.from(this.tableName).select("*").eq("id", id).eq("queue", this.queueName);
|
|
3415
|
+
query = this.applyPrefixFilters(query);
|
|
3416
|
+
const { data, error } = await query.single();
|
|
2725
3417
|
if (error) {
|
|
2726
3418
|
if (error.code === "PGRST116")
|
|
2727
3419
|
return;
|
|
@@ -2731,36 +3423,53 @@ class SupabaseQueueStorage {
|
|
|
2731
3423
|
}
|
|
2732
3424
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2733
3425
|
num = Number(num) || 100;
|
|
2734
|
-
|
|
3426
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", status);
|
|
3427
|
+
query = this.applyPrefixFilters(query);
|
|
3428
|
+
const { data, error } = await query.order("run_after", { ascending: true }).limit(num);
|
|
2735
3429
|
if (error)
|
|
2736
3430
|
throw error;
|
|
2737
3431
|
return data ?? [];
|
|
2738
3432
|
}
|
|
2739
|
-
async next() {
|
|
2740
|
-
|
|
3433
|
+
async next(workerId) {
|
|
3434
|
+
let selectQuery = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", "PENDING" /* PENDING */).lte("run_after", new Date().toISOString());
|
|
3435
|
+
selectQuery = this.applyPrefixFilters(selectQuery);
|
|
3436
|
+
const { data: jobs, error: selectError } = await selectQuery.order("run_after", { ascending: true }).limit(1);
|
|
2741
3437
|
if (selectError)
|
|
2742
3438
|
throw selectError;
|
|
2743
3439
|
if (!jobs || jobs.length === 0)
|
|
2744
3440
|
return;
|
|
2745
3441
|
const job = jobs[0];
|
|
2746
|
-
|
|
3442
|
+
let updateQuery = this.client.from(this.tableName).update({
|
|
2747
3443
|
status: "PROCESSING" /* PROCESSING */,
|
|
2748
|
-
last_ran_at: new Date().toISOString()
|
|
2749
|
-
|
|
3444
|
+
last_ran_at: new Date().toISOString(),
|
|
3445
|
+
worker_id: workerId ?? null
|
|
3446
|
+
}).eq("id", job.id).eq("queue", this.queueName);
|
|
3447
|
+
updateQuery = this.applyPrefixFilters(updateQuery);
|
|
3448
|
+
const { data: updatedJob, error: updateError } = await updateQuery.select().single();
|
|
2750
3449
|
if (updateError)
|
|
2751
3450
|
throw updateError;
|
|
2752
3451
|
return updatedJob;
|
|
2753
3452
|
}
|
|
2754
3453
|
async size(status = "PENDING" /* PENDING */) {
|
|
2755
|
-
|
|
3454
|
+
let query = this.client.from(this.tableName).select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
|
|
3455
|
+
query = this.applyPrefixFilters(query);
|
|
3456
|
+
const { count, error } = await query;
|
|
2756
3457
|
if (error)
|
|
2757
3458
|
throw error;
|
|
2758
3459
|
return count ?? 0;
|
|
2759
3460
|
}
|
|
3461
|
+
async getAllJobs() {
|
|
3462
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
3463
|
+
query = this.applyPrefixFilters(query);
|
|
3464
|
+
const { data, error } = await query;
|
|
3465
|
+
if (error)
|
|
3466
|
+
throw error;
|
|
3467
|
+
return data ?? [];
|
|
3468
|
+
}
|
|
2760
3469
|
async complete(jobDetails) {
|
|
2761
3470
|
const now = new Date().toISOString();
|
|
2762
3471
|
if (jobDetails.status === "DISABLED" /* DISABLED */) {
|
|
2763
|
-
|
|
3472
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2764
3473
|
status: jobDetails.status,
|
|
2765
3474
|
progress: 100,
|
|
2766
3475
|
progress_message: "",
|
|
@@ -2768,16 +3477,20 @@ class SupabaseQueueStorage {
|
|
|
2768
3477
|
completed_at: now,
|
|
2769
3478
|
last_ran_at: now
|
|
2770
3479
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3480
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3481
|
+
const { error: error2 } = await query2;
|
|
2771
3482
|
if (error2)
|
|
2772
3483
|
throw error2;
|
|
2773
3484
|
return;
|
|
2774
3485
|
}
|
|
2775
|
-
|
|
3486
|
+
let getQuery = this.client.from(this.tableName).select("run_attempts").eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3487
|
+
getQuery = this.applyPrefixFilters(getQuery);
|
|
3488
|
+
const { data: current, error: getError } = await getQuery.single();
|
|
2776
3489
|
if (getError)
|
|
2777
3490
|
throw getError;
|
|
2778
3491
|
const nextAttempts = (current?.run_attempts ?? 0) + 1;
|
|
2779
3492
|
if (jobDetails.status === "PENDING" /* PENDING */) {
|
|
2780
|
-
|
|
3493
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2781
3494
|
error: jobDetails.error ?? null,
|
|
2782
3495
|
error_code: jobDetails.error_code ?? null,
|
|
2783
3496
|
status: jobDetails.status,
|
|
@@ -2788,12 +3501,14 @@ class SupabaseQueueStorage {
|
|
|
2788
3501
|
run_attempts: nextAttempts,
|
|
2789
3502
|
last_ran_at: now
|
|
2790
3503
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3504
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3505
|
+
const { error: error2 } = await query2;
|
|
2791
3506
|
if (error2)
|
|
2792
3507
|
throw error2;
|
|
2793
3508
|
return;
|
|
2794
3509
|
}
|
|
2795
3510
|
if (jobDetails.status === "COMPLETED" /* COMPLETED */ || jobDetails.status === "FAILED" /* FAILED */) {
|
|
2796
|
-
|
|
3511
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2797
3512
|
output: jobDetails.output ?? null,
|
|
2798
3513
|
error: jobDetails.error ?? null,
|
|
2799
3514
|
error_code: jobDetails.error_code ?? null,
|
|
@@ -2805,11 +3520,13 @@ class SupabaseQueueStorage {
|
|
|
2805
3520
|
completed_at: now,
|
|
2806
3521
|
last_ran_at: now
|
|
2807
3522
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3523
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3524
|
+
const { error: error2 } = await query2;
|
|
2808
3525
|
if (error2)
|
|
2809
3526
|
throw error2;
|
|
2810
3527
|
return;
|
|
2811
3528
|
}
|
|
2812
|
-
|
|
3529
|
+
let query = this.client.from(this.tableName).update({
|
|
2813
3530
|
status: jobDetails.status,
|
|
2814
3531
|
output: jobDetails.output ?? null,
|
|
2815
3532
|
error: jobDetails.error ?? null,
|
|
@@ -2818,17 +3535,23 @@ class SupabaseQueueStorage {
|
|
|
2818
3535
|
run_attempts: nextAttempts,
|
|
2819
3536
|
last_ran_at: now
|
|
2820
3537
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3538
|
+
query = this.applyPrefixFilters(query);
|
|
3539
|
+
const { error } = await query;
|
|
2821
3540
|
if (error)
|
|
2822
3541
|
throw error;
|
|
2823
3542
|
}
|
|
2824
3543
|
async deleteAll() {
|
|
2825
|
-
|
|
3544
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
|
|
3545
|
+
query = this.applyPrefixFilters(query);
|
|
3546
|
+
const { error } = await query;
|
|
2826
3547
|
if (error)
|
|
2827
3548
|
throw error;
|
|
2828
3549
|
}
|
|
2829
3550
|
async outputForInput(input) {
|
|
2830
3551
|
const fingerprint = await makeFingerprint7(input);
|
|
2831
|
-
|
|
3552
|
+
let query = this.client.from(this.tableName).select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", "COMPLETED" /* COMPLETED */);
|
|
3553
|
+
query = this.applyPrefixFilters(query);
|
|
3554
|
+
const { data, error } = await query.single();
|
|
2832
3555
|
if (error) {
|
|
2833
3556
|
if (error.code === "PGRST116")
|
|
2834
3557
|
return null;
|
|
@@ -2837,47 +3560,622 @@ class SupabaseQueueStorage {
|
|
|
2837
3560
|
return data?.output ?? null;
|
|
2838
3561
|
}
|
|
2839
3562
|
async abort(jobId) {
|
|
2840
|
-
|
|
3563
|
+
let query = this.client.from(this.tableName).update({ status: "ABORTING" /* ABORTING */ }).eq("id", jobId).eq("queue", this.queueName);
|
|
3564
|
+
query = this.applyPrefixFilters(query);
|
|
3565
|
+
const { error } = await query;
|
|
2841
3566
|
if (error)
|
|
2842
3567
|
throw error;
|
|
2843
3568
|
}
|
|
2844
3569
|
async getByRunId(job_run_id) {
|
|
2845
|
-
|
|
3570
|
+
let query = this.client.from(this.tableName).select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
|
|
3571
|
+
query = this.applyPrefixFilters(query);
|
|
3572
|
+
const { data, error } = await query;
|
|
2846
3573
|
if (error)
|
|
2847
3574
|
throw error;
|
|
2848
3575
|
return data ?? [];
|
|
2849
3576
|
}
|
|
2850
3577
|
async saveProgress(jobId, progress, message, details) {
|
|
2851
|
-
|
|
3578
|
+
let query = this.client.from(this.tableName).update({
|
|
2852
3579
|
progress,
|
|
2853
3580
|
progress_message: message,
|
|
2854
3581
|
progress_details: details
|
|
2855
3582
|
}).eq("id", jobId).eq("queue", this.queueName);
|
|
3583
|
+
query = this.applyPrefixFilters(query);
|
|
3584
|
+
const { error } = await query;
|
|
2856
3585
|
if (error)
|
|
2857
3586
|
throw error;
|
|
2858
3587
|
}
|
|
2859
3588
|
async delete(jobId) {
|
|
2860
|
-
|
|
3589
|
+
let query = this.client.from(this.tableName).delete().eq("id", jobId).eq("queue", this.queueName);
|
|
3590
|
+
query = this.applyPrefixFilters(query);
|
|
3591
|
+
const { error } = await query;
|
|
2861
3592
|
if (error)
|
|
2862
3593
|
throw error;
|
|
2863
3594
|
}
|
|
2864
3595
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2865
3596
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
2866
|
-
|
|
3597
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
|
|
3598
|
+
query = this.applyPrefixFilters(query);
|
|
3599
|
+
const { error } = await query;
|
|
2867
3600
|
if (error)
|
|
2868
3601
|
throw error;
|
|
2869
3602
|
}
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2879
|
-
|
|
2880
|
-
|
|
3603
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
3604
|
+
if (!job)
|
|
3605
|
+
return false;
|
|
3606
|
+
if (job.queue !== this.queueName) {
|
|
3607
|
+
return false;
|
|
3608
|
+
}
|
|
3609
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
3610
|
+
return true;
|
|
3611
|
+
}
|
|
3612
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
3613
|
+
if (Object.keys(filterValues).length === 0) {
|
|
3614
|
+
return true;
|
|
3615
|
+
}
|
|
3616
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
3617
|
+
if (job[key] !== value) {
|
|
3618
|
+
return false;
|
|
3619
|
+
}
|
|
3620
|
+
}
|
|
3621
|
+
return true;
|
|
3622
|
+
}
|
|
3623
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
3624
|
+
if (prefixFilter === undefined) {
|
|
3625
|
+
return false;
|
|
3626
|
+
}
|
|
3627
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
3628
|
+
return true;
|
|
3629
|
+
}
|
|
3630
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
3631
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
3632
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
3633
|
+
return true;
|
|
3634
|
+
}
|
|
3635
|
+
for (const key of instanceKeys) {
|
|
3636
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
3637
|
+
return true;
|
|
3638
|
+
}
|
|
3639
|
+
}
|
|
3640
|
+
return false;
|
|
3641
|
+
}
|
|
3642
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
3643
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
3644
|
+
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
3645
|
+
query = query.eq(key, value);
|
|
3646
|
+
}
|
|
3647
|
+
const { data, error } = await query;
|
|
3648
|
+
if (error)
|
|
3649
|
+
throw error;
|
|
3650
|
+
return data ?? [];
|
|
3651
|
+
}
|
|
3652
|
+
subscribeToChanges(callback, options) {
|
|
3653
|
+
return this.subscribeToChangesWithRealtime(callback, options?.prefixFilter);
|
|
3654
|
+
}
|
|
3655
|
+
subscribeToChangesWithRealtime(callback, prefixFilter) {
|
|
3656
|
+
const channelName = `queue-${this.tableName}-${this.queueName}-${Date.now()}`;
|
|
3657
|
+
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
3658
|
+
event: "*",
|
|
3659
|
+
schema: "public",
|
|
3660
|
+
table: this.tableName,
|
|
3661
|
+
filter: `queue=eq.${this.queueName}`
|
|
3662
|
+
}, (payload) => {
|
|
3663
|
+
const newJob = payload.new;
|
|
3664
|
+
const oldJob = payload.old;
|
|
3665
|
+
const newMatches = this.matchesPrefixFilter(newJob, prefixFilter);
|
|
3666
|
+
const oldMatches = this.matchesPrefixFilter(oldJob, prefixFilter);
|
|
3667
|
+
if (!newMatches && !oldMatches) {
|
|
3668
|
+
return;
|
|
3669
|
+
}
|
|
3670
|
+
callback({
|
|
3671
|
+
type: payload.eventType.toUpperCase(),
|
|
3672
|
+
old: oldJob && Object.keys(oldJob).length > 0 ? oldJob : undefined,
|
|
3673
|
+
new: newJob && Object.keys(newJob).length > 0 ? newJob : undefined
|
|
3674
|
+
});
|
|
3675
|
+
}).subscribe();
|
|
3676
|
+
return () => {
|
|
3677
|
+
if (this.realtimeChannel) {
|
|
3678
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
3679
|
+
this.realtimeChannel = null;
|
|
3680
|
+
}
|
|
3681
|
+
};
|
|
3682
|
+
}
|
|
3683
|
+
getPollingManager() {
|
|
3684
|
+
if (!this.pollingManager) {
|
|
3685
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
3686
|
+
const jobs = await this.getAllJobs();
|
|
3687
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
3688
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
3689
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
3690
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
3691
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
3692
|
+
});
|
|
3693
|
+
}
|
|
3694
|
+
return this.pollingManager;
|
|
3695
|
+
}
|
|
3696
|
+
subscribeWithCustomPrefixFilterPolling(callback, prefixFilter, intervalMs) {
|
|
3697
|
+
let lastKnownJobs = new Map;
|
|
3698
|
+
let cancelled = false;
|
|
3699
|
+
const poll = async () => {
|
|
3700
|
+
if (cancelled)
|
|
3701
|
+
return;
|
|
3702
|
+
try {
|
|
3703
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
3704
|
+
if (cancelled)
|
|
3705
|
+
return;
|
|
3706
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
3707
|
+
for (const [id, job] of currentMap) {
|
|
3708
|
+
const old = lastKnownJobs.get(id);
|
|
3709
|
+
if (!old) {
|
|
3710
|
+
callback({ type: "INSERT", new: job });
|
|
3711
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
3712
|
+
callback({ type: "UPDATE", old, new: job });
|
|
3713
|
+
}
|
|
3714
|
+
}
|
|
3715
|
+
for (const [id, job] of lastKnownJobs) {
|
|
3716
|
+
if (!currentMap.has(id)) {
|
|
3717
|
+
callback({ type: "DELETE", old: job });
|
|
3718
|
+
}
|
|
3719
|
+
}
|
|
3720
|
+
lastKnownJobs = currentMap;
|
|
3721
|
+
} catch {}
|
|
3722
|
+
};
|
|
3723
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
3724
|
+
poll();
|
|
3725
|
+
return () => {
|
|
3726
|
+
cancelled = true;
|
|
3727
|
+
clearInterval(intervalId);
|
|
3728
|
+
};
|
|
3729
|
+
}
|
|
3730
|
+
subscribeToChangesWithPolling(callback, options) {
|
|
3731
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
3732
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
3733
|
+
return this.subscribeWithCustomPrefixFilterPolling(callback, options.prefixFilter, intervalMs);
|
|
3734
|
+
}
|
|
3735
|
+
const manager = this.getPollingManager();
|
|
3736
|
+
return manager.subscribe(callback, { intervalMs });
|
|
3737
|
+
}
|
|
3738
|
+
}
|
|
3739
|
+
// src/limiter/PostgresRateLimiterStorage.ts
|
|
3740
|
+
import { createServiceToken as createServiceToken22 } from "@workglow/util";
|
|
3741
|
+
var POSTGRES_RATE_LIMITER_STORAGE = createServiceToken22("ratelimiter.storage.postgres");
|
|
3742
|
+
|
|
3743
|
+
class PostgresRateLimiterStorage {
|
|
3744
|
+
db;
|
|
3745
|
+
prefixes;
|
|
3746
|
+
prefixValues;
|
|
3747
|
+
executionTableName;
|
|
3748
|
+
nextAvailableTableName;
|
|
3749
|
+
constructor(db, options) {
|
|
3750
|
+
this.db = db;
|
|
3751
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3752
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3753
|
+
if (this.prefixes.length > 0) {
|
|
3754
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3755
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
3756
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
3757
|
+
} else {
|
|
3758
|
+
this.executionTableName = "rate_limit_executions";
|
|
3759
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
3760
|
+
}
|
|
3761
|
+
}
|
|
3762
|
+
getPrefixColumnType(type) {
|
|
3763
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
3764
|
+
}
|
|
3765
|
+
buildPrefixColumnsSql() {
|
|
3766
|
+
if (this.prefixes.length === 0)
|
|
3767
|
+
return "";
|
|
3768
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
3769
|
+
`) + `,
|
|
3770
|
+
`;
|
|
3771
|
+
}
|
|
3772
|
+
getPrefixColumnNames() {
|
|
3773
|
+
return this.prefixes.map((p) => p.name);
|
|
3774
|
+
}
|
|
3775
|
+
buildPrefixWhereClause(startParam) {
|
|
3776
|
+
if (this.prefixes.length === 0) {
|
|
3777
|
+
return { conditions: "", params: [] };
|
|
3778
|
+
}
|
|
3779
|
+
const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
|
|
3780
|
+
const params = this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3781
|
+
return { conditions: " AND " + conditions, params };
|
|
3782
|
+
}
|
|
3783
|
+
getPrefixParamValues() {
|
|
3784
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3785
|
+
}
|
|
3786
|
+
async setupDatabase() {
|
|
3787
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
3788
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3789
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3790
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
3791
|
+
await this.db.query(`
|
|
3792
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
3793
|
+
id SERIAL PRIMARY KEY,
|
|
3794
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3795
|
+
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
3796
|
+
)
|
|
3797
|
+
`);
|
|
3798
|
+
await this.db.query(`
|
|
3799
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
3800
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
3801
|
+
`);
|
|
3802
|
+
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
3803
|
+
await this.db.query(`
|
|
3804
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
3805
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3806
|
+
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
3807
|
+
PRIMARY KEY (${primaryKeyColumns})
|
|
3808
|
+
)
|
|
3809
|
+
`);
|
|
3810
|
+
}
|
|
3811
|
+
async recordExecution(queueName) {
|
|
3812
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3813
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3814
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
3815
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
|
|
3816
|
+
const queueParamNum = prefixColumnNames.length + 1;
|
|
3817
|
+
await this.db.query(`
|
|
3818
|
+
INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
|
|
3819
|
+
VALUES (${prefixParamPlaceholders}$${queueParamNum})
|
|
3820
|
+
`, [...prefixParamValues, queueName]);
|
|
3821
|
+
}
|
|
3822
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
3823
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
3824
|
+
const result = await this.db.query(`
|
|
3825
|
+
SELECT COUNT(*) AS count
|
|
3826
|
+
FROM ${this.executionTableName}
|
|
3827
|
+
WHERE queue_name = $1 AND executed_at > $2${prefixConditions}
|
|
3828
|
+
`, [queueName, windowStartTime, ...prefixParams]);
|
|
3829
|
+
return parseInt(result.rows[0]?.count ?? "0", 10);
|
|
3830
|
+
}
|
|
3831
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
3832
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
3833
|
+
const result = await this.db.query(`
|
|
3834
|
+
SELECT executed_at
|
|
3835
|
+
FROM ${this.executionTableName}
|
|
3836
|
+
WHERE queue_name = $1${prefixConditions}
|
|
3837
|
+
ORDER BY executed_at ASC
|
|
3838
|
+
LIMIT 1 OFFSET $2
|
|
3839
|
+
`, [queueName, offset, ...prefixParams]);
|
|
3840
|
+
const executedAt = result.rows[0]?.executed_at;
|
|
3841
|
+
if (!executedAt)
|
|
3842
|
+
return;
|
|
3843
|
+
return new Date(executedAt).toISOString();
|
|
3844
|
+
}
|
|
3845
|
+
async getNextAvailableTime(queueName) {
|
|
3846
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
3847
|
+
const result = await this.db.query(`
|
|
3848
|
+
SELECT next_available_at
|
|
3849
|
+
FROM ${this.nextAvailableTableName}
|
|
3850
|
+
WHERE queue_name = $1${prefixConditions}
|
|
3851
|
+
`, [queueName, ...prefixParams]);
|
|
3852
|
+
const nextAvailableAt = result.rows[0]?.next_available_at;
|
|
3853
|
+
if (!nextAvailableAt)
|
|
3854
|
+
return;
|
|
3855
|
+
return new Date(nextAvailableAt).toISOString();
|
|
3856
|
+
}
|
|
3857
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
3858
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3859
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3860
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
3861
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
|
|
3862
|
+
const baseParamStart = prefixColumnNames.length + 1;
|
|
3863
|
+
const conflictColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
3864
|
+
await this.db.query(`
|
|
3865
|
+
INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
|
|
3866
|
+
VALUES (${prefixParamPlaceholders}$${baseParamStart}, $${baseParamStart + 1})
|
|
3867
|
+
ON CONFLICT (${conflictColumns})
|
|
3868
|
+
DO UPDATE SET next_available_at = EXCLUDED.next_available_at
|
|
3869
|
+
`, [...prefixParamValues, queueName, nextAvailableAt]);
|
|
3870
|
+
}
|
|
3871
|
+
async clear(queueName) {
|
|
3872
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
3873
|
+
await this.db.query(`DELETE FROM ${this.executionTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
|
|
3874
|
+
await this.db.query(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
|
|
3875
|
+
}
|
|
3876
|
+
}
|
|
3877
|
+
// src/limiter/SqliteRateLimiterStorage.ts
|
|
3878
|
+
import { createServiceToken as createServiceToken23, sleep as sleep5, toSQLiteTimestamp } from "@workglow/util";
|
|
3879
|
+
var SQLITE_RATE_LIMITER_STORAGE = createServiceToken23("ratelimiter.storage.sqlite");
|
|
3880
|
+
|
|
3881
|
+
class SqliteRateLimiterStorage {
|
|
3882
|
+
db;
|
|
3883
|
+
prefixes;
|
|
3884
|
+
prefixValues;
|
|
3885
|
+
executionTableName;
|
|
3886
|
+
nextAvailableTableName;
|
|
3887
|
+
constructor(db, options) {
|
|
3888
|
+
this.db = db;
|
|
3889
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3890
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3891
|
+
if (this.prefixes.length > 0) {
|
|
3892
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3893
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
3894
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
3895
|
+
} else {
|
|
3896
|
+
this.executionTableName = "rate_limit_executions";
|
|
3897
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
3898
|
+
}
|
|
3899
|
+
}
|
|
3900
|
+
getPrefixColumnType(type) {
|
|
3901
|
+
return type === "uuid" ? "TEXT" : "INTEGER";
|
|
3902
|
+
}
|
|
3903
|
+
buildPrefixColumnsSql() {
|
|
3904
|
+
if (this.prefixes.length === 0)
|
|
3905
|
+
return "";
|
|
3906
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
3907
|
+
`) + `,
|
|
3908
|
+
`;
|
|
3909
|
+
}
|
|
3910
|
+
getPrefixColumnNames() {
|
|
3911
|
+
return this.prefixes.map((p) => p.name);
|
|
3912
|
+
}
|
|
3913
|
+
buildPrefixWhereClause() {
|
|
3914
|
+
if (this.prefixes.length === 0) {
|
|
3915
|
+
return "";
|
|
3916
|
+
}
|
|
3917
|
+
const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
|
|
3918
|
+
return " AND " + conditions;
|
|
3919
|
+
}
|
|
3920
|
+
getPrefixParamValues() {
|
|
3921
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3922
|
+
}
|
|
3923
|
+
async setupDatabase() {
|
|
3924
|
+
await sleep5(0);
|
|
3925
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
3926
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3927
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3928
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
3929
|
+
this.db.exec(`
|
|
3930
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
3931
|
+
id INTEGER PRIMARY KEY,
|
|
3932
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3933
|
+
executed_at TEXT DEFAULT CURRENT_TIMESTAMP
|
|
3934
|
+
);
|
|
3935
|
+
|
|
3936
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
3937
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at);
|
|
3938
|
+
`);
|
|
3939
|
+
this.db.exec(`
|
|
3940
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
3941
|
+
${prefixColumnsSql}queue_name TEXT PRIMARY KEY,
|
|
3942
|
+
next_available_at TEXT
|
|
3943
|
+
);
|
|
3944
|
+
`);
|
|
3945
|
+
}
|
|
3946
|
+
async recordExecution(queueName) {
|
|
3947
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3948
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3949
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
3950
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
3951
|
+
const stmt = this.db.prepare(`
|
|
3952
|
+
INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
|
|
3953
|
+
VALUES (${prefixPlaceholders}?)
|
|
3954
|
+
`);
|
|
3955
|
+
stmt.run(...prefixParamValues, queueName);
|
|
3956
|
+
}
|
|
3957
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
3958
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3959
|
+
const prefixParams = this.getPrefixParamValues();
|
|
3960
|
+
const thresholdTime = toSQLiteTimestamp(new Date(windowStartTime));
|
|
3961
|
+
const stmt = this.db.prepare(`
|
|
3962
|
+
SELECT COUNT(*) AS count
|
|
3963
|
+
FROM ${this.executionTableName}
|
|
3964
|
+
WHERE queue_name = ? AND executed_at > ?${prefixConditions}
|
|
3965
|
+
`);
|
|
3966
|
+
const result = stmt.get(queueName, thresholdTime, ...prefixParams);
|
|
3967
|
+
return result?.count ?? 0;
|
|
3968
|
+
}
|
|
3969
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
3970
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3971
|
+
const prefixParams = this.getPrefixParamValues();
|
|
3972
|
+
const stmt = this.db.prepare(`
|
|
3973
|
+
SELECT executed_at
|
|
3974
|
+
FROM ${this.executionTableName}
|
|
3975
|
+
WHERE queue_name = ?${prefixConditions}
|
|
3976
|
+
ORDER BY executed_at ASC
|
|
3977
|
+
LIMIT 1 OFFSET ?
|
|
3978
|
+
`);
|
|
3979
|
+
const result = stmt.get(queueName, ...prefixParams, offset);
|
|
3980
|
+
if (!result)
|
|
3981
|
+
return;
|
|
3982
|
+
return result.executed_at + "Z";
|
|
3983
|
+
}
|
|
3984
|
+
async getNextAvailableTime(queueName) {
|
|
3985
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3986
|
+
const prefixParams = this.getPrefixParamValues();
|
|
3987
|
+
const stmt = this.db.prepare(`
|
|
3988
|
+
SELECT next_available_at
|
|
3989
|
+
FROM ${this.nextAvailableTableName}
|
|
3990
|
+
WHERE queue_name = ?${prefixConditions}
|
|
3991
|
+
`);
|
|
3992
|
+
const result = stmt.get(queueName, ...prefixParams);
|
|
3993
|
+
if (!result?.next_available_at)
|
|
3994
|
+
return;
|
|
3995
|
+
return result.next_available_at + "Z";
|
|
3996
|
+
}
|
|
3997
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
3998
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3999
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
4000
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
4001
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
4002
|
+
const stmt = this.db.prepare(`
|
|
4003
|
+
INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
|
|
4004
|
+
VALUES (${prefixPlaceholders}?, ?)
|
|
4005
|
+
ON CONFLICT(queue_name) DO UPDATE SET next_available_at = excluded.next_available_at
|
|
4006
|
+
`);
|
|
4007
|
+
stmt.run(...prefixParamValues, queueName, nextAvailableAt);
|
|
4008
|
+
}
|
|
4009
|
+
async clear(queueName) {
|
|
4010
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
4011
|
+
const prefixParams = this.getPrefixParamValues();
|
|
4012
|
+
this.db.prepare(`DELETE FROM ${this.executionTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
|
|
4013
|
+
this.db.prepare(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
|
|
4014
|
+
}
|
|
4015
|
+
}
|
|
4016
|
+
// src/limiter/SupabaseRateLimiterStorage.ts
|
|
4017
|
+
import { createServiceToken as createServiceToken24 } from "@workglow/util";
|
|
4018
|
+
var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken24("ratelimiter.storage.supabase");
|
|
4019
|
+
|
|
4020
|
+
class SupabaseRateLimiterStorage {
|
|
4021
|
+
client;
|
|
4022
|
+
prefixes;
|
|
4023
|
+
prefixValues;
|
|
4024
|
+
executionTableName;
|
|
4025
|
+
nextAvailableTableName;
|
|
4026
|
+
constructor(client, options) {
|
|
4027
|
+
this.client = client;
|
|
4028
|
+
this.prefixes = options?.prefixes ?? [];
|
|
4029
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
4030
|
+
if (this.prefixes.length > 0) {
|
|
4031
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
4032
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
4033
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
4034
|
+
} else {
|
|
4035
|
+
this.executionTableName = "rate_limit_executions";
|
|
4036
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
4037
|
+
}
|
|
4038
|
+
}
|
|
4039
|
+
getPrefixColumnType(type) {
|
|
4040
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
4041
|
+
}
|
|
4042
|
+
buildPrefixColumnsSql() {
|
|
4043
|
+
if (this.prefixes.length === 0)
|
|
4044
|
+
return "";
|
|
4045
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
4046
|
+
`) + `,
|
|
4047
|
+
`;
|
|
4048
|
+
}
|
|
4049
|
+
getPrefixColumnNames() {
|
|
4050
|
+
return this.prefixes.map((p) => p.name);
|
|
4051
|
+
}
|
|
4052
|
+
applyPrefixFilters(query) {
|
|
4053
|
+
let result = query;
|
|
4054
|
+
for (const prefix of this.prefixes) {
|
|
4055
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
4056
|
+
}
|
|
4057
|
+
return result;
|
|
4058
|
+
}
|
|
4059
|
+
getPrefixInsertValues() {
|
|
4060
|
+
const values = {};
|
|
4061
|
+
for (const prefix of this.prefixes) {
|
|
4062
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
4063
|
+
}
|
|
4064
|
+
return values;
|
|
4065
|
+
}
|
|
4066
|
+
async setupDatabase() {
|
|
4067
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
4068
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4069
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
4070
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
4071
|
+
const createExecTableSql = `
|
|
4072
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
4073
|
+
id SERIAL PRIMARY KEY,
|
|
4074
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
4075
|
+
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
4076
|
+
)
|
|
4077
|
+
`;
|
|
4078
|
+
const { error: execTableError } = await this.client.rpc("exec_sql", {
|
|
4079
|
+
query: createExecTableSql
|
|
4080
|
+
});
|
|
4081
|
+
if (execTableError && execTableError.code !== "42P07") {
|
|
4082
|
+
throw execTableError;
|
|
4083
|
+
}
|
|
4084
|
+
const createExecIndexSql = `
|
|
4085
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
4086
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
4087
|
+
`;
|
|
4088
|
+
await this.client.rpc("exec_sql", { query: createExecIndexSql });
|
|
4089
|
+
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
4090
|
+
const createNextTableSql = `
|
|
4091
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
4092
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
4093
|
+
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
4094
|
+
PRIMARY KEY (${primaryKeyColumns})
|
|
4095
|
+
)
|
|
4096
|
+
`;
|
|
4097
|
+
const { error: nextTableError } = await this.client.rpc("exec_sql", {
|
|
4098
|
+
query: createNextTableSql
|
|
4099
|
+
});
|
|
4100
|
+
if (nextTableError && nextTableError.code !== "42P07") {
|
|
4101
|
+
throw nextTableError;
|
|
4102
|
+
}
|
|
4103
|
+
}
|
|
4104
|
+
async recordExecution(queueName) {
|
|
4105
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
4106
|
+
const { error } = await this.client.from(this.executionTableName).insert({
|
|
4107
|
+
...prefixInsertValues,
|
|
4108
|
+
queue_name: queueName
|
|
4109
|
+
});
|
|
4110
|
+
if (error)
|
|
4111
|
+
throw error;
|
|
4112
|
+
}
|
|
4113
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
4114
|
+
let query = this.client.from(this.executionTableName).select("*", { count: "exact", head: true }).eq("queue_name", queueName).gt("executed_at", windowStartTime);
|
|
4115
|
+
query = this.applyPrefixFilters(query);
|
|
4116
|
+
const { count, error } = await query;
|
|
4117
|
+
if (error)
|
|
4118
|
+
throw error;
|
|
4119
|
+
return count ?? 0;
|
|
4120
|
+
}
|
|
4121
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
4122
|
+
let query = this.client.from(this.executionTableName).select("executed_at").eq("queue_name", queueName);
|
|
4123
|
+
query = this.applyPrefixFilters(query);
|
|
4124
|
+
const { data, error } = await query.order("executed_at", { ascending: true }).range(offset, offset);
|
|
4125
|
+
if (error)
|
|
4126
|
+
throw error;
|
|
4127
|
+
if (!data || data.length === 0)
|
|
4128
|
+
return;
|
|
4129
|
+
return new Date(data[0].executed_at).toISOString();
|
|
4130
|
+
}
|
|
4131
|
+
async getNextAvailableTime(queueName) {
|
|
4132
|
+
let query = this.client.from(this.nextAvailableTableName).select("next_available_at").eq("queue_name", queueName);
|
|
4133
|
+
query = this.applyPrefixFilters(query);
|
|
4134
|
+
const { data, error } = await query.single();
|
|
4135
|
+
if (error) {
|
|
4136
|
+
if (error.code === "PGRST116")
|
|
4137
|
+
return;
|
|
4138
|
+
throw error;
|
|
4139
|
+
}
|
|
4140
|
+
if (!data?.next_available_at)
|
|
4141
|
+
return;
|
|
4142
|
+
return new Date(data.next_available_at).toISOString();
|
|
4143
|
+
}
|
|
4144
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
4145
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
4146
|
+
const { error } = await this.client.from(this.nextAvailableTableName).upsert({
|
|
4147
|
+
...prefixInsertValues,
|
|
4148
|
+
queue_name: queueName,
|
|
4149
|
+
next_available_at: nextAvailableAt
|
|
4150
|
+
}, {
|
|
4151
|
+
onConflict: this.prefixes.length > 0 ? `${this.getPrefixColumnNames().join(",")},queue_name` : "queue_name"
|
|
4152
|
+
});
|
|
4153
|
+
if (error)
|
|
4154
|
+
throw error;
|
|
4155
|
+
}
|
|
4156
|
+
async clear(queueName) {
|
|
4157
|
+
let execQuery = this.client.from(this.executionTableName).delete().eq("queue_name", queueName);
|
|
4158
|
+
execQuery = this.applyPrefixFilters(execQuery);
|
|
4159
|
+
const { error: execError } = await execQuery;
|
|
4160
|
+
if (execError)
|
|
4161
|
+
throw execError;
|
|
4162
|
+
let nextQuery = this.client.from(this.nextAvailableTableName).delete().eq("queue_name", queueName);
|
|
4163
|
+
nextQuery = this.applyPrefixFilters(nextQuery);
|
|
4164
|
+
const { error: nextError } = await nextQuery;
|
|
4165
|
+
if (nextError)
|
|
4166
|
+
throw nextError;
|
|
4167
|
+
}
|
|
4168
|
+
}
|
|
4169
|
+
// src/kv/IndexedDbKvRepository.ts
|
|
4170
|
+
import { createServiceToken as createServiceToken26 } from "@workglow/util";
|
|
4171
|
+
|
|
4172
|
+
// src/tabular/IndexedDbTabularRepository.ts
|
|
4173
|
+
import { createServiceToken as createServiceToken25 } from "@workglow/util";
|
|
4174
|
+
|
|
4175
|
+
// src/util/IndexedDbTable.ts
|
|
4176
|
+
var METADATA_STORE_NAME = "__schema_metadata__";
|
|
4177
|
+
async function saveSchemaMetadata(db, tableName, snapshot) {
|
|
4178
|
+
return new Promise((resolve, reject) => {
|
|
2881
4179
|
try {
|
|
2882
4180
|
const transaction = db.transaction(METADATA_STORE_NAME, "readwrite");
|
|
2883
4181
|
const store = transaction.objectStore(METADATA_STORE_NAME);
|
|
@@ -3212,7 +4510,7 @@ async function dropIndexedDbTable(tableName) {
|
|
|
3212
4510
|
}
|
|
3213
4511
|
|
|
3214
4512
|
// src/tabular/IndexedDbTabularRepository.ts
|
|
3215
|
-
var IDB_TABULAR_REPOSITORY =
|
|
4513
|
+
var IDB_TABULAR_REPOSITORY = createServiceToken25("storage.tabularRepository.indexedDb");
|
|
3216
4514
|
|
|
3217
4515
|
class IndexedDbTabularRepository extends TabularRepository {
|
|
3218
4516
|
table;
|
|
@@ -3548,7 +4846,7 @@ class IndexedDbTabularRepository extends TabularRepository {
|
|
|
3548
4846
|
}
|
|
3549
4847
|
|
|
3550
4848
|
// src/kv/IndexedDbKvRepository.ts
|
|
3551
|
-
var IDB_KV_REPOSITORY =
|
|
4849
|
+
var IDB_KV_REPOSITORY = createServiceToken26("storage.kvRepository.indexedDb");
|
|
3552
4850
|
|
|
3553
4851
|
class IndexedDbKvRepository extends KvViaTabularRepository {
|
|
3554
4852
|
dbName;
|
|
@@ -3560,18 +4858,42 @@ class IndexedDbKvRepository extends KvViaTabularRepository {
|
|
|
3560
4858
|
}
|
|
3561
4859
|
}
|
|
3562
4860
|
// src/queue/IndexedDbQueueStorage.ts
|
|
3563
|
-
import { createServiceToken as
|
|
3564
|
-
var INDEXED_DB_QUEUE_STORAGE =
|
|
4861
|
+
import { createServiceToken as createServiceToken27, makeFingerprint as makeFingerprint8, uuid4 as uuid45 } from "@workglow/util";
|
|
4862
|
+
var INDEXED_DB_QUEUE_STORAGE = createServiceToken27("jobqueue.storage.indexedDb");
|
|
3565
4863
|
|
|
3566
4864
|
class IndexedDbQueueStorage {
|
|
3567
4865
|
queueName;
|
|
3568
4866
|
db;
|
|
3569
4867
|
tableName;
|
|
3570
4868
|
migrationOptions;
|
|
3571
|
-
|
|
4869
|
+
prefixes;
|
|
4870
|
+
prefixValues;
|
|
4871
|
+
pollingManager = null;
|
|
4872
|
+
constructor(queueName, options = {}) {
|
|
3572
4873
|
this.queueName = queueName;
|
|
3573
|
-
this.
|
|
3574
|
-
this.
|
|
4874
|
+
this.migrationOptions = options;
|
|
4875
|
+
this.prefixes = options.prefixes ?? [];
|
|
4876
|
+
this.prefixValues = options.prefixValues ?? {};
|
|
4877
|
+
if (this.prefixes.length > 0) {
|
|
4878
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
4879
|
+
this.tableName = `jobs_${prefixNames}`;
|
|
4880
|
+
} else {
|
|
4881
|
+
this.tableName = "jobs";
|
|
4882
|
+
}
|
|
4883
|
+
}
|
|
4884
|
+
getPrefixColumnNames() {
|
|
4885
|
+
return this.prefixes.map((p) => p.name);
|
|
4886
|
+
}
|
|
4887
|
+
matchesPrefixes(job) {
|
|
4888
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
4889
|
+
if (job[key] !== value) {
|
|
4890
|
+
return false;
|
|
4891
|
+
}
|
|
4892
|
+
}
|
|
4893
|
+
return true;
|
|
4894
|
+
}
|
|
4895
|
+
getPrefixKeyValues() {
|
|
4896
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3575
4897
|
}
|
|
3576
4898
|
async getDb() {
|
|
3577
4899
|
if (this.db)
|
|
@@ -3580,25 +4902,29 @@ class IndexedDbQueueStorage {
|
|
|
3580
4902
|
return this.db;
|
|
3581
4903
|
}
|
|
3582
4904
|
async setupDatabase() {
|
|
4905
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4906
|
+
const buildKeyPath = (basePath) => {
|
|
4907
|
+
return [...prefixColumnNames, ...basePath];
|
|
4908
|
+
};
|
|
3583
4909
|
const expectedIndexes = [
|
|
3584
4910
|
{
|
|
3585
|
-
name: "
|
|
3586
|
-
keyPath:
|
|
4911
|
+
name: "queue_status",
|
|
4912
|
+
keyPath: buildKeyPath(["queue", "status"]),
|
|
3587
4913
|
options: { unique: false }
|
|
3588
4914
|
},
|
|
3589
4915
|
{
|
|
3590
|
-
name: "
|
|
3591
|
-
keyPath: ["status", "run_after"],
|
|
4916
|
+
name: "queue_status_run_after",
|
|
4917
|
+
keyPath: buildKeyPath(["queue", "status", "run_after"]),
|
|
3592
4918
|
options: { unique: false }
|
|
3593
4919
|
},
|
|
3594
4920
|
{
|
|
3595
|
-
name: "
|
|
3596
|
-
keyPath:
|
|
4921
|
+
name: "queue_job_run_id",
|
|
4922
|
+
keyPath: buildKeyPath(["queue", "job_run_id"]),
|
|
3597
4923
|
options: { unique: false }
|
|
3598
4924
|
},
|
|
3599
4925
|
{
|
|
3600
|
-
name: "
|
|
3601
|
-
keyPath: ["fingerprint", "status"],
|
|
4926
|
+
name: "queue_fingerprint_status",
|
|
4927
|
+
keyPath: buildKeyPath(["queue", "fingerprint", "status"]),
|
|
3602
4928
|
options: { unique: false }
|
|
3603
4929
|
}
|
|
3604
4930
|
];
|
|
@@ -3607,21 +4933,25 @@ class IndexedDbQueueStorage {
|
|
|
3607
4933
|
async add(job) {
|
|
3608
4934
|
const db = await this.getDb();
|
|
3609
4935
|
const now = new Date().toISOString();
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
4936
|
+
const jobWithPrefixes = job;
|
|
4937
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid45();
|
|
4938
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid45();
|
|
4939
|
+
jobWithPrefixes.queue = this.queueName;
|
|
4940
|
+
jobWithPrefixes.fingerprint = await makeFingerprint8(jobWithPrefixes.input);
|
|
4941
|
+
jobWithPrefixes.status = "PENDING" /* PENDING */;
|
|
4942
|
+
jobWithPrefixes.progress = 0;
|
|
4943
|
+
jobWithPrefixes.progress_message = "";
|
|
4944
|
+
jobWithPrefixes.progress_details = null;
|
|
4945
|
+
jobWithPrefixes.created_at = now;
|
|
4946
|
+
jobWithPrefixes.run_after = now;
|
|
4947
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
4948
|
+
jobWithPrefixes[key] = value;
|
|
4949
|
+
}
|
|
3620
4950
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3621
4951
|
const store = tx.objectStore(this.tableName);
|
|
3622
4952
|
return new Promise((resolve, reject) => {
|
|
3623
|
-
const request = store.add(
|
|
3624
|
-
tx.oncomplete = () => resolve(
|
|
4953
|
+
const request = store.add(jobWithPrefixes);
|
|
4954
|
+
tx.oncomplete = () => resolve(jobWithPrefixes.id);
|
|
3625
4955
|
tx.onerror = () => reject(tx.error);
|
|
3626
4956
|
request.onerror = () => reject(request.error);
|
|
3627
4957
|
});
|
|
@@ -3632,7 +4962,14 @@ class IndexedDbQueueStorage {
|
|
|
3632
4962
|
const store = tx.objectStore(this.tableName);
|
|
3633
4963
|
const request = store.get(id);
|
|
3634
4964
|
return new Promise((resolve, reject) => {
|
|
3635
|
-
request.onsuccess = () =>
|
|
4965
|
+
request.onsuccess = () => {
|
|
4966
|
+
const job = request.result;
|
|
4967
|
+
if (job && job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
4968
|
+
resolve(job);
|
|
4969
|
+
} else {
|
|
4970
|
+
resolve(undefined);
|
|
4971
|
+
}
|
|
4972
|
+
};
|
|
3636
4973
|
request.onerror = () => reject(request.error);
|
|
3637
4974
|
tx.onerror = () => reject(tx.error);
|
|
3638
4975
|
});
|
|
@@ -3641,10 +4978,11 @@ class IndexedDbQueueStorage {
|
|
|
3641
4978
|
const db = await this.getDb();
|
|
3642
4979
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3643
4980
|
const store = tx.objectStore(this.tableName);
|
|
3644
|
-
const index = store.index("
|
|
4981
|
+
const index = store.index("queue_status_run_after");
|
|
4982
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3645
4983
|
return new Promise((resolve, reject) => {
|
|
3646
4984
|
const ret = new Map;
|
|
3647
|
-
const keyRange = IDBKeyRange.bound([status, ""], [status, "\uFFFF"]);
|
|
4985
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, status, ""], [...prefixKeyValues, this.queueName, status, "\uFFFF"]);
|
|
3648
4986
|
const cursorRequest = index.openCursor(keyRange);
|
|
3649
4987
|
const handleCursor = (e) => {
|
|
3650
4988
|
const cursor = e.target.result;
|
|
@@ -3652,7 +4990,10 @@ class IndexedDbQueueStorage {
|
|
|
3652
4990
|
resolve(Array.from(ret.values()));
|
|
3653
4991
|
return;
|
|
3654
4992
|
}
|
|
3655
|
-
|
|
4993
|
+
const job = cursor.value;
|
|
4994
|
+
if (this.matchesPrefixes(job)) {
|
|
4995
|
+
ret.set(cursor.value.id, cursor.value);
|
|
4996
|
+
}
|
|
3656
4997
|
cursor.continue();
|
|
3657
4998
|
};
|
|
3658
4999
|
cursorRequest.onsuccess = handleCursor;
|
|
@@ -3660,14 +5001,15 @@ class IndexedDbQueueStorage {
|
|
|
3660
5001
|
tx.onerror = () => reject(tx.error);
|
|
3661
5002
|
});
|
|
3662
5003
|
}
|
|
3663
|
-
async next() {
|
|
5004
|
+
async next(workerId) {
|
|
3664
5005
|
const db = await this.getDb();
|
|
3665
5006
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3666
5007
|
const store = tx.objectStore(this.tableName);
|
|
3667
|
-
const index = store.index("
|
|
5008
|
+
const index = store.index("queue_status_run_after");
|
|
3668
5009
|
const now = new Date().toISOString();
|
|
5010
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3669
5011
|
return new Promise((resolve, reject) => {
|
|
3670
|
-
const cursorRequest = index.openCursor(IDBKeyRange.bound(["PENDING" /* PENDING */, ""], ["PENDING" /* PENDING */, now], false, true));
|
|
5012
|
+
const cursorRequest = index.openCursor(IDBKeyRange.bound([...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, ""], [...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, now], false, true));
|
|
3671
5013
|
let jobToReturn;
|
|
3672
5014
|
cursorRequest.onsuccess = (e) => {
|
|
3673
5015
|
const cursor = e.target.result;
|
|
@@ -3680,12 +5022,13 @@ class IndexedDbQueueStorage {
|
|
|
3680
5022
|
return;
|
|
3681
5023
|
}
|
|
3682
5024
|
const job = cursor.value;
|
|
3683
|
-
if (job.status !== "PENDING" /* PENDING */) {
|
|
5025
|
+
if (job.queue !== this.queueName || job.status !== "PENDING" /* PENDING */ || !this.matchesPrefixes(job)) {
|
|
3684
5026
|
cursor.continue();
|
|
3685
5027
|
return;
|
|
3686
5028
|
}
|
|
3687
5029
|
job.status = "PROCESSING" /* PROCESSING */;
|
|
3688
5030
|
job.last_ran_at = now;
|
|
5031
|
+
job.worker_id = workerId ?? null;
|
|
3689
5032
|
try {
|
|
3690
5033
|
const updateRequest = store.put(job);
|
|
3691
5034
|
updateRequest.onsuccess = () => {
|
|
@@ -3709,11 +5052,13 @@ class IndexedDbQueueStorage {
|
|
|
3709
5052
|
}
|
|
3710
5053
|
async size(status = "PENDING" /* PENDING */) {
|
|
3711
5054
|
const db = await this.getDb();
|
|
5055
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3712
5056
|
return new Promise((resolve, reject) => {
|
|
3713
5057
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3714
5058
|
const store = tx.objectStore(this.tableName);
|
|
3715
|
-
const index = store.index("
|
|
3716
|
-
const
|
|
5059
|
+
const index = store.index("queue_status");
|
|
5060
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
5061
|
+
const request = index.count(keyRange);
|
|
3717
5062
|
request.onsuccess = () => resolve(request.result);
|
|
3718
5063
|
request.onerror = () => reject(request.error);
|
|
3719
5064
|
tx.onerror = () => reject(tx.error);
|
|
@@ -3727,9 +5072,18 @@ class IndexedDbQueueStorage {
|
|
|
3727
5072
|
const getReq = store.get(job.id);
|
|
3728
5073
|
getReq.onsuccess = () => {
|
|
3729
5074
|
const existing = getReq.result;
|
|
3730
|
-
|
|
5075
|
+
if (!existing || existing.queue !== this.queueName || !this.matchesPrefixes(existing)) {
|
|
5076
|
+
reject(new Error(`Job ${job.id} not found or does not belong to queue ${this.queueName}`));
|
|
5077
|
+
return;
|
|
5078
|
+
}
|
|
5079
|
+
const currentAttempts = existing.run_attempts ?? 0;
|
|
3731
5080
|
job.run_attempts = currentAttempts + 1;
|
|
3732
|
-
|
|
5081
|
+
job.queue = this.queueName;
|
|
5082
|
+
const jobWithPrefixes = job;
|
|
5083
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5084
|
+
jobWithPrefixes[key] = value;
|
|
5085
|
+
}
|
|
5086
|
+
const putReq = store.put(jobWithPrefixes);
|
|
3733
5087
|
putReq.onsuccess = () => {};
|
|
3734
5088
|
putReq.onerror = () => reject(putReq.error);
|
|
3735
5089
|
};
|
|
@@ -3749,10 +5103,15 @@ class IndexedDbQueueStorage {
|
|
|
3749
5103
|
const db = await this.getDb();
|
|
3750
5104
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3751
5105
|
const store = tx.objectStore(this.tableName);
|
|
3752
|
-
const index = store.index("
|
|
3753
|
-
const
|
|
5106
|
+
const index = store.index("queue_job_run_id");
|
|
5107
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5108
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, job_run_id]);
|
|
5109
|
+
const request = index.getAll(keyRange);
|
|
3754
5110
|
return new Promise((resolve, reject) => {
|
|
3755
|
-
request.onsuccess = () =>
|
|
5111
|
+
request.onsuccess = () => {
|
|
5112
|
+
const results = (request.result || []).filter((job) => this.matchesPrefixes(job));
|
|
5113
|
+
resolve(results);
|
|
5114
|
+
};
|
|
3756
5115
|
request.onerror = () => reject(request.error);
|
|
3757
5116
|
tx.onerror = () => reject(tx.error);
|
|
3758
5117
|
});
|
|
@@ -3761,11 +5120,31 @@ class IndexedDbQueueStorage {
|
|
|
3761
5120
|
const db = await this.getDb();
|
|
3762
5121
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3763
5122
|
const store = tx.objectStore(this.tableName);
|
|
3764
|
-
const
|
|
5123
|
+
const index = store.index("queue_status");
|
|
5124
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3765
5125
|
return new Promise((resolve, reject) => {
|
|
3766
|
-
|
|
3767
|
-
request
|
|
5126
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, "\uFFFF"]);
|
|
5127
|
+
const request = index.openCursor(keyRange);
|
|
5128
|
+
request.onsuccess = (event) => {
|
|
5129
|
+
const cursor = event.target.result;
|
|
5130
|
+
if (cursor) {
|
|
5131
|
+
const job = cursor.value;
|
|
5132
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
5133
|
+
const deleteRequest = cursor.delete();
|
|
5134
|
+
deleteRequest.onsuccess = () => {
|
|
5135
|
+
cursor.continue();
|
|
5136
|
+
};
|
|
5137
|
+
deleteRequest.onerror = () => {
|
|
5138
|
+
cursor.continue();
|
|
5139
|
+
};
|
|
5140
|
+
} else {
|
|
5141
|
+
cursor.continue();
|
|
5142
|
+
}
|
|
5143
|
+
}
|
|
5144
|
+
};
|
|
5145
|
+
tx.oncomplete = () => resolve();
|
|
3768
5146
|
tx.onerror = () => reject(tx.error);
|
|
5147
|
+
request.onerror = () => reject(request.error);
|
|
3769
5148
|
});
|
|
3770
5149
|
}
|
|
3771
5150
|
async outputForInput(input) {
|
|
@@ -3773,10 +5152,23 @@ class IndexedDbQueueStorage {
|
|
|
3773
5152
|
const db = await this.getDb();
|
|
3774
5153
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3775
5154
|
const store = tx.objectStore(this.tableName);
|
|
3776
|
-
const index = store.index("
|
|
3777
|
-
const
|
|
5155
|
+
const index = store.index("queue_fingerprint_status");
|
|
5156
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5157
|
+
const request = index.get([
|
|
5158
|
+
...prefixKeyValues,
|
|
5159
|
+
this.queueName,
|
|
5160
|
+
fingerprint,
|
|
5161
|
+
"COMPLETED" /* COMPLETED */
|
|
5162
|
+
]);
|
|
3778
5163
|
return new Promise((resolve, reject) => {
|
|
3779
|
-
request.onsuccess = () =>
|
|
5164
|
+
request.onsuccess = () => {
|
|
5165
|
+
const job = request.result;
|
|
5166
|
+
if (job && this.matchesPrefixes(job)) {
|
|
5167
|
+
resolve(job.output ?? null);
|
|
5168
|
+
} else {
|
|
5169
|
+
resolve(null);
|
|
5170
|
+
}
|
|
5171
|
+
};
|
|
3780
5172
|
request.onerror = () => reject(request.error);
|
|
3781
5173
|
tx.onerror = () => reject(tx.error);
|
|
3782
5174
|
});
|
|
@@ -3791,6 +5183,9 @@ class IndexedDbQueueStorage {
|
|
|
3791
5183
|
await this.complete(job);
|
|
3792
5184
|
}
|
|
3793
5185
|
async delete(id) {
|
|
5186
|
+
const job = await this.get(id);
|
|
5187
|
+
if (!job)
|
|
5188
|
+
return;
|
|
3794
5189
|
const db = await this.getDb();
|
|
3795
5190
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3796
5191
|
const store = tx.objectStore(this.tableName);
|
|
@@ -3805,15 +5200,17 @@ class IndexedDbQueueStorage {
|
|
|
3805
5200
|
const db = await this.getDb();
|
|
3806
5201
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3807
5202
|
const store = tx.objectStore(this.tableName);
|
|
3808
|
-
const index = store.index("
|
|
5203
|
+
const index = store.index("queue_status");
|
|
3809
5204
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
5205
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5206
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
3810
5207
|
return new Promise((resolve, reject) => {
|
|
3811
|
-
const request = index.openCursor();
|
|
5208
|
+
const request = index.openCursor(keyRange);
|
|
3812
5209
|
request.onsuccess = (event) => {
|
|
3813
5210
|
const cursor = event.target.result;
|
|
3814
5211
|
if (cursor) {
|
|
3815
5212
|
const job = cursor.value;
|
|
3816
|
-
if (job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
|
|
5213
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
|
|
3817
5214
|
cursor.delete();
|
|
3818
5215
|
}
|
|
3819
5216
|
cursor.continue();
|
|
@@ -3824,6 +5221,363 @@ class IndexedDbQueueStorage {
|
|
|
3824
5221
|
request.onerror = () => reject(request.error);
|
|
3825
5222
|
});
|
|
3826
5223
|
}
|
|
5224
|
+
async getAllJobs() {
|
|
5225
|
+
const db = await this.getDb();
|
|
5226
|
+
const tx = db.transaction(this.tableName, "readonly");
|
|
5227
|
+
const store = tx.objectStore(this.tableName);
|
|
5228
|
+
const index = store.index("queue_status");
|
|
5229
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5230
|
+
return new Promise((resolve, reject) => {
|
|
5231
|
+
const jobs = [];
|
|
5232
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, "\uFFFF"]);
|
|
5233
|
+
const request = index.openCursor(keyRange);
|
|
5234
|
+
request.onsuccess = (event) => {
|
|
5235
|
+
const cursor = event.target.result;
|
|
5236
|
+
if (cursor) {
|
|
5237
|
+
const job = cursor.value;
|
|
5238
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
5239
|
+
jobs.push(job);
|
|
5240
|
+
}
|
|
5241
|
+
cursor.continue();
|
|
5242
|
+
}
|
|
5243
|
+
};
|
|
5244
|
+
tx.oncomplete = () => resolve(jobs);
|
|
5245
|
+
tx.onerror = () => reject(tx.error);
|
|
5246
|
+
request.onerror = () => reject(request.error);
|
|
5247
|
+
});
|
|
5248
|
+
}
|
|
5249
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
5250
|
+
const db = await this.getDb();
|
|
5251
|
+
const tx = db.transaction(this.tableName, "readonly");
|
|
5252
|
+
const store = tx.objectStore(this.tableName);
|
|
5253
|
+
return new Promise((resolve, reject) => {
|
|
5254
|
+
const jobs = [];
|
|
5255
|
+
const request = store.openCursor();
|
|
5256
|
+
request.onsuccess = (event) => {
|
|
5257
|
+
const cursor = event.target.result;
|
|
5258
|
+
if (cursor) {
|
|
5259
|
+
const job = cursor.value;
|
|
5260
|
+
if (job.queue !== this.queueName) {
|
|
5261
|
+
cursor.continue();
|
|
5262
|
+
return;
|
|
5263
|
+
}
|
|
5264
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
5265
|
+
jobs.push(job);
|
|
5266
|
+
} else {
|
|
5267
|
+
let matches = true;
|
|
5268
|
+
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
5269
|
+
if (job[key] !== value) {
|
|
5270
|
+
matches = false;
|
|
5271
|
+
break;
|
|
5272
|
+
}
|
|
5273
|
+
}
|
|
5274
|
+
if (matches) {
|
|
5275
|
+
jobs.push(job);
|
|
5276
|
+
}
|
|
5277
|
+
}
|
|
5278
|
+
cursor.continue();
|
|
5279
|
+
}
|
|
5280
|
+
};
|
|
5281
|
+
tx.oncomplete = () => resolve(jobs);
|
|
5282
|
+
tx.onerror = () => reject(tx.error);
|
|
5283
|
+
request.onerror = () => reject(request.error);
|
|
5284
|
+
});
|
|
5285
|
+
}
|
|
5286
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
5287
|
+
if (prefixFilter === undefined) {
|
|
5288
|
+
return false;
|
|
5289
|
+
}
|
|
5290
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
5291
|
+
return true;
|
|
5292
|
+
}
|
|
5293
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
5294
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
5295
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
5296
|
+
return true;
|
|
5297
|
+
}
|
|
5298
|
+
for (const key of instanceKeys) {
|
|
5299
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
5300
|
+
return true;
|
|
5301
|
+
}
|
|
5302
|
+
}
|
|
5303
|
+
return false;
|
|
5304
|
+
}
|
|
5305
|
+
getPollingManager() {
|
|
5306
|
+
if (!this.pollingManager) {
|
|
5307
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
5308
|
+
const jobs = await this.getAllJobs();
|
|
5309
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
5310
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
5311
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
5312
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
5313
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
5314
|
+
});
|
|
5315
|
+
}
|
|
5316
|
+
return this.pollingManager;
|
|
5317
|
+
}
|
|
5318
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
5319
|
+
let lastKnownJobs = new Map;
|
|
5320
|
+
let cancelled = false;
|
|
5321
|
+
const poll = async () => {
|
|
5322
|
+
if (cancelled)
|
|
5323
|
+
return;
|
|
5324
|
+
try {
|
|
5325
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
5326
|
+
if (cancelled)
|
|
5327
|
+
return;
|
|
5328
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
5329
|
+
for (const [id, job] of currentMap) {
|
|
5330
|
+
const old = lastKnownJobs.get(id);
|
|
5331
|
+
if (!old) {
|
|
5332
|
+
callback({ type: "INSERT", new: job });
|
|
5333
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
5334
|
+
callback({ type: "UPDATE", old, new: job });
|
|
5335
|
+
}
|
|
5336
|
+
}
|
|
5337
|
+
for (const [id, job] of lastKnownJobs) {
|
|
5338
|
+
if (!currentMap.has(id)) {
|
|
5339
|
+
callback({ type: "DELETE", old: job });
|
|
5340
|
+
}
|
|
5341
|
+
}
|
|
5342
|
+
lastKnownJobs = currentMap;
|
|
5343
|
+
} catch {}
|
|
5344
|
+
};
|
|
5345
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
5346
|
+
poll();
|
|
5347
|
+
return () => {
|
|
5348
|
+
cancelled = true;
|
|
5349
|
+
clearInterval(intervalId);
|
|
5350
|
+
};
|
|
5351
|
+
}
|
|
5352
|
+
subscribeToChanges(callback, options) {
|
|
5353
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
5354
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
5355
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
5356
|
+
}
|
|
5357
|
+
const manager = this.getPollingManager();
|
|
5358
|
+
return manager.subscribe(callback, { intervalMs });
|
|
5359
|
+
}
|
|
5360
|
+
}
|
|
5361
|
+
// src/limiter/IndexedDbRateLimiterStorage.ts
|
|
5362
|
+
import { createServiceToken as createServiceToken28 } from "@workglow/util";
|
|
5363
|
+
var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken28("ratelimiter.storage.indexedDb");
|
|
5364
|
+
|
|
5365
|
+
class IndexedDbRateLimiterStorage {
|
|
5366
|
+
executionDb;
|
|
5367
|
+
nextAvailableDb;
|
|
5368
|
+
executionTableName;
|
|
5369
|
+
nextAvailableTableName;
|
|
5370
|
+
migrationOptions;
|
|
5371
|
+
prefixes;
|
|
5372
|
+
prefixValues;
|
|
5373
|
+
constructor(options = {}) {
|
|
5374
|
+
this.migrationOptions = options;
|
|
5375
|
+
this.prefixes = options.prefixes ?? [];
|
|
5376
|
+
this.prefixValues = options.prefixValues ?? {};
|
|
5377
|
+
if (this.prefixes.length > 0) {
|
|
5378
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
5379
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
5380
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
5381
|
+
} else {
|
|
5382
|
+
this.executionTableName = "rate_limit_executions";
|
|
5383
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
5384
|
+
}
|
|
5385
|
+
}
|
|
5386
|
+
getPrefixColumnNames() {
|
|
5387
|
+
return this.prefixes.map((p) => p.name);
|
|
5388
|
+
}
|
|
5389
|
+
matchesPrefixes(record) {
|
|
5390
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5391
|
+
if (record[key] !== value) {
|
|
5392
|
+
return false;
|
|
5393
|
+
}
|
|
5394
|
+
}
|
|
5395
|
+
return true;
|
|
5396
|
+
}
|
|
5397
|
+
getPrefixKeyValues() {
|
|
5398
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
5399
|
+
}
|
|
5400
|
+
async getExecutionDb() {
|
|
5401
|
+
if (this.executionDb)
|
|
5402
|
+
return this.executionDb;
|
|
5403
|
+
await this.setupDatabase();
|
|
5404
|
+
return this.executionDb;
|
|
5405
|
+
}
|
|
5406
|
+
async getNextAvailableDb() {
|
|
5407
|
+
if (this.nextAvailableDb)
|
|
5408
|
+
return this.nextAvailableDb;
|
|
5409
|
+
await this.setupDatabase();
|
|
5410
|
+
return this.nextAvailableDb;
|
|
5411
|
+
}
|
|
5412
|
+
async setupDatabase() {
|
|
5413
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
5414
|
+
const buildKeyPath = (basePath) => {
|
|
5415
|
+
return [...prefixColumnNames, ...basePath];
|
|
5416
|
+
};
|
|
5417
|
+
const executionIndexes = [
|
|
5418
|
+
{
|
|
5419
|
+
name: "queue_executed_at",
|
|
5420
|
+
keyPath: buildKeyPath(["queue_name", "executed_at"]),
|
|
5421
|
+
options: { unique: false }
|
|
5422
|
+
}
|
|
5423
|
+
];
|
|
5424
|
+
this.executionDb = await ensureIndexedDbTable(this.executionTableName, "id", executionIndexes, this.migrationOptions);
|
|
5425
|
+
const nextAvailableIndexes = [
|
|
5426
|
+
{
|
|
5427
|
+
name: "queue_name",
|
|
5428
|
+
keyPath: buildKeyPath(["queue_name"]),
|
|
5429
|
+
options: { unique: true }
|
|
5430
|
+
}
|
|
5431
|
+
];
|
|
5432
|
+
this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
|
|
5433
|
+
}
|
|
5434
|
+
async recordExecution(queueName) {
|
|
5435
|
+
const db = await this.getExecutionDb();
|
|
5436
|
+
const tx = db.transaction(this.executionTableName, "readwrite");
|
|
5437
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5438
|
+
const record = {
|
|
5439
|
+
id: crypto.randomUUID(),
|
|
5440
|
+
queue_name: queueName,
|
|
5441
|
+
executed_at: new Date().toISOString()
|
|
5442
|
+
};
|
|
5443
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5444
|
+
record[key] = value;
|
|
5445
|
+
}
|
|
5446
|
+
return new Promise((resolve, reject) => {
|
|
5447
|
+
const request = store.add(record);
|
|
5448
|
+
tx.oncomplete = () => resolve();
|
|
5449
|
+
tx.onerror = () => reject(tx.error);
|
|
5450
|
+
request.onerror = () => reject(request.error);
|
|
5451
|
+
});
|
|
5452
|
+
}
|
|
5453
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
5454
|
+
const db = await this.getExecutionDb();
|
|
5455
|
+
const tx = db.transaction(this.executionTableName, "readonly");
|
|
5456
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5457
|
+
const index = store.index("queue_executed_at");
|
|
5458
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5459
|
+
return new Promise((resolve, reject) => {
|
|
5460
|
+
let count = 0;
|
|
5461
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartTime], [...prefixKeyValues, queueName, "\uFFFF"], true, false);
|
|
5462
|
+
const request = index.openCursor(keyRange);
|
|
5463
|
+
request.onsuccess = (event) => {
|
|
5464
|
+
const cursor = event.target.result;
|
|
5465
|
+
if (cursor) {
|
|
5466
|
+
const record = cursor.value;
|
|
5467
|
+
if (this.matchesPrefixes(record)) {
|
|
5468
|
+
count++;
|
|
5469
|
+
}
|
|
5470
|
+
cursor.continue();
|
|
5471
|
+
}
|
|
5472
|
+
};
|
|
5473
|
+
tx.oncomplete = () => resolve(count);
|
|
5474
|
+
tx.onerror = () => reject(tx.error);
|
|
5475
|
+
request.onerror = () => reject(request.error);
|
|
5476
|
+
});
|
|
5477
|
+
}
|
|
5478
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
5479
|
+
const db = await this.getExecutionDb();
|
|
5480
|
+
const tx = db.transaction(this.executionTableName, "readonly");
|
|
5481
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5482
|
+
const index = store.index("queue_executed_at");
|
|
5483
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5484
|
+
return new Promise((resolve, reject) => {
|
|
5485
|
+
const executions = [];
|
|
5486
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, "\uFFFF"]);
|
|
5487
|
+
const request = index.openCursor(keyRange);
|
|
5488
|
+
request.onsuccess = (event) => {
|
|
5489
|
+
const cursor = event.target.result;
|
|
5490
|
+
if (cursor) {
|
|
5491
|
+
const record = cursor.value;
|
|
5492
|
+
if (this.matchesPrefixes(record)) {
|
|
5493
|
+
executions.push(record.executed_at);
|
|
5494
|
+
}
|
|
5495
|
+
cursor.continue();
|
|
5496
|
+
}
|
|
5497
|
+
};
|
|
5498
|
+
tx.oncomplete = () => {
|
|
5499
|
+
executions.sort();
|
|
5500
|
+
resolve(executions[offset]);
|
|
5501
|
+
};
|
|
5502
|
+
tx.onerror = () => reject(tx.error);
|
|
5503
|
+
request.onerror = () => reject(request.error);
|
|
5504
|
+
});
|
|
5505
|
+
}
|
|
5506
|
+
async getNextAvailableTime(queueName) {
|
|
5507
|
+
const db = await this.getNextAvailableDb();
|
|
5508
|
+
const tx = db.transaction(this.nextAvailableTableName, "readonly");
|
|
5509
|
+
const store = tx.objectStore(this.nextAvailableTableName);
|
|
5510
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5511
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5512
|
+
return new Promise((resolve, reject) => {
|
|
5513
|
+
const request = store.get(key);
|
|
5514
|
+
request.onsuccess = () => {
|
|
5515
|
+
const record = request.result;
|
|
5516
|
+
if (record && this.matchesPrefixes(record)) {
|
|
5517
|
+
resolve(record.next_available_at);
|
|
5518
|
+
} else {
|
|
5519
|
+
resolve(undefined);
|
|
5520
|
+
}
|
|
5521
|
+
};
|
|
5522
|
+
request.onerror = () => reject(request.error);
|
|
5523
|
+
tx.onerror = () => reject(tx.error);
|
|
5524
|
+
});
|
|
5525
|
+
}
|
|
5526
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
5527
|
+
const db = await this.getNextAvailableDb();
|
|
5528
|
+
const tx = db.transaction(this.nextAvailableTableName, "readwrite");
|
|
5529
|
+
const store = tx.objectStore(this.nextAvailableTableName);
|
|
5530
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5531
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5532
|
+
const record = {
|
|
5533
|
+
queue_name: queueName,
|
|
5534
|
+
next_available_at: nextAvailableAt
|
|
5535
|
+
};
|
|
5536
|
+
for (const [k, value] of Object.entries(this.prefixValues)) {
|
|
5537
|
+
record[k] = value;
|
|
5538
|
+
}
|
|
5539
|
+
record[this.getPrefixColumnNames().concat(["queue_name"]).join("_")] = key;
|
|
5540
|
+
return new Promise((resolve, reject) => {
|
|
5541
|
+
const request = store.put(record);
|
|
5542
|
+
tx.oncomplete = () => resolve();
|
|
5543
|
+
tx.onerror = () => reject(tx.error);
|
|
5544
|
+
request.onerror = () => reject(request.error);
|
|
5545
|
+
});
|
|
5546
|
+
}
|
|
5547
|
+
async clear(queueName) {
|
|
5548
|
+
const execDb = await this.getExecutionDb();
|
|
5549
|
+
const execTx = execDb.transaction(this.executionTableName, "readwrite");
|
|
5550
|
+
const execStore = execTx.objectStore(this.executionTableName);
|
|
5551
|
+
const execIndex = execStore.index("queue_executed_at");
|
|
5552
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5553
|
+
await new Promise((resolve, reject) => {
|
|
5554
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, "\uFFFF"]);
|
|
5555
|
+
const request = execIndex.openCursor(keyRange);
|
|
5556
|
+
request.onsuccess = (event) => {
|
|
5557
|
+
const cursor = event.target.result;
|
|
5558
|
+
if (cursor) {
|
|
5559
|
+
const record = cursor.value;
|
|
5560
|
+
if (this.matchesPrefixes(record)) {
|
|
5561
|
+
cursor.delete();
|
|
5562
|
+
}
|
|
5563
|
+
cursor.continue();
|
|
5564
|
+
}
|
|
5565
|
+
};
|
|
5566
|
+
execTx.oncomplete = () => resolve();
|
|
5567
|
+
execTx.onerror = () => reject(execTx.error);
|
|
5568
|
+
request.onerror = () => reject(request.error);
|
|
5569
|
+
});
|
|
5570
|
+
const nextDb = await this.getNextAvailableDb();
|
|
5571
|
+
const nextTx = nextDb.transaction(this.nextAvailableTableName, "readwrite");
|
|
5572
|
+
const nextStore = nextTx.objectStore(this.nextAvailableTableName);
|
|
5573
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5574
|
+
await new Promise((resolve, reject) => {
|
|
5575
|
+
const request = nextStore.delete(key);
|
|
5576
|
+
nextTx.oncomplete = () => resolve();
|
|
5577
|
+
nextTx.onerror = () => reject(nextTx.error);
|
|
5578
|
+
request.onerror = () => reject(request.error);
|
|
5579
|
+
});
|
|
5580
|
+
}
|
|
3827
5581
|
}
|
|
3828
5582
|
export {
|
|
3829
5583
|
ensureIndexedDbTable,
|
|
@@ -3831,22 +5585,29 @@ export {
|
|
|
3831
5585
|
TabularRepository,
|
|
3832
5586
|
TABULAR_REPOSITORY,
|
|
3833
5587
|
SupabaseTabularRepository,
|
|
5588
|
+
SupabaseRateLimiterStorage,
|
|
3834
5589
|
SupabaseQueueStorage,
|
|
3835
5590
|
SupabaseKvRepository,
|
|
3836
5591
|
SqliteTabularRepository,
|
|
5592
|
+
SqliteRateLimiterStorage,
|
|
3837
5593
|
SqliteQueueStorage,
|
|
3838
5594
|
SqliteKvRepository,
|
|
3839
5595
|
SUPABASE_TABULAR_REPOSITORY,
|
|
5596
|
+
SUPABASE_RATE_LIMITER_STORAGE,
|
|
3840
5597
|
SUPABASE_QUEUE_STORAGE,
|
|
3841
5598
|
SUPABASE_KV_REPOSITORY,
|
|
3842
5599
|
SQLITE_TABULAR_REPOSITORY,
|
|
5600
|
+
SQLITE_RATE_LIMITER_STORAGE,
|
|
3843
5601
|
SQLITE_QUEUE_STORAGE,
|
|
3844
5602
|
SQLITE_KV_REPOSITORY,
|
|
5603
|
+
RATE_LIMITER_STORAGE,
|
|
3845
5604
|
QUEUE_STORAGE,
|
|
3846
5605
|
PostgresTabularRepository,
|
|
5606
|
+
PostgresRateLimiterStorage,
|
|
3847
5607
|
PostgresQueueStorage,
|
|
3848
5608
|
PostgresKvRepository,
|
|
3849
5609
|
POSTGRES_TABULAR_REPOSITORY,
|
|
5610
|
+
POSTGRES_RATE_LIMITER_STORAGE,
|
|
3850
5611
|
POSTGRES_QUEUE_STORAGE,
|
|
3851
5612
|
POSTGRES_KV_REPOSITORY,
|
|
3852
5613
|
MEMORY_TABULAR_REPOSITORY,
|
|
@@ -3856,12 +5617,16 @@ export {
|
|
|
3856
5617
|
KV_REPOSITORY,
|
|
3857
5618
|
JobStatus,
|
|
3858
5619
|
IndexedDbTabularRepository,
|
|
5620
|
+
IndexedDbRateLimiterStorage,
|
|
3859
5621
|
IndexedDbQueueStorage,
|
|
3860
5622
|
IndexedDbKvRepository,
|
|
3861
5623
|
InMemoryTabularRepository,
|
|
5624
|
+
InMemoryRateLimiterStorage,
|
|
3862
5625
|
InMemoryQueueStorage,
|
|
3863
5626
|
InMemoryKvRepository,
|
|
5627
|
+
IN_MEMORY_RATE_LIMITER_STORAGE,
|
|
3864
5628
|
IN_MEMORY_QUEUE_STORAGE,
|
|
5629
|
+
INDEXED_DB_RATE_LIMITER_STORAGE,
|
|
3865
5630
|
INDEXED_DB_QUEUE_STORAGE,
|
|
3866
5631
|
IDB_TABULAR_REPOSITORY,
|
|
3867
5632
|
IDB_KV_REPOSITORY,
|
|
@@ -3877,4 +5642,4 @@ export {
|
|
|
3877
5642
|
CACHED_TABULAR_REPOSITORY
|
|
3878
5643
|
};
|
|
3879
5644
|
|
|
3880
|
-
//# debugId=
|
|
5645
|
+
//# debugId=DBE095F06C6029D864756E2164756E21
|