@workglow/storage 0.0.56 → 0.0.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +1266 -121
- package/dist/browser.js.map +15 -10
- package/dist/bun.js +2017 -252
- package/dist/bun.js.map +19 -12
- package/dist/common-server.d.ts +4 -0
- package/dist/common-server.d.ts.map +1 -1
- package/dist/common.d.ts +2 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/limiter/IRateLimiterStorage.d.ts +81 -0
- package/dist/limiter/IRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/InMemoryRateLimiterStorage.d.ts +32 -0
- package/dist/limiter/InMemoryRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/IndexedDbRateLimiterStorage.d.ts +52 -0
- package/dist/limiter/IndexedDbRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/PostgresRateLimiterStorage.d.ts +54 -0
- package/dist/limiter/PostgresRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/SqliteRateLimiterStorage.d.ts +53 -0
- package/dist/limiter/SqliteRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/SupabaseRateLimiterStorage.d.ts +53 -0
- package/dist/limiter/SupabaseRateLimiterStorage.d.ts.map +1 -0
- package/dist/node.js +2017 -252
- package/dist/node.js.map +19 -12
- package/dist/queue/IQueueStorage.d.ts +72 -1
- package/dist/queue/IQueueStorage.d.ts.map +1 -1
- package/dist/queue/InMemoryQueueStorage.d.ts +44 -11
- package/dist/queue/InMemoryQueueStorage.d.ts.map +1 -1
- package/dist/queue/IndexedDbQueueStorage.d.ts +70 -5
- package/dist/queue/IndexedDbQueueStorage.d.ts.map +1 -1
- package/dist/queue/PostgresQueueStorage.d.ts +80 -8
- package/dist/queue/PostgresQueueStorage.d.ts.map +1 -1
- package/dist/queue/SqliteQueueStorage.d.ts +90 -34
- package/dist/queue/SqliteQueueStorage.d.ts.map +1 -1
- package/dist/queue/SupabaseQueueStorage.d.ts +98 -4
- package/dist/queue/SupabaseQueueStorage.d.ts.map +1 -1
- package/dist/tabular/ITabularRepository.d.ts +18 -0
- package/dist/tabular/ITabularRepository.d.ts.map +1 -1
- package/dist/tabular/InMemoryTabularRepository.d.ts +9 -1
- package/dist/tabular/InMemoryTabularRepository.d.ts.map +1 -1
- package/dist/tabular/SupabaseTabularRepository.d.ts +21 -1
- package/dist/tabular/SupabaseTabularRepository.d.ts.map +1 -1
- package/dist/tabular/TabularRepository.d.ts +10 -1
- package/dist/tabular/TabularRepository.d.ts.map +1 -1
- package/dist/util/PollingSubscriptionManager.d.ts +112 -0
- package/dist/util/PollingSubscriptionManager.d.ts.map +1 -0
- package/package.json +5 -5
package/dist/browser.js
CHANGED
|
@@ -106,6 +106,9 @@ class TabularRepository {
|
|
|
106
106
|
waitOn(name) {
|
|
107
107
|
return this.events.waitOn(name);
|
|
108
108
|
}
|
|
109
|
+
subscribeToChanges(_callback) {
|
|
110
|
+
throw new Error(`subscribeToChanges is not implemented for ${this.constructor.name}. ` + `Use InMemoryTabularRepository or SupabaseTabularRepository for subscription support.`);
|
|
111
|
+
}
|
|
109
112
|
primaryKeyColumns() {
|
|
110
113
|
const columns = [];
|
|
111
114
|
for (const key of Object.keys(this.primaryKeySchema.properties)) {
|
|
@@ -273,12 +276,30 @@ class InMemoryTabularRepository extends TabularRepository {
|
|
|
273
276
|
return false;
|
|
274
277
|
}
|
|
275
278
|
});
|
|
276
|
-
for (const [id,
|
|
279
|
+
for (const [id, entity] of entriesToDelete) {
|
|
277
280
|
this.values.delete(id);
|
|
281
|
+
const { key } = this.separateKeyValueFromCombined(entity);
|
|
282
|
+
this.events.emit("delete", key);
|
|
278
283
|
}
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
284
|
+
}
|
|
285
|
+
subscribeToChanges(callback) {
|
|
286
|
+
const handlePut = (entity) => {
|
|
287
|
+
callback({ type: "UPDATE", new: entity });
|
|
288
|
+
};
|
|
289
|
+
const handleDelete = (_key) => {
|
|
290
|
+
callback({ type: "DELETE" });
|
|
291
|
+
};
|
|
292
|
+
const handleClearAll = () => {
|
|
293
|
+
callback({ type: "DELETE" });
|
|
294
|
+
};
|
|
295
|
+
this.events.on("put", handlePut);
|
|
296
|
+
this.events.on("delete", handleDelete);
|
|
297
|
+
this.events.on("clearall", handleClearAll);
|
|
298
|
+
return () => {
|
|
299
|
+
this.events.off("put", handlePut);
|
|
300
|
+
this.events.off("delete", handleDelete);
|
|
301
|
+
this.events.off("clearall", handleClearAll);
|
|
302
|
+
};
|
|
282
303
|
}
|
|
283
304
|
destroy() {
|
|
284
305
|
this.values.clear();
|
|
@@ -540,7 +561,7 @@ class InMemoryKvRepository extends KvViaTabularRepository {
|
|
|
540
561
|
}
|
|
541
562
|
}
|
|
542
563
|
// src/queue/InMemoryQueueStorage.ts
|
|
543
|
-
import { createServiceToken as createServiceToken7, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
|
|
564
|
+
import { createServiceToken as createServiceToken7, EventEmitter as EventEmitter3, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
|
|
544
565
|
|
|
545
566
|
// src/queue/IQueueStorage.ts
|
|
546
567
|
import { createServiceToken as createServiceToken6 } from "@workglow/util";
|
|
@@ -560,63 +581,88 @@ var IN_MEMORY_QUEUE_STORAGE = createServiceToken7("jobqueue.storage.inMemory");
|
|
|
560
581
|
|
|
561
582
|
class InMemoryQueueStorage {
|
|
562
583
|
queueName;
|
|
563
|
-
|
|
584
|
+
prefixValues;
|
|
585
|
+
events = new EventEmitter3;
|
|
586
|
+
constructor(queueName, options) {
|
|
564
587
|
this.queueName = queueName;
|
|
565
588
|
this.jobQueue = [];
|
|
589
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
566
590
|
}
|
|
567
591
|
jobQueue;
|
|
592
|
+
matchesPrefixes(job) {
|
|
593
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
594
|
+
if (job[key] !== value) {
|
|
595
|
+
return false;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
return true;
|
|
599
|
+
}
|
|
568
600
|
pendingQueue() {
|
|
569
601
|
const now = new Date().toISOString();
|
|
570
|
-
return this.jobQueue.filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
602
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
571
603
|
}
|
|
572
604
|
async add(job) {
|
|
573
605
|
await sleep(0);
|
|
574
606
|
const now = new Date().toISOString();
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
607
|
+
const jobWithPrefixes = job;
|
|
608
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid4();
|
|
609
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid4();
|
|
610
|
+
jobWithPrefixes.queue = this.queueName;
|
|
611
|
+
jobWithPrefixes.fingerprint = await makeFingerprint4(jobWithPrefixes.input);
|
|
612
|
+
jobWithPrefixes.status = "PENDING" /* PENDING */;
|
|
613
|
+
jobWithPrefixes.progress = 0;
|
|
614
|
+
jobWithPrefixes.progress_message = "";
|
|
615
|
+
jobWithPrefixes.progress_details = null;
|
|
616
|
+
jobWithPrefixes.created_at = now;
|
|
617
|
+
jobWithPrefixes.run_after = now;
|
|
618
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
619
|
+
jobWithPrefixes[key] = value;
|
|
620
|
+
}
|
|
621
|
+
this.jobQueue.push(jobWithPrefixes);
|
|
622
|
+
this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
|
|
623
|
+
return jobWithPrefixes.id;
|
|
587
624
|
}
|
|
588
625
|
async get(id) {
|
|
589
626
|
await sleep(0);
|
|
590
|
-
|
|
627
|
+
const job = this.jobQueue.find((j) => j.id === id);
|
|
628
|
+
if (job && this.matchesPrefixes(job)) {
|
|
629
|
+
return job;
|
|
630
|
+
}
|
|
631
|
+
return;
|
|
591
632
|
}
|
|
592
633
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
593
634
|
await sleep(0);
|
|
594
635
|
num = Number(num) || 100;
|
|
595
|
-
return this.jobQueue.sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
636
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
596
637
|
}
|
|
597
|
-
async next() {
|
|
638
|
+
async next(workerId) {
|
|
598
639
|
await sleep(0);
|
|
599
640
|
const top = this.pendingQueue();
|
|
600
641
|
const job = top[0];
|
|
601
642
|
if (job) {
|
|
643
|
+
const oldJob = { ...job };
|
|
602
644
|
job.status = "PROCESSING" /* PROCESSING */;
|
|
603
645
|
job.last_ran_at = new Date().toISOString();
|
|
646
|
+
job.worker_id = workerId ?? null;
|
|
647
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
604
648
|
return job;
|
|
605
649
|
}
|
|
606
650
|
}
|
|
607
651
|
async size(status = "PENDING" /* PENDING */) {
|
|
608
652
|
await sleep(0);
|
|
609
|
-
return this.jobQueue.filter((j) => j.status === status).length;
|
|
653
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
|
|
610
654
|
}
|
|
611
655
|
async saveProgress(id, progress, message, details) {
|
|
612
656
|
await sleep(0);
|
|
613
|
-
const job = this.jobQueue.find((j) => j.id === id);
|
|
657
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
614
658
|
if (!job) {
|
|
615
659
|
throw new Error(`Job ${id} not found`);
|
|
616
660
|
}
|
|
661
|
+
const oldJob = { ...job };
|
|
617
662
|
job.progress = progress;
|
|
618
663
|
job.progress_message = message;
|
|
619
664
|
job.progress_details = details;
|
|
665
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
620
666
|
}
|
|
621
667
|
async complete(job) {
|
|
622
668
|
await sleep(0);
|
|
@@ -626,41 +672,146 @@ class InMemoryQueueStorage {
|
|
|
626
672
|
const currentAttempts = existing?.run_attempts ?? 0;
|
|
627
673
|
job.run_attempts = currentAttempts + 1;
|
|
628
674
|
this.jobQueue[index] = job;
|
|
675
|
+
this.events.emit("change", { type: "UPDATE", old: existing, new: job });
|
|
629
676
|
}
|
|
630
677
|
}
|
|
631
678
|
async abort(id) {
|
|
632
679
|
await sleep(0);
|
|
633
|
-
const job = this.jobQueue.find((j) => j.id === id);
|
|
680
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
634
681
|
if (job) {
|
|
682
|
+
const oldJob = { ...job };
|
|
635
683
|
job.status = "ABORTING" /* ABORTING */;
|
|
684
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
636
685
|
}
|
|
637
686
|
}
|
|
638
687
|
async getByRunId(runId) {
|
|
639
688
|
await sleep(0);
|
|
640
|
-
return this.jobQueue.filter((job) => job.job_run_id === runId);
|
|
689
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
|
|
641
690
|
}
|
|
642
691
|
async deleteAll() {
|
|
643
692
|
await sleep(0);
|
|
644
|
-
this.jobQueue
|
|
693
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
|
|
694
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
|
|
695
|
+
for (const job of deletedJobs) {
|
|
696
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
697
|
+
}
|
|
645
698
|
}
|
|
646
699
|
async outputForInput(input) {
|
|
647
700
|
await sleep(0);
|
|
648
701
|
const fingerprint = await makeFingerprint4(input);
|
|
649
|
-
return this.jobQueue.find((j) => j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
|
|
702
|
+
return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
|
|
650
703
|
}
|
|
651
704
|
async delete(id) {
|
|
652
705
|
await sleep(0);
|
|
653
|
-
|
|
706
|
+
const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
|
|
707
|
+
this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
|
|
708
|
+
if (deletedJob) {
|
|
709
|
+
this.events.emit("change", { type: "DELETE", old: deletedJob });
|
|
710
|
+
}
|
|
654
711
|
}
|
|
655
712
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
656
713
|
await sleep(0);
|
|
657
714
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
658
|
-
|
|
715
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
|
|
716
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
|
|
717
|
+
for (const job of deletedJobs) {
|
|
718
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
719
|
+
}
|
|
659
720
|
}
|
|
660
721
|
async setupDatabase() {}
|
|
722
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
723
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
724
|
+
return true;
|
|
725
|
+
}
|
|
726
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
727
|
+
if (Object.keys(filterValues).length === 0) {
|
|
728
|
+
return true;
|
|
729
|
+
}
|
|
730
|
+
const jobWithPrefixes = job;
|
|
731
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
732
|
+
if (jobWithPrefixes[key] !== value) {
|
|
733
|
+
return false;
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
return true;
|
|
737
|
+
}
|
|
738
|
+
subscribeToChanges(callback, options) {
|
|
739
|
+
const prefixFilter = options?.prefixFilter;
|
|
740
|
+
const filteredCallback = (change) => {
|
|
741
|
+
const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
|
|
742
|
+
const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
|
|
743
|
+
if (!newMatches && !oldMatches) {
|
|
744
|
+
return;
|
|
745
|
+
}
|
|
746
|
+
callback(change);
|
|
747
|
+
};
|
|
748
|
+
return this.events.subscribe("change", filteredCallback);
|
|
749
|
+
}
|
|
661
750
|
}
|
|
662
|
-
// src/
|
|
751
|
+
// src/limiter/IRateLimiterStorage.ts
|
|
663
752
|
import { createServiceToken as createServiceToken8 } from "@workglow/util";
|
|
753
|
+
var RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage");
|
|
754
|
+
// src/limiter/InMemoryRateLimiterStorage.ts
|
|
755
|
+
import { createServiceToken as createServiceToken9, sleep as sleep2 } from "@workglow/util";
|
|
756
|
+
var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken9("ratelimiter.storage.inMemory");
|
|
757
|
+
|
|
758
|
+
class InMemoryRateLimiterStorage {
|
|
759
|
+
prefixValues;
|
|
760
|
+
executions = new Map;
|
|
761
|
+
nextAvailableTimes = new Map;
|
|
762
|
+
constructor(options) {
|
|
763
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
764
|
+
}
|
|
765
|
+
makeKey(queueName) {
|
|
766
|
+
const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
|
|
767
|
+
return prefixPart ? `${prefixPart}|${queueName}` : queueName;
|
|
768
|
+
}
|
|
769
|
+
async setupDatabase() {}
|
|
770
|
+
async recordExecution(queueName) {
|
|
771
|
+
await sleep2(0);
|
|
772
|
+
const key = this.makeKey(queueName);
|
|
773
|
+
const executions = this.executions.get(key) ?? [];
|
|
774
|
+
executions.push({
|
|
775
|
+
queueName,
|
|
776
|
+
executedAt: new Date
|
|
777
|
+
});
|
|
778
|
+
this.executions.set(key, executions);
|
|
779
|
+
}
|
|
780
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
781
|
+
await sleep2(0);
|
|
782
|
+
const key = this.makeKey(queueName);
|
|
783
|
+
const executions = this.executions.get(key) ?? [];
|
|
784
|
+
const windowStart = new Date(windowStartTime);
|
|
785
|
+
return executions.filter((e) => e.executedAt > windowStart).length;
|
|
786
|
+
}
|
|
787
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
788
|
+
await sleep2(0);
|
|
789
|
+
const key = this.makeKey(queueName);
|
|
790
|
+
const executions = this.executions.get(key) ?? [];
|
|
791
|
+
const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
|
|
792
|
+
const execution = sorted[offset];
|
|
793
|
+
return execution?.executedAt.toISOString();
|
|
794
|
+
}
|
|
795
|
+
async getNextAvailableTime(queueName) {
|
|
796
|
+
await sleep2(0);
|
|
797
|
+
const key = this.makeKey(queueName);
|
|
798
|
+
const time = this.nextAvailableTimes.get(key);
|
|
799
|
+
return time?.toISOString();
|
|
800
|
+
}
|
|
801
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
802
|
+
await sleep2(0);
|
|
803
|
+
const key = this.makeKey(queueName);
|
|
804
|
+
this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
|
|
805
|
+
}
|
|
806
|
+
async clear(queueName) {
|
|
807
|
+
await sleep2(0);
|
|
808
|
+
const key = this.makeKey(queueName);
|
|
809
|
+
this.executions.delete(key);
|
|
810
|
+
this.nextAvailableTimes.delete(key);
|
|
811
|
+
}
|
|
812
|
+
}
|
|
813
|
+
// src/tabular/IndexedDbTabularRepository.ts
|
|
814
|
+
import { createServiceToken as createServiceToken10 } from "@workglow/util";
|
|
664
815
|
|
|
665
816
|
// src/util/IndexedDbTable.ts
|
|
666
817
|
var METADATA_STORE_NAME = "__schema_metadata__";
|
|
@@ -1000,7 +1151,7 @@ async function dropIndexedDbTable(tableName) {
|
|
|
1000
1151
|
}
|
|
1001
1152
|
|
|
1002
1153
|
// src/tabular/IndexedDbTabularRepository.ts
|
|
1003
|
-
var IDB_TABULAR_REPOSITORY =
|
|
1154
|
+
var IDB_TABULAR_REPOSITORY = createServiceToken10("storage.tabularRepository.indexedDb");
|
|
1004
1155
|
|
|
1005
1156
|
class IndexedDbTabularRepository extends TabularRepository {
|
|
1006
1157
|
table;
|
|
@@ -1335,8 +1486,8 @@ class IndexedDbTabularRepository extends TabularRepository {
|
|
|
1335
1486
|
}
|
|
1336
1487
|
}
|
|
1337
1488
|
// src/tabular/SharedInMemoryTabularRepository.ts
|
|
1338
|
-
import { createServiceToken as
|
|
1339
|
-
var SHARED_IN_MEMORY_TABULAR_REPOSITORY =
|
|
1489
|
+
import { createServiceToken as createServiceToken11 } from "@workglow/util";
|
|
1490
|
+
var SHARED_IN_MEMORY_TABULAR_REPOSITORY = createServiceToken11("storage.tabularRepository.sharedInMemory");
|
|
1340
1491
|
|
|
1341
1492
|
class SharedInMemoryTabularRepository extends TabularRepository {
|
|
1342
1493
|
channel = null;
|
|
@@ -1506,7 +1657,7 @@ class SharedInMemoryTabularRepository extends TabularRepository {
|
|
|
1506
1657
|
}
|
|
1507
1658
|
}
|
|
1508
1659
|
// src/tabular/SupabaseTabularRepository.ts
|
|
1509
|
-
import { createServiceToken as
|
|
1660
|
+
import { createServiceToken as createServiceToken12 } from "@workglow/util";
|
|
1510
1661
|
|
|
1511
1662
|
// src/tabular/BaseSqlTabularRepository.ts
|
|
1512
1663
|
class BaseSqlTabularRepository extends TabularRepository {
|
|
@@ -1692,10 +1843,11 @@ class BaseSqlTabularRepository extends TabularRepository {
|
|
|
1692
1843
|
}
|
|
1693
1844
|
|
|
1694
1845
|
// src/tabular/SupabaseTabularRepository.ts
|
|
1695
|
-
var SUPABASE_TABULAR_REPOSITORY =
|
|
1846
|
+
var SUPABASE_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.supabase");
|
|
1696
1847
|
|
|
1697
1848
|
class SupabaseTabularRepository extends BaseSqlTabularRepository {
|
|
1698
1849
|
client;
|
|
1850
|
+
realtimeChannel = null;
|
|
1699
1851
|
constructor(client, table = "tabular_store", schema, primaryKeyNames, indexes = []) {
|
|
1700
1852
|
super(table, schema, primaryKeyNames, indexes);
|
|
1701
1853
|
this.client = client;
|
|
@@ -2068,10 +2220,44 @@ class SupabaseTabularRepository extends BaseSqlTabularRepository {
|
|
|
2068
2220
|
throw error;
|
|
2069
2221
|
this.events.emit("delete", column);
|
|
2070
2222
|
}
|
|
2223
|
+
convertRealtimeRow(row) {
|
|
2224
|
+
const entity = { ...row };
|
|
2225
|
+
for (const key in this.schema.properties) {
|
|
2226
|
+
entity[key] = this.sqlToJsValue(key, row[key]);
|
|
2227
|
+
}
|
|
2228
|
+
return entity;
|
|
2229
|
+
}
|
|
2230
|
+
subscribeToChanges(callback) {
|
|
2231
|
+
const channelName = `tabular-${this.table}-${Date.now()}`;
|
|
2232
|
+
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
2233
|
+
event: "*",
|
|
2234
|
+
schema: "public",
|
|
2235
|
+
table: this.table
|
|
2236
|
+
}, (payload) => {
|
|
2237
|
+
const change = {
|
|
2238
|
+
type: payload.eventType.toUpperCase(),
|
|
2239
|
+
old: payload.old && Object.keys(payload.old).length > 0 ? this.convertRealtimeRow(payload.old) : undefined,
|
|
2240
|
+
new: payload.new && Object.keys(payload.new).length > 0 ? this.convertRealtimeRow(payload.new) : undefined
|
|
2241
|
+
};
|
|
2242
|
+
callback(change);
|
|
2243
|
+
}).subscribe();
|
|
2244
|
+
return () => {
|
|
2245
|
+
if (this.realtimeChannel) {
|
|
2246
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
2247
|
+
this.realtimeChannel = null;
|
|
2248
|
+
}
|
|
2249
|
+
};
|
|
2250
|
+
}
|
|
2251
|
+
destroy() {
|
|
2252
|
+
if (this.realtimeChannel) {
|
|
2253
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
2254
|
+
this.realtimeChannel = null;
|
|
2255
|
+
}
|
|
2256
|
+
}
|
|
2071
2257
|
}
|
|
2072
2258
|
// src/kv/IndexedDbKvRepository.ts
|
|
2073
|
-
import { createServiceToken as
|
|
2074
|
-
var IDB_KV_REPOSITORY =
|
|
2259
|
+
import { createServiceToken as createServiceToken13 } from "@workglow/util";
|
|
2260
|
+
var IDB_KV_REPOSITORY = createServiceToken13("storage.kvRepository.indexedDb");
|
|
2075
2261
|
|
|
2076
2262
|
class IndexedDbKvRepository extends KvViaTabularRepository {
|
|
2077
2263
|
dbName;
|
|
@@ -2083,8 +2269,8 @@ class IndexedDbKvRepository extends KvViaTabularRepository {
|
|
|
2083
2269
|
}
|
|
2084
2270
|
}
|
|
2085
2271
|
// src/kv/SupabaseKvRepository.ts
|
|
2086
|
-
import { createServiceToken as
|
|
2087
|
-
var SUPABASE_KV_REPOSITORY =
|
|
2272
|
+
import { createServiceToken as createServiceToken14 } from "@workglow/util";
|
|
2273
|
+
var SUPABASE_KV_REPOSITORY = createServiceToken14("storage.kvRepository.supabase");
|
|
2088
2274
|
|
|
2089
2275
|
class SupabaseKvRepository extends KvViaTabularRepository {
|
|
2090
2276
|
client;
|
|
@@ -2098,18 +2284,160 @@ class SupabaseKvRepository extends KvViaTabularRepository {
|
|
|
2098
2284
|
}
|
|
2099
2285
|
}
|
|
2100
2286
|
// src/queue/IndexedDbQueueStorage.ts
|
|
2101
|
-
import { createServiceToken as
|
|
2102
|
-
|
|
2287
|
+
import { createServiceToken as createServiceToken15, makeFingerprint as makeFingerprint5, uuid4 as uuid42 } from "@workglow/util";
|
|
2288
|
+
|
|
2289
|
+
// src/util/PollingSubscriptionManager.ts
|
|
2290
|
+
class PollingSubscriptionManager {
|
|
2291
|
+
intervals = new Map;
|
|
2292
|
+
lastKnownState = new Map;
|
|
2293
|
+
initialized = false;
|
|
2294
|
+
fetchState;
|
|
2295
|
+
compareItems;
|
|
2296
|
+
payloadFactory;
|
|
2297
|
+
defaultIntervalMs;
|
|
2298
|
+
constructor(fetchState, compareItems, payloadFactory, options) {
|
|
2299
|
+
this.fetchState = fetchState;
|
|
2300
|
+
this.compareItems = compareItems;
|
|
2301
|
+
this.payloadFactory = payloadFactory;
|
|
2302
|
+
this.defaultIntervalMs = options?.defaultIntervalMs ?? 1000;
|
|
2303
|
+
}
|
|
2304
|
+
subscribe(callback, options) {
|
|
2305
|
+
const interval = options?.intervalMs ?? this.defaultIntervalMs;
|
|
2306
|
+
const subscription = {
|
|
2307
|
+
callback,
|
|
2308
|
+
intervalMs: interval
|
|
2309
|
+
};
|
|
2310
|
+
let intervalGroup = this.intervals.get(interval);
|
|
2311
|
+
if (!intervalGroup) {
|
|
2312
|
+
const subscribers = new Set;
|
|
2313
|
+
const intervalId = setInterval(() => this.poll(subscribers), interval);
|
|
2314
|
+
intervalGroup = { intervalId, subscribers };
|
|
2315
|
+
this.intervals.set(interval, intervalGroup);
|
|
2316
|
+
if (!this.initialized) {
|
|
2317
|
+
this.initialized = true;
|
|
2318
|
+
this.initAndPoll(subscribers, subscription);
|
|
2319
|
+
} else {
|
|
2320
|
+
this.pollForNewSubscriber(subscription);
|
|
2321
|
+
}
|
|
2322
|
+
} else {
|
|
2323
|
+
this.pollForNewSubscriber(subscription);
|
|
2324
|
+
}
|
|
2325
|
+
intervalGroup.subscribers.add(subscription);
|
|
2326
|
+
return () => {
|
|
2327
|
+
const group = this.intervals.get(interval);
|
|
2328
|
+
if (group) {
|
|
2329
|
+
group.subscribers.delete(subscription);
|
|
2330
|
+
if (group.subscribers.size === 0) {
|
|
2331
|
+
clearInterval(group.intervalId);
|
|
2332
|
+
this.intervals.delete(interval);
|
|
2333
|
+
}
|
|
2334
|
+
}
|
|
2335
|
+
};
|
|
2336
|
+
}
|
|
2337
|
+
async initAndPoll(subscribers, newSubscription) {
|
|
2338
|
+
try {
|
|
2339
|
+
this.lastKnownState = await this.fetchState();
|
|
2340
|
+
for (const [, item] of this.lastKnownState) {
|
|
2341
|
+
const payload = this.payloadFactory.insert(item);
|
|
2342
|
+
try {
|
|
2343
|
+
newSubscription.callback(payload);
|
|
2344
|
+
} catch {}
|
|
2345
|
+
}
|
|
2346
|
+
} catch {}
|
|
2347
|
+
}
|
|
2348
|
+
pollForNewSubscriber(subscription) {
|
|
2349
|
+
for (const [, item] of this.lastKnownState) {
|
|
2350
|
+
const payload = this.payloadFactory.insert(item);
|
|
2351
|
+
try {
|
|
2352
|
+
subscription.callback(payload);
|
|
2353
|
+
} catch {}
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
async poll(subscribers) {
|
|
2357
|
+
if (subscribers.size === 0)
|
|
2358
|
+
return;
|
|
2359
|
+
try {
|
|
2360
|
+
const currentState = await this.fetchState();
|
|
2361
|
+
const changes = [];
|
|
2362
|
+
for (const [key, item] of currentState) {
|
|
2363
|
+
const oldItem = this.lastKnownState.get(key);
|
|
2364
|
+
if (!oldItem) {
|
|
2365
|
+
changes.push(this.payloadFactory.insert(item));
|
|
2366
|
+
} else if (!this.compareItems(oldItem, item)) {
|
|
2367
|
+
changes.push(this.payloadFactory.update(oldItem, item));
|
|
2368
|
+
}
|
|
2369
|
+
}
|
|
2370
|
+
for (const [key, item] of this.lastKnownState) {
|
|
2371
|
+
if (!currentState.has(key)) {
|
|
2372
|
+
changes.push(this.payloadFactory.delete(item));
|
|
2373
|
+
}
|
|
2374
|
+
}
|
|
2375
|
+
this.lastKnownState = currentState;
|
|
2376
|
+
for (const change of changes) {
|
|
2377
|
+
for (const sub of subscribers) {
|
|
2378
|
+
try {
|
|
2379
|
+
sub.callback(change);
|
|
2380
|
+
} catch {}
|
|
2381
|
+
}
|
|
2382
|
+
}
|
|
2383
|
+
} catch {}
|
|
2384
|
+
}
|
|
2385
|
+
get subscriptionCount() {
|
|
2386
|
+
let count = 0;
|
|
2387
|
+
for (const group of this.intervals.values()) {
|
|
2388
|
+
count += group.subscribers.size;
|
|
2389
|
+
}
|
|
2390
|
+
return count;
|
|
2391
|
+
}
|
|
2392
|
+
get hasSubscriptions() {
|
|
2393
|
+
return this.intervals.size > 0;
|
|
2394
|
+
}
|
|
2395
|
+
destroy() {
|
|
2396
|
+
for (const group of this.intervals.values()) {
|
|
2397
|
+
clearInterval(group.intervalId);
|
|
2398
|
+
}
|
|
2399
|
+
this.intervals.clear();
|
|
2400
|
+
this.lastKnownState.clear();
|
|
2401
|
+
this.initialized = false;
|
|
2402
|
+
}
|
|
2403
|
+
}
|
|
2404
|
+
|
|
2405
|
+
// src/queue/IndexedDbQueueStorage.ts
|
|
2406
|
+
var INDEXED_DB_QUEUE_STORAGE = createServiceToken15("jobqueue.storage.indexedDb");
|
|
2103
2407
|
|
|
2104
2408
|
class IndexedDbQueueStorage {
|
|
2105
2409
|
queueName;
|
|
2106
2410
|
db;
|
|
2107
2411
|
tableName;
|
|
2108
2412
|
migrationOptions;
|
|
2109
|
-
|
|
2413
|
+
prefixes;
|
|
2414
|
+
prefixValues;
|
|
2415
|
+
pollingManager = null;
|
|
2416
|
+
constructor(queueName, options = {}) {
|
|
2110
2417
|
this.queueName = queueName;
|
|
2111
|
-
this.
|
|
2112
|
-
this.
|
|
2418
|
+
this.migrationOptions = options;
|
|
2419
|
+
this.prefixes = options.prefixes ?? [];
|
|
2420
|
+
this.prefixValues = options.prefixValues ?? {};
|
|
2421
|
+
if (this.prefixes.length > 0) {
|
|
2422
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
2423
|
+
this.tableName = `jobs_${prefixNames}`;
|
|
2424
|
+
} else {
|
|
2425
|
+
this.tableName = "jobs";
|
|
2426
|
+
}
|
|
2427
|
+
}
|
|
2428
|
+
getPrefixColumnNames() {
|
|
2429
|
+
return this.prefixes.map((p) => p.name);
|
|
2430
|
+
}
|
|
2431
|
+
matchesPrefixes(job) {
|
|
2432
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
2433
|
+
if (job[key] !== value) {
|
|
2434
|
+
return false;
|
|
2435
|
+
}
|
|
2436
|
+
}
|
|
2437
|
+
return true;
|
|
2438
|
+
}
|
|
2439
|
+
getPrefixKeyValues() {
|
|
2440
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2113
2441
|
}
|
|
2114
2442
|
async getDb() {
|
|
2115
2443
|
if (this.db)
|
|
@@ -2118,25 +2446,29 @@ class IndexedDbQueueStorage {
|
|
|
2118
2446
|
return this.db;
|
|
2119
2447
|
}
|
|
2120
2448
|
async setupDatabase() {
|
|
2449
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2450
|
+
const buildKeyPath = (basePath) => {
|
|
2451
|
+
return [...prefixColumnNames, ...basePath];
|
|
2452
|
+
};
|
|
2121
2453
|
const expectedIndexes = [
|
|
2122
2454
|
{
|
|
2123
|
-
name: "
|
|
2124
|
-
keyPath:
|
|
2455
|
+
name: "queue_status",
|
|
2456
|
+
keyPath: buildKeyPath(["queue", "status"]),
|
|
2125
2457
|
options: { unique: false }
|
|
2126
2458
|
},
|
|
2127
2459
|
{
|
|
2128
|
-
name: "
|
|
2129
|
-
keyPath: ["status", "run_after"],
|
|
2460
|
+
name: "queue_status_run_after",
|
|
2461
|
+
keyPath: buildKeyPath(["queue", "status", "run_after"]),
|
|
2130
2462
|
options: { unique: false }
|
|
2131
2463
|
},
|
|
2132
2464
|
{
|
|
2133
|
-
name: "
|
|
2134
|
-
keyPath:
|
|
2465
|
+
name: "queue_job_run_id",
|
|
2466
|
+
keyPath: buildKeyPath(["queue", "job_run_id"]),
|
|
2135
2467
|
options: { unique: false }
|
|
2136
2468
|
},
|
|
2137
2469
|
{
|
|
2138
|
-
name: "
|
|
2139
|
-
keyPath: ["fingerprint", "status"],
|
|
2470
|
+
name: "queue_fingerprint_status",
|
|
2471
|
+
keyPath: buildKeyPath(["queue", "fingerprint", "status"]),
|
|
2140
2472
|
options: { unique: false }
|
|
2141
2473
|
}
|
|
2142
2474
|
];
|
|
@@ -2145,21 +2477,25 @@ class IndexedDbQueueStorage {
|
|
|
2145
2477
|
async add(job) {
|
|
2146
2478
|
const db = await this.getDb();
|
|
2147
2479
|
const now = new Date().toISOString();
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2480
|
+
const jobWithPrefixes = job;
|
|
2481
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid42();
|
|
2482
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid42();
|
|
2483
|
+
jobWithPrefixes.queue = this.queueName;
|
|
2484
|
+
jobWithPrefixes.fingerprint = await makeFingerprint5(jobWithPrefixes.input);
|
|
2485
|
+
jobWithPrefixes.status = "PENDING" /* PENDING */;
|
|
2486
|
+
jobWithPrefixes.progress = 0;
|
|
2487
|
+
jobWithPrefixes.progress_message = "";
|
|
2488
|
+
jobWithPrefixes.progress_details = null;
|
|
2489
|
+
jobWithPrefixes.created_at = now;
|
|
2490
|
+
jobWithPrefixes.run_after = now;
|
|
2491
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
2492
|
+
jobWithPrefixes[key] = value;
|
|
2493
|
+
}
|
|
2158
2494
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
2159
2495
|
const store = tx.objectStore(this.tableName);
|
|
2160
2496
|
return new Promise((resolve, reject) => {
|
|
2161
|
-
const request = store.add(
|
|
2162
|
-
tx.oncomplete = () => resolve(
|
|
2497
|
+
const request = store.add(jobWithPrefixes);
|
|
2498
|
+
tx.oncomplete = () => resolve(jobWithPrefixes.id);
|
|
2163
2499
|
tx.onerror = () => reject(tx.error);
|
|
2164
2500
|
request.onerror = () => reject(request.error);
|
|
2165
2501
|
});
|
|
@@ -2170,7 +2506,14 @@ class IndexedDbQueueStorage {
|
|
|
2170
2506
|
const store = tx.objectStore(this.tableName);
|
|
2171
2507
|
const request = store.get(id);
|
|
2172
2508
|
return new Promise((resolve, reject) => {
|
|
2173
|
-
request.onsuccess = () =>
|
|
2509
|
+
request.onsuccess = () => {
|
|
2510
|
+
const job = request.result;
|
|
2511
|
+
if (job && job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
2512
|
+
resolve(job);
|
|
2513
|
+
} else {
|
|
2514
|
+
resolve(undefined);
|
|
2515
|
+
}
|
|
2516
|
+
};
|
|
2174
2517
|
request.onerror = () => reject(request.error);
|
|
2175
2518
|
tx.onerror = () => reject(tx.error);
|
|
2176
2519
|
});
|
|
@@ -2179,10 +2522,11 @@ class IndexedDbQueueStorage {
|
|
|
2179
2522
|
const db = await this.getDb();
|
|
2180
2523
|
const tx = db.transaction(this.tableName, "readonly");
|
|
2181
2524
|
const store = tx.objectStore(this.tableName);
|
|
2182
|
-
const index = store.index("
|
|
2525
|
+
const index = store.index("queue_status_run_after");
|
|
2526
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
2183
2527
|
return new Promise((resolve, reject) => {
|
|
2184
2528
|
const ret = new Map;
|
|
2185
|
-
const keyRange = IDBKeyRange.bound([status, ""], [status, ""]);
|
|
2529
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, status, ""], [...prefixKeyValues, this.queueName, status, ""]);
|
|
2186
2530
|
const cursorRequest = index.openCursor(keyRange);
|
|
2187
2531
|
const handleCursor = (e) => {
|
|
2188
2532
|
const cursor = e.target.result;
|
|
@@ -2190,7 +2534,10 @@ class IndexedDbQueueStorage {
|
|
|
2190
2534
|
resolve(Array.from(ret.values()));
|
|
2191
2535
|
return;
|
|
2192
2536
|
}
|
|
2193
|
-
|
|
2537
|
+
const job = cursor.value;
|
|
2538
|
+
if (this.matchesPrefixes(job)) {
|
|
2539
|
+
ret.set(cursor.value.id, cursor.value);
|
|
2540
|
+
}
|
|
2194
2541
|
cursor.continue();
|
|
2195
2542
|
};
|
|
2196
2543
|
cursorRequest.onsuccess = handleCursor;
|
|
@@ -2198,14 +2545,15 @@ class IndexedDbQueueStorage {
|
|
|
2198
2545
|
tx.onerror = () => reject(tx.error);
|
|
2199
2546
|
});
|
|
2200
2547
|
}
|
|
2201
|
-
async next() {
|
|
2548
|
+
async next(workerId) {
|
|
2202
2549
|
const db = await this.getDb();
|
|
2203
2550
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
2204
2551
|
const store = tx.objectStore(this.tableName);
|
|
2205
|
-
const index = store.index("
|
|
2552
|
+
const index = store.index("queue_status_run_after");
|
|
2206
2553
|
const now = new Date().toISOString();
|
|
2554
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
2207
2555
|
return new Promise((resolve, reject) => {
|
|
2208
|
-
const cursorRequest = index.openCursor(IDBKeyRange.bound(["PENDING" /* PENDING */, ""], ["PENDING" /* PENDING */, now], false, true));
|
|
2556
|
+
const cursorRequest = index.openCursor(IDBKeyRange.bound([...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, ""], [...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, now], false, true));
|
|
2209
2557
|
let jobToReturn;
|
|
2210
2558
|
cursorRequest.onsuccess = (e) => {
|
|
2211
2559
|
const cursor = e.target.result;
|
|
@@ -2218,12 +2566,13 @@ class IndexedDbQueueStorage {
|
|
|
2218
2566
|
return;
|
|
2219
2567
|
}
|
|
2220
2568
|
const job = cursor.value;
|
|
2221
|
-
if (job.status !== "PENDING" /* PENDING */) {
|
|
2569
|
+
if (job.queue !== this.queueName || job.status !== "PENDING" /* PENDING */ || !this.matchesPrefixes(job)) {
|
|
2222
2570
|
cursor.continue();
|
|
2223
2571
|
return;
|
|
2224
2572
|
}
|
|
2225
2573
|
job.status = "PROCESSING" /* PROCESSING */;
|
|
2226
2574
|
job.last_ran_at = now;
|
|
2575
|
+
job.worker_id = workerId ?? null;
|
|
2227
2576
|
try {
|
|
2228
2577
|
const updateRequest = store.put(job);
|
|
2229
2578
|
updateRequest.onsuccess = () => {
|
|
@@ -2247,11 +2596,13 @@ class IndexedDbQueueStorage {
|
|
|
2247
2596
|
}
|
|
2248
2597
|
async size(status = "PENDING" /* PENDING */) {
|
|
2249
2598
|
const db = await this.getDb();
|
|
2599
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
2250
2600
|
return new Promise((resolve, reject) => {
|
|
2251
2601
|
const tx = db.transaction(this.tableName, "readonly");
|
|
2252
2602
|
const store = tx.objectStore(this.tableName);
|
|
2253
|
-
const index = store.index("
|
|
2254
|
-
const
|
|
2603
|
+
const index = store.index("queue_status");
|
|
2604
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
2605
|
+
const request = index.count(keyRange);
|
|
2255
2606
|
request.onsuccess = () => resolve(request.result);
|
|
2256
2607
|
request.onerror = () => reject(request.error);
|
|
2257
2608
|
tx.onerror = () => reject(tx.error);
|
|
@@ -2265,9 +2616,18 @@ class IndexedDbQueueStorage {
|
|
|
2265
2616
|
const getReq = store.get(job.id);
|
|
2266
2617
|
getReq.onsuccess = () => {
|
|
2267
2618
|
const existing = getReq.result;
|
|
2268
|
-
|
|
2619
|
+
if (!existing || existing.queue !== this.queueName || !this.matchesPrefixes(existing)) {
|
|
2620
|
+
reject(new Error(`Job ${job.id} not found or does not belong to queue ${this.queueName}`));
|
|
2621
|
+
return;
|
|
2622
|
+
}
|
|
2623
|
+
const currentAttempts = existing.run_attempts ?? 0;
|
|
2269
2624
|
job.run_attempts = currentAttempts + 1;
|
|
2270
|
-
|
|
2625
|
+
job.queue = this.queueName;
|
|
2626
|
+
const jobWithPrefixes = job;
|
|
2627
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
2628
|
+
jobWithPrefixes[key] = value;
|
|
2629
|
+
}
|
|
2630
|
+
const putReq = store.put(jobWithPrefixes);
|
|
2271
2631
|
putReq.onsuccess = () => {};
|
|
2272
2632
|
putReq.onerror = () => reject(putReq.error);
|
|
2273
2633
|
};
|
|
@@ -2287,10 +2647,15 @@ class IndexedDbQueueStorage {
|
|
|
2287
2647
|
const db = await this.getDb();
|
|
2288
2648
|
const tx = db.transaction(this.tableName, "readonly");
|
|
2289
2649
|
const store = tx.objectStore(this.tableName);
|
|
2290
|
-
const index = store.index("
|
|
2291
|
-
const
|
|
2650
|
+
const index = store.index("queue_job_run_id");
|
|
2651
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
2652
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, job_run_id]);
|
|
2653
|
+
const request = index.getAll(keyRange);
|
|
2292
2654
|
return new Promise((resolve, reject) => {
|
|
2293
|
-
request.onsuccess = () =>
|
|
2655
|
+
request.onsuccess = () => {
|
|
2656
|
+
const results = (request.result || []).filter((job) => this.matchesPrefixes(job));
|
|
2657
|
+
resolve(results);
|
|
2658
|
+
};
|
|
2294
2659
|
request.onerror = () => reject(request.error);
|
|
2295
2660
|
tx.onerror = () => reject(tx.error);
|
|
2296
2661
|
});
|
|
@@ -2299,11 +2664,31 @@ class IndexedDbQueueStorage {
|
|
|
2299
2664
|
const db = await this.getDb();
|
|
2300
2665
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
2301
2666
|
const store = tx.objectStore(this.tableName);
|
|
2302
|
-
const
|
|
2667
|
+
const index = store.index("queue_status");
|
|
2668
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
2303
2669
|
return new Promise((resolve, reject) => {
|
|
2304
|
-
|
|
2305
|
-
request
|
|
2670
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, ""]);
|
|
2671
|
+
const request = index.openCursor(keyRange);
|
|
2672
|
+
request.onsuccess = (event) => {
|
|
2673
|
+
const cursor = event.target.result;
|
|
2674
|
+
if (cursor) {
|
|
2675
|
+
const job = cursor.value;
|
|
2676
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
2677
|
+
const deleteRequest = cursor.delete();
|
|
2678
|
+
deleteRequest.onsuccess = () => {
|
|
2679
|
+
cursor.continue();
|
|
2680
|
+
};
|
|
2681
|
+
deleteRequest.onerror = () => {
|
|
2682
|
+
cursor.continue();
|
|
2683
|
+
};
|
|
2684
|
+
} else {
|
|
2685
|
+
cursor.continue();
|
|
2686
|
+
}
|
|
2687
|
+
}
|
|
2688
|
+
};
|
|
2689
|
+
tx.oncomplete = () => resolve();
|
|
2306
2690
|
tx.onerror = () => reject(tx.error);
|
|
2691
|
+
request.onerror = () => reject(request.error);
|
|
2307
2692
|
});
|
|
2308
2693
|
}
|
|
2309
2694
|
async outputForInput(input) {
|
|
@@ -2311,10 +2696,23 @@ class IndexedDbQueueStorage {
|
|
|
2311
2696
|
const db = await this.getDb();
|
|
2312
2697
|
const tx = db.transaction(this.tableName, "readonly");
|
|
2313
2698
|
const store = tx.objectStore(this.tableName);
|
|
2314
|
-
const index = store.index("
|
|
2315
|
-
const
|
|
2699
|
+
const index = store.index("queue_fingerprint_status");
|
|
2700
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
2701
|
+
const request = index.get([
|
|
2702
|
+
...prefixKeyValues,
|
|
2703
|
+
this.queueName,
|
|
2704
|
+
fingerprint,
|
|
2705
|
+
"COMPLETED" /* COMPLETED */
|
|
2706
|
+
]);
|
|
2316
2707
|
return new Promise((resolve, reject) => {
|
|
2317
|
-
request.onsuccess = () =>
|
|
2708
|
+
request.onsuccess = () => {
|
|
2709
|
+
const job = request.result;
|
|
2710
|
+
if (job && this.matchesPrefixes(job)) {
|
|
2711
|
+
resolve(job.output ?? null);
|
|
2712
|
+
} else {
|
|
2713
|
+
resolve(null);
|
|
2714
|
+
}
|
|
2715
|
+
};
|
|
2318
2716
|
request.onerror = () => reject(request.error);
|
|
2319
2717
|
tx.onerror = () => reject(tx.error);
|
|
2320
2718
|
});
|
|
@@ -2329,6 +2727,9 @@ class IndexedDbQueueStorage {
|
|
|
2329
2727
|
await this.complete(job);
|
|
2330
2728
|
}
|
|
2331
2729
|
async delete(id) {
|
|
2730
|
+
const job = await this.get(id);
|
|
2731
|
+
if (!job)
|
|
2732
|
+
return;
|
|
2332
2733
|
const db = await this.getDb();
|
|
2333
2734
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
2334
2735
|
const store = tx.objectStore(this.tableName);
|
|
@@ -2343,15 +2744,17 @@ class IndexedDbQueueStorage {
|
|
|
2343
2744
|
const db = await this.getDb();
|
|
2344
2745
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
2345
2746
|
const store = tx.objectStore(this.tableName);
|
|
2346
|
-
const index = store.index("
|
|
2747
|
+
const index = store.index("queue_status");
|
|
2347
2748
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
2749
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
2750
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
2348
2751
|
return new Promise((resolve, reject) => {
|
|
2349
|
-
const request = index.openCursor();
|
|
2752
|
+
const request = index.openCursor(keyRange);
|
|
2350
2753
|
request.onsuccess = (event) => {
|
|
2351
2754
|
const cursor = event.target.result;
|
|
2352
2755
|
if (cursor) {
|
|
2353
2756
|
const job = cursor.value;
|
|
2354
|
-
if (job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
|
|
2757
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
|
|
2355
2758
|
cursor.delete();
|
|
2356
2759
|
}
|
|
2357
2760
|
cursor.continue();
|
|
@@ -2362,17 +2765,193 @@ class IndexedDbQueueStorage {
|
|
|
2362
2765
|
request.onerror = () => reject(request.error);
|
|
2363
2766
|
});
|
|
2364
2767
|
}
|
|
2768
|
+
async getAllJobs() {
|
|
2769
|
+
const db = await this.getDb();
|
|
2770
|
+
const tx = db.transaction(this.tableName, "readonly");
|
|
2771
|
+
const store = tx.objectStore(this.tableName);
|
|
2772
|
+
const index = store.index("queue_status");
|
|
2773
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
2774
|
+
return new Promise((resolve, reject) => {
|
|
2775
|
+
const jobs = [];
|
|
2776
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, ""]);
|
|
2777
|
+
const request = index.openCursor(keyRange);
|
|
2778
|
+
request.onsuccess = (event) => {
|
|
2779
|
+
const cursor = event.target.result;
|
|
2780
|
+
if (cursor) {
|
|
2781
|
+
const job = cursor.value;
|
|
2782
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
2783
|
+
jobs.push(job);
|
|
2784
|
+
}
|
|
2785
|
+
cursor.continue();
|
|
2786
|
+
}
|
|
2787
|
+
};
|
|
2788
|
+
tx.oncomplete = () => resolve(jobs);
|
|
2789
|
+
tx.onerror = () => reject(tx.error);
|
|
2790
|
+
request.onerror = () => reject(request.error);
|
|
2791
|
+
});
|
|
2792
|
+
}
|
|
2793
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
2794
|
+
const db = await this.getDb();
|
|
2795
|
+
const tx = db.transaction(this.tableName, "readonly");
|
|
2796
|
+
const store = tx.objectStore(this.tableName);
|
|
2797
|
+
return new Promise((resolve, reject) => {
|
|
2798
|
+
const jobs = [];
|
|
2799
|
+
const request = store.openCursor();
|
|
2800
|
+
request.onsuccess = (event) => {
|
|
2801
|
+
const cursor = event.target.result;
|
|
2802
|
+
if (cursor) {
|
|
2803
|
+
const job = cursor.value;
|
|
2804
|
+
if (job.queue !== this.queueName) {
|
|
2805
|
+
cursor.continue();
|
|
2806
|
+
return;
|
|
2807
|
+
}
|
|
2808
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
2809
|
+
jobs.push(job);
|
|
2810
|
+
} else {
|
|
2811
|
+
let matches = true;
|
|
2812
|
+
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
2813
|
+
if (job[key] !== value) {
|
|
2814
|
+
matches = false;
|
|
2815
|
+
break;
|
|
2816
|
+
}
|
|
2817
|
+
}
|
|
2818
|
+
if (matches) {
|
|
2819
|
+
jobs.push(job);
|
|
2820
|
+
}
|
|
2821
|
+
}
|
|
2822
|
+
cursor.continue();
|
|
2823
|
+
}
|
|
2824
|
+
};
|
|
2825
|
+
tx.oncomplete = () => resolve(jobs);
|
|
2826
|
+
tx.onerror = () => reject(tx.error);
|
|
2827
|
+
request.onerror = () => reject(request.error);
|
|
2828
|
+
});
|
|
2829
|
+
}
|
|
2830
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
2831
|
+
if (prefixFilter === undefined) {
|
|
2832
|
+
return false;
|
|
2833
|
+
}
|
|
2834
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
2835
|
+
return true;
|
|
2836
|
+
}
|
|
2837
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
2838
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
2839
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
2840
|
+
return true;
|
|
2841
|
+
}
|
|
2842
|
+
for (const key of instanceKeys) {
|
|
2843
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
2844
|
+
return true;
|
|
2845
|
+
}
|
|
2846
|
+
}
|
|
2847
|
+
return false;
|
|
2848
|
+
}
|
|
2849
|
+
getPollingManager() {
|
|
2850
|
+
if (!this.pollingManager) {
|
|
2851
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
2852
|
+
const jobs = await this.getAllJobs();
|
|
2853
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
2854
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
2855
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
2856
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
2857
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
2858
|
+
});
|
|
2859
|
+
}
|
|
2860
|
+
return this.pollingManager;
|
|
2861
|
+
}
|
|
2862
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
2863
|
+
let lastKnownJobs = new Map;
|
|
2864
|
+
let cancelled = false;
|
|
2865
|
+
const poll = async () => {
|
|
2866
|
+
if (cancelled)
|
|
2867
|
+
return;
|
|
2868
|
+
try {
|
|
2869
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
2870
|
+
if (cancelled)
|
|
2871
|
+
return;
|
|
2872
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
2873
|
+
for (const [id, job] of currentMap) {
|
|
2874
|
+
const old = lastKnownJobs.get(id);
|
|
2875
|
+
if (!old) {
|
|
2876
|
+
callback({ type: "INSERT", new: job });
|
|
2877
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
2878
|
+
callback({ type: "UPDATE", old, new: job });
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
for (const [id, job] of lastKnownJobs) {
|
|
2882
|
+
if (!currentMap.has(id)) {
|
|
2883
|
+
callback({ type: "DELETE", old: job });
|
|
2884
|
+
}
|
|
2885
|
+
}
|
|
2886
|
+
lastKnownJobs = currentMap;
|
|
2887
|
+
} catch {}
|
|
2888
|
+
};
|
|
2889
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
2890
|
+
poll();
|
|
2891
|
+
return () => {
|
|
2892
|
+
cancelled = true;
|
|
2893
|
+
clearInterval(intervalId);
|
|
2894
|
+
};
|
|
2895
|
+
}
|
|
2896
|
+
subscribeToChanges(callback, options) {
|
|
2897
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
2898
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
2899
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
2900
|
+
}
|
|
2901
|
+
const manager = this.getPollingManager();
|
|
2902
|
+
return manager.subscribe(callback, { intervalMs });
|
|
2903
|
+
}
|
|
2365
2904
|
}
|
|
2366
2905
|
// src/queue/SupabaseQueueStorage.ts
|
|
2367
|
-
import { createServiceToken as
|
|
2368
|
-
var SUPABASE_QUEUE_STORAGE =
|
|
2906
|
+
import { createServiceToken as createServiceToken16, makeFingerprint as makeFingerprint6, uuid4 as uuid43 } from "@workglow/util";
|
|
2907
|
+
var SUPABASE_QUEUE_STORAGE = createServiceToken16("jobqueue.storage.supabase");
|
|
2369
2908
|
|
|
2370
2909
|
class SupabaseQueueStorage {
|
|
2371
2910
|
client;
|
|
2372
2911
|
queueName;
|
|
2373
|
-
|
|
2912
|
+
prefixes;
|
|
2913
|
+
prefixValues;
|
|
2914
|
+
tableName;
|
|
2915
|
+
realtimeChannel = null;
|
|
2916
|
+
pollingManager = null;
|
|
2917
|
+
constructor(client, queueName, options) {
|
|
2374
2918
|
this.client = client;
|
|
2375
2919
|
this.queueName = queueName;
|
|
2920
|
+
this.prefixes = options?.prefixes ?? [];
|
|
2921
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
2922
|
+
if (this.prefixes.length > 0) {
|
|
2923
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
2924
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
2925
|
+
} else {
|
|
2926
|
+
this.tableName = "job_queue";
|
|
2927
|
+
}
|
|
2928
|
+
}
|
|
2929
|
+
getPrefixColumnType(type) {
|
|
2930
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
2931
|
+
}
|
|
2932
|
+
buildPrefixColumnsSql() {
|
|
2933
|
+
if (this.prefixes.length === 0)
|
|
2934
|
+
return "";
|
|
2935
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
2936
|
+
`) + `,
|
|
2937
|
+
`;
|
|
2938
|
+
}
|
|
2939
|
+
getPrefixColumnNames() {
|
|
2940
|
+
return this.prefixes.map((p) => p.name);
|
|
2941
|
+
}
|
|
2942
|
+
applyPrefixFilters(query) {
|
|
2943
|
+
let result = query;
|
|
2944
|
+
for (const prefix of this.prefixes) {
|
|
2945
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
2946
|
+
}
|
|
2947
|
+
return result;
|
|
2948
|
+
}
|
|
2949
|
+
getPrefixInsertValues() {
|
|
2950
|
+
const values = {};
|
|
2951
|
+
for (const prefix of this.prefixes) {
|
|
2952
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
2953
|
+
}
|
|
2954
|
+
return values;
|
|
2376
2955
|
}
|
|
2377
2956
|
async setupDatabase() {
|
|
2378
2957
|
const createTypeSql = `CREATE TYPE job_status AS ENUM (${Object.values(JobStatus).map((v) => `'${v}'`).join(",")})`;
|
|
@@ -2380,10 +2959,14 @@ class SupabaseQueueStorage {
|
|
|
2380
2959
|
if (typeError && typeError.code !== "42710") {
|
|
2381
2960
|
throw typeError;
|
|
2382
2961
|
}
|
|
2962
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
2963
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2964
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2965
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
2383
2966
|
const createTableSql = `
|
|
2384
|
-
CREATE TABLE IF NOT EXISTS
|
|
2967
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
2385
2968
|
id SERIAL NOT NULL,
|
|
2386
|
-
fingerprint text NOT NULL,
|
|
2969
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
2387
2970
|
queue text NOT NULL,
|
|
2388
2971
|
job_run_id text NOT NULL,
|
|
2389
2972
|
status job_status NOT NULL default 'PENDING',
|
|
@@ -2400,7 +2983,8 @@ class SupabaseQueueStorage {
|
|
|
2400
2983
|
error_code text,
|
|
2401
2984
|
progress real DEFAULT 0,
|
|
2402
2985
|
progress_message text DEFAULT '',
|
|
2403
|
-
progress_details jsonb
|
|
2986
|
+
progress_details jsonb,
|
|
2987
|
+
worker_id text
|
|
2404
2988
|
)`;
|
|
2405
2989
|
const { error: tableError } = await this.client.rpc("exec_sql", { query: createTableSql });
|
|
2406
2990
|
if (tableError) {
|
|
@@ -2409,12 +2993,12 @@ class SupabaseQueueStorage {
|
|
|
2409
2993
|
}
|
|
2410
2994
|
}
|
|
2411
2995
|
const indexes = [
|
|
2412
|
-
`CREATE INDEX IF NOT EXISTS
|
|
2413
|
-
`CREATE INDEX IF NOT EXISTS
|
|
2414
|
-
`CREATE INDEX IF NOT EXISTS
|
|
2996
|
+
`CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`,
|
|
2997
|
+
`CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`,
|
|
2998
|
+
`CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`
|
|
2415
2999
|
];
|
|
2416
3000
|
for (const indexSql of indexes) {
|
|
2417
|
-
|
|
3001
|
+
await this.client.rpc("exec_sql", { query: indexSql });
|
|
2418
3002
|
}
|
|
2419
3003
|
}
|
|
2420
3004
|
async add(job) {
|
|
@@ -2428,7 +3012,9 @@ class SupabaseQueueStorage {
|
|
|
2428
3012
|
job.progress_details = null;
|
|
2429
3013
|
job.created_at = now;
|
|
2430
3014
|
job.run_after = now;
|
|
2431
|
-
const
|
|
3015
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
3016
|
+
const { data, error } = await this.client.from(this.tableName).insert({
|
|
3017
|
+
...prefixInsertValues,
|
|
2432
3018
|
queue: job.queue,
|
|
2433
3019
|
fingerprint: job.fingerprint,
|
|
2434
3020
|
input: job.input,
|
|
@@ -2449,7 +3035,9 @@ class SupabaseQueueStorage {
|
|
|
2449
3035
|
return job.id;
|
|
2450
3036
|
}
|
|
2451
3037
|
async get(id) {
|
|
2452
|
-
|
|
3038
|
+
let query = this.client.from(this.tableName).select("*").eq("id", id).eq("queue", this.queueName);
|
|
3039
|
+
query = this.applyPrefixFilters(query);
|
|
3040
|
+
const { data, error } = await query.single();
|
|
2453
3041
|
if (error) {
|
|
2454
3042
|
if (error.code === "PGRST116")
|
|
2455
3043
|
return;
|
|
@@ -2459,36 +3047,53 @@ class SupabaseQueueStorage {
|
|
|
2459
3047
|
}
|
|
2460
3048
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2461
3049
|
num = Number(num) || 100;
|
|
2462
|
-
|
|
3050
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", status);
|
|
3051
|
+
query = this.applyPrefixFilters(query);
|
|
3052
|
+
const { data, error } = await query.order("run_after", { ascending: true }).limit(num);
|
|
2463
3053
|
if (error)
|
|
2464
3054
|
throw error;
|
|
2465
3055
|
return data ?? [];
|
|
2466
3056
|
}
|
|
2467
|
-
async next() {
|
|
2468
|
-
|
|
3057
|
+
async next(workerId) {
|
|
3058
|
+
let selectQuery = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", "PENDING" /* PENDING */).lte("run_after", new Date().toISOString());
|
|
3059
|
+
selectQuery = this.applyPrefixFilters(selectQuery);
|
|
3060
|
+
const { data: jobs, error: selectError } = await selectQuery.order("run_after", { ascending: true }).limit(1);
|
|
2469
3061
|
if (selectError)
|
|
2470
3062
|
throw selectError;
|
|
2471
3063
|
if (!jobs || jobs.length === 0)
|
|
2472
3064
|
return;
|
|
2473
3065
|
const job = jobs[0];
|
|
2474
|
-
|
|
3066
|
+
let updateQuery = this.client.from(this.tableName).update({
|
|
2475
3067
|
status: "PROCESSING" /* PROCESSING */,
|
|
2476
|
-
last_ran_at: new Date().toISOString()
|
|
2477
|
-
|
|
3068
|
+
last_ran_at: new Date().toISOString(),
|
|
3069
|
+
worker_id: workerId ?? null
|
|
3070
|
+
}).eq("id", job.id).eq("queue", this.queueName);
|
|
3071
|
+
updateQuery = this.applyPrefixFilters(updateQuery);
|
|
3072
|
+
const { data: updatedJob, error: updateError } = await updateQuery.select().single();
|
|
2478
3073
|
if (updateError)
|
|
2479
3074
|
throw updateError;
|
|
2480
3075
|
return updatedJob;
|
|
2481
3076
|
}
|
|
2482
3077
|
async size(status = "PENDING" /* PENDING */) {
|
|
2483
|
-
|
|
3078
|
+
let query = this.client.from(this.tableName).select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
|
|
3079
|
+
query = this.applyPrefixFilters(query);
|
|
3080
|
+
const { count, error } = await query;
|
|
2484
3081
|
if (error)
|
|
2485
3082
|
throw error;
|
|
2486
3083
|
return count ?? 0;
|
|
2487
3084
|
}
|
|
3085
|
+
async getAllJobs() {
|
|
3086
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
3087
|
+
query = this.applyPrefixFilters(query);
|
|
3088
|
+
const { data, error } = await query;
|
|
3089
|
+
if (error)
|
|
3090
|
+
throw error;
|
|
3091
|
+
return data ?? [];
|
|
3092
|
+
}
|
|
2488
3093
|
async complete(jobDetails) {
|
|
2489
3094
|
const now = new Date().toISOString();
|
|
2490
3095
|
if (jobDetails.status === "DISABLED" /* DISABLED */) {
|
|
2491
|
-
|
|
3096
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2492
3097
|
status: jobDetails.status,
|
|
2493
3098
|
progress: 100,
|
|
2494
3099
|
progress_message: "",
|
|
@@ -2496,16 +3101,20 @@ class SupabaseQueueStorage {
|
|
|
2496
3101
|
completed_at: now,
|
|
2497
3102
|
last_ran_at: now
|
|
2498
3103
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3104
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3105
|
+
const { error: error2 } = await query2;
|
|
2499
3106
|
if (error2)
|
|
2500
3107
|
throw error2;
|
|
2501
3108
|
return;
|
|
2502
3109
|
}
|
|
2503
|
-
|
|
3110
|
+
let getQuery = this.client.from(this.tableName).select("run_attempts").eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3111
|
+
getQuery = this.applyPrefixFilters(getQuery);
|
|
3112
|
+
const { data: current, error: getError } = await getQuery.single();
|
|
2504
3113
|
if (getError)
|
|
2505
3114
|
throw getError;
|
|
2506
3115
|
const nextAttempts = (current?.run_attempts ?? 0) + 1;
|
|
2507
3116
|
if (jobDetails.status === "PENDING" /* PENDING */) {
|
|
2508
|
-
|
|
3117
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2509
3118
|
error: jobDetails.error ?? null,
|
|
2510
3119
|
error_code: jobDetails.error_code ?? null,
|
|
2511
3120
|
status: jobDetails.status,
|
|
@@ -2516,12 +3125,14 @@ class SupabaseQueueStorage {
|
|
|
2516
3125
|
run_attempts: nextAttempts,
|
|
2517
3126
|
last_ran_at: now
|
|
2518
3127
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3128
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3129
|
+
const { error: error2 } = await query2;
|
|
2519
3130
|
if (error2)
|
|
2520
3131
|
throw error2;
|
|
2521
3132
|
return;
|
|
2522
3133
|
}
|
|
2523
3134
|
if (jobDetails.status === "COMPLETED" /* COMPLETED */ || jobDetails.status === "FAILED" /* FAILED */) {
|
|
2524
|
-
|
|
3135
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2525
3136
|
output: jobDetails.output ?? null,
|
|
2526
3137
|
error: jobDetails.error ?? null,
|
|
2527
3138
|
error_code: jobDetails.error_code ?? null,
|
|
@@ -2533,11 +3144,13 @@ class SupabaseQueueStorage {
|
|
|
2533
3144
|
completed_at: now,
|
|
2534
3145
|
last_ran_at: now
|
|
2535
3146
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3147
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3148
|
+
const { error: error2 } = await query2;
|
|
2536
3149
|
if (error2)
|
|
2537
3150
|
throw error2;
|
|
2538
3151
|
return;
|
|
2539
3152
|
}
|
|
2540
|
-
|
|
3153
|
+
let query = this.client.from(this.tableName).update({
|
|
2541
3154
|
status: jobDetails.status,
|
|
2542
3155
|
output: jobDetails.output ?? null,
|
|
2543
3156
|
error: jobDetails.error ?? null,
|
|
@@ -2546,17 +3159,23 @@ class SupabaseQueueStorage {
|
|
|
2546
3159
|
run_attempts: nextAttempts,
|
|
2547
3160
|
last_ran_at: now
|
|
2548
3161
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3162
|
+
query = this.applyPrefixFilters(query);
|
|
3163
|
+
const { error } = await query;
|
|
2549
3164
|
if (error)
|
|
2550
3165
|
throw error;
|
|
2551
3166
|
}
|
|
2552
3167
|
async deleteAll() {
|
|
2553
|
-
|
|
3168
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
|
|
3169
|
+
query = this.applyPrefixFilters(query);
|
|
3170
|
+
const { error } = await query;
|
|
2554
3171
|
if (error)
|
|
2555
3172
|
throw error;
|
|
2556
3173
|
}
|
|
2557
3174
|
async outputForInput(input) {
|
|
2558
3175
|
const fingerprint = await makeFingerprint6(input);
|
|
2559
|
-
|
|
3176
|
+
let query = this.client.from(this.tableName).select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", "COMPLETED" /* COMPLETED */);
|
|
3177
|
+
query = this.applyPrefixFilters(query);
|
|
3178
|
+
const { data, error } = await query.single();
|
|
2560
3179
|
if (error) {
|
|
2561
3180
|
if (error.code === "PGRST116")
|
|
2562
3181
|
return null;
|
|
@@ -2565,36 +3184,555 @@ class SupabaseQueueStorage {
|
|
|
2565
3184
|
return data?.output ?? null;
|
|
2566
3185
|
}
|
|
2567
3186
|
async abort(jobId) {
|
|
2568
|
-
|
|
3187
|
+
let query = this.client.from(this.tableName).update({ status: "ABORTING" /* ABORTING */ }).eq("id", jobId).eq("queue", this.queueName);
|
|
3188
|
+
query = this.applyPrefixFilters(query);
|
|
3189
|
+
const { error } = await query;
|
|
2569
3190
|
if (error)
|
|
2570
3191
|
throw error;
|
|
2571
3192
|
}
|
|
2572
3193
|
async getByRunId(job_run_id) {
|
|
2573
|
-
|
|
3194
|
+
let query = this.client.from(this.tableName).select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
|
|
3195
|
+
query = this.applyPrefixFilters(query);
|
|
3196
|
+
const { data, error } = await query;
|
|
2574
3197
|
if (error)
|
|
2575
3198
|
throw error;
|
|
2576
3199
|
return data ?? [];
|
|
2577
3200
|
}
|
|
2578
3201
|
async saveProgress(jobId, progress, message, details) {
|
|
2579
|
-
|
|
3202
|
+
let query = this.client.from(this.tableName).update({
|
|
2580
3203
|
progress,
|
|
2581
3204
|
progress_message: message,
|
|
2582
3205
|
progress_details: details
|
|
2583
3206
|
}).eq("id", jobId).eq("queue", this.queueName);
|
|
3207
|
+
query = this.applyPrefixFilters(query);
|
|
3208
|
+
const { error } = await query;
|
|
2584
3209
|
if (error)
|
|
2585
3210
|
throw error;
|
|
2586
3211
|
}
|
|
2587
3212
|
async delete(jobId) {
|
|
2588
|
-
|
|
3213
|
+
let query = this.client.from(this.tableName).delete().eq("id", jobId).eq("queue", this.queueName);
|
|
3214
|
+
query = this.applyPrefixFilters(query);
|
|
3215
|
+
const { error } = await query;
|
|
2589
3216
|
if (error)
|
|
2590
3217
|
throw error;
|
|
2591
3218
|
}
|
|
2592
3219
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2593
3220
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
2594
|
-
|
|
3221
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
|
|
3222
|
+
query = this.applyPrefixFilters(query);
|
|
3223
|
+
const { error } = await query;
|
|
2595
3224
|
if (error)
|
|
2596
3225
|
throw error;
|
|
2597
3226
|
}
|
|
3227
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
3228
|
+
if (!job)
|
|
3229
|
+
return false;
|
|
3230
|
+
if (job.queue !== this.queueName) {
|
|
3231
|
+
return false;
|
|
3232
|
+
}
|
|
3233
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
3234
|
+
return true;
|
|
3235
|
+
}
|
|
3236
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
3237
|
+
if (Object.keys(filterValues).length === 0) {
|
|
3238
|
+
return true;
|
|
3239
|
+
}
|
|
3240
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
3241
|
+
if (job[key] !== value) {
|
|
3242
|
+
return false;
|
|
3243
|
+
}
|
|
3244
|
+
}
|
|
3245
|
+
return true;
|
|
3246
|
+
}
|
|
3247
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
3248
|
+
if (prefixFilter === undefined) {
|
|
3249
|
+
return false;
|
|
3250
|
+
}
|
|
3251
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
3252
|
+
return true;
|
|
3253
|
+
}
|
|
3254
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
3255
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
3256
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
3257
|
+
return true;
|
|
3258
|
+
}
|
|
3259
|
+
for (const key of instanceKeys) {
|
|
3260
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
3261
|
+
return true;
|
|
3262
|
+
}
|
|
3263
|
+
}
|
|
3264
|
+
return false;
|
|
3265
|
+
}
|
|
3266
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
3267
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
3268
|
+
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
3269
|
+
query = query.eq(key, value);
|
|
3270
|
+
}
|
|
3271
|
+
const { data, error } = await query;
|
|
3272
|
+
if (error)
|
|
3273
|
+
throw error;
|
|
3274
|
+
return data ?? [];
|
|
3275
|
+
}
|
|
3276
|
+
subscribeToChanges(callback, options) {
|
|
3277
|
+
return this.subscribeToChangesWithRealtime(callback, options?.prefixFilter);
|
|
3278
|
+
}
|
|
3279
|
+
subscribeToChangesWithRealtime(callback, prefixFilter) {
|
|
3280
|
+
const channelName = `queue-${this.tableName}-${this.queueName}-${Date.now()}`;
|
|
3281
|
+
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
3282
|
+
event: "*",
|
|
3283
|
+
schema: "public",
|
|
3284
|
+
table: this.tableName,
|
|
3285
|
+
filter: `queue=eq.${this.queueName}`
|
|
3286
|
+
}, (payload) => {
|
|
3287
|
+
const newJob = payload.new;
|
|
3288
|
+
const oldJob = payload.old;
|
|
3289
|
+
const newMatches = this.matchesPrefixFilter(newJob, prefixFilter);
|
|
3290
|
+
const oldMatches = this.matchesPrefixFilter(oldJob, prefixFilter);
|
|
3291
|
+
if (!newMatches && !oldMatches) {
|
|
3292
|
+
return;
|
|
3293
|
+
}
|
|
3294
|
+
callback({
|
|
3295
|
+
type: payload.eventType.toUpperCase(),
|
|
3296
|
+
old: oldJob && Object.keys(oldJob).length > 0 ? oldJob : undefined,
|
|
3297
|
+
new: newJob && Object.keys(newJob).length > 0 ? newJob : undefined
|
|
3298
|
+
});
|
|
3299
|
+
}).subscribe();
|
|
3300
|
+
return () => {
|
|
3301
|
+
if (this.realtimeChannel) {
|
|
3302
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
3303
|
+
this.realtimeChannel = null;
|
|
3304
|
+
}
|
|
3305
|
+
};
|
|
3306
|
+
}
|
|
3307
|
+
getPollingManager() {
|
|
3308
|
+
if (!this.pollingManager) {
|
|
3309
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
3310
|
+
const jobs = await this.getAllJobs();
|
|
3311
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
3312
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
3313
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
3314
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
3315
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
3316
|
+
});
|
|
3317
|
+
}
|
|
3318
|
+
return this.pollingManager;
|
|
3319
|
+
}
|
|
3320
|
+
subscribeWithCustomPrefixFilterPolling(callback, prefixFilter, intervalMs) {
|
|
3321
|
+
let lastKnownJobs = new Map;
|
|
3322
|
+
let cancelled = false;
|
|
3323
|
+
const poll = async () => {
|
|
3324
|
+
if (cancelled)
|
|
3325
|
+
return;
|
|
3326
|
+
try {
|
|
3327
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
3328
|
+
if (cancelled)
|
|
3329
|
+
return;
|
|
3330
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
3331
|
+
for (const [id, job] of currentMap) {
|
|
3332
|
+
const old = lastKnownJobs.get(id);
|
|
3333
|
+
if (!old) {
|
|
3334
|
+
callback({ type: "INSERT", new: job });
|
|
3335
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
3336
|
+
callback({ type: "UPDATE", old, new: job });
|
|
3337
|
+
}
|
|
3338
|
+
}
|
|
3339
|
+
for (const [id, job] of lastKnownJobs) {
|
|
3340
|
+
if (!currentMap.has(id)) {
|
|
3341
|
+
callback({ type: "DELETE", old: job });
|
|
3342
|
+
}
|
|
3343
|
+
}
|
|
3344
|
+
lastKnownJobs = currentMap;
|
|
3345
|
+
} catch {}
|
|
3346
|
+
};
|
|
3347
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
3348
|
+
poll();
|
|
3349
|
+
return () => {
|
|
3350
|
+
cancelled = true;
|
|
3351
|
+
clearInterval(intervalId);
|
|
3352
|
+
};
|
|
3353
|
+
}
|
|
3354
|
+
subscribeToChangesWithPolling(callback, options) {
|
|
3355
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
3356
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
3357
|
+
return this.subscribeWithCustomPrefixFilterPolling(callback, options.prefixFilter, intervalMs);
|
|
3358
|
+
}
|
|
3359
|
+
const manager = this.getPollingManager();
|
|
3360
|
+
return manager.subscribe(callback, { intervalMs });
|
|
3361
|
+
}
|
|
3362
|
+
}
|
|
3363
|
+
// src/limiter/IndexedDbRateLimiterStorage.ts
|
|
3364
|
+
import { createServiceToken as createServiceToken17 } from "@workglow/util";
|
|
3365
|
+
var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken17("ratelimiter.storage.indexedDb");
|
|
3366
|
+
|
|
3367
|
+
class IndexedDbRateLimiterStorage {
|
|
3368
|
+
executionDb;
|
|
3369
|
+
nextAvailableDb;
|
|
3370
|
+
executionTableName;
|
|
3371
|
+
nextAvailableTableName;
|
|
3372
|
+
migrationOptions;
|
|
3373
|
+
prefixes;
|
|
3374
|
+
prefixValues;
|
|
3375
|
+
constructor(options = {}) {
|
|
3376
|
+
this.migrationOptions = options;
|
|
3377
|
+
this.prefixes = options.prefixes ?? [];
|
|
3378
|
+
this.prefixValues = options.prefixValues ?? {};
|
|
3379
|
+
if (this.prefixes.length > 0) {
|
|
3380
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3381
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
3382
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
3383
|
+
} else {
|
|
3384
|
+
this.executionTableName = "rate_limit_executions";
|
|
3385
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
3386
|
+
}
|
|
3387
|
+
}
|
|
3388
|
+
getPrefixColumnNames() {
|
|
3389
|
+
return this.prefixes.map((p) => p.name);
|
|
3390
|
+
}
|
|
3391
|
+
matchesPrefixes(record) {
|
|
3392
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
3393
|
+
if (record[key] !== value) {
|
|
3394
|
+
return false;
|
|
3395
|
+
}
|
|
3396
|
+
}
|
|
3397
|
+
return true;
|
|
3398
|
+
}
|
|
3399
|
+
getPrefixKeyValues() {
|
|
3400
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3401
|
+
}
|
|
3402
|
+
async getExecutionDb() {
|
|
3403
|
+
if (this.executionDb)
|
|
3404
|
+
return this.executionDb;
|
|
3405
|
+
await this.setupDatabase();
|
|
3406
|
+
return this.executionDb;
|
|
3407
|
+
}
|
|
3408
|
+
async getNextAvailableDb() {
|
|
3409
|
+
if (this.nextAvailableDb)
|
|
3410
|
+
return this.nextAvailableDb;
|
|
3411
|
+
await this.setupDatabase();
|
|
3412
|
+
return this.nextAvailableDb;
|
|
3413
|
+
}
|
|
3414
|
+
async setupDatabase() {
|
|
3415
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3416
|
+
const buildKeyPath = (basePath) => {
|
|
3417
|
+
return [...prefixColumnNames, ...basePath];
|
|
3418
|
+
};
|
|
3419
|
+
const executionIndexes = [
|
|
3420
|
+
{
|
|
3421
|
+
name: "queue_executed_at",
|
|
3422
|
+
keyPath: buildKeyPath(["queue_name", "executed_at"]),
|
|
3423
|
+
options: { unique: false }
|
|
3424
|
+
}
|
|
3425
|
+
];
|
|
3426
|
+
this.executionDb = await ensureIndexedDbTable(this.executionTableName, "id", executionIndexes, this.migrationOptions);
|
|
3427
|
+
const nextAvailableIndexes = [
|
|
3428
|
+
{
|
|
3429
|
+
name: "queue_name",
|
|
3430
|
+
keyPath: buildKeyPath(["queue_name"]),
|
|
3431
|
+
options: { unique: true }
|
|
3432
|
+
}
|
|
3433
|
+
];
|
|
3434
|
+
this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
|
|
3435
|
+
}
|
|
3436
|
+
async recordExecution(queueName) {
|
|
3437
|
+
const db = await this.getExecutionDb();
|
|
3438
|
+
const tx = db.transaction(this.executionTableName, "readwrite");
|
|
3439
|
+
const store = tx.objectStore(this.executionTableName);
|
|
3440
|
+
const record = {
|
|
3441
|
+
id: crypto.randomUUID(),
|
|
3442
|
+
queue_name: queueName,
|
|
3443
|
+
executed_at: new Date().toISOString()
|
|
3444
|
+
};
|
|
3445
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
3446
|
+
record[key] = value;
|
|
3447
|
+
}
|
|
3448
|
+
return new Promise((resolve, reject) => {
|
|
3449
|
+
const request = store.add(record);
|
|
3450
|
+
tx.oncomplete = () => resolve();
|
|
3451
|
+
tx.onerror = () => reject(tx.error);
|
|
3452
|
+
request.onerror = () => reject(request.error);
|
|
3453
|
+
});
|
|
3454
|
+
}
|
|
3455
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
3456
|
+
const db = await this.getExecutionDb();
|
|
3457
|
+
const tx = db.transaction(this.executionTableName, "readonly");
|
|
3458
|
+
const store = tx.objectStore(this.executionTableName);
|
|
3459
|
+
const index = store.index("queue_executed_at");
|
|
3460
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3461
|
+
return new Promise((resolve, reject) => {
|
|
3462
|
+
let count = 0;
|
|
3463
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartTime], [...prefixKeyValues, queueName, ""], true, false);
|
|
3464
|
+
const request = index.openCursor(keyRange);
|
|
3465
|
+
request.onsuccess = (event) => {
|
|
3466
|
+
const cursor = event.target.result;
|
|
3467
|
+
if (cursor) {
|
|
3468
|
+
const record = cursor.value;
|
|
3469
|
+
if (this.matchesPrefixes(record)) {
|
|
3470
|
+
count++;
|
|
3471
|
+
}
|
|
3472
|
+
cursor.continue();
|
|
3473
|
+
}
|
|
3474
|
+
};
|
|
3475
|
+
tx.oncomplete = () => resolve(count);
|
|
3476
|
+
tx.onerror = () => reject(tx.error);
|
|
3477
|
+
request.onerror = () => reject(request.error);
|
|
3478
|
+
});
|
|
3479
|
+
}
|
|
3480
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
3481
|
+
const db = await this.getExecutionDb();
|
|
3482
|
+
const tx = db.transaction(this.executionTableName, "readonly");
|
|
3483
|
+
const store = tx.objectStore(this.executionTableName);
|
|
3484
|
+
const index = store.index("queue_executed_at");
|
|
3485
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3486
|
+
return new Promise((resolve, reject) => {
|
|
3487
|
+
const executions = [];
|
|
3488
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, ""]);
|
|
3489
|
+
const request = index.openCursor(keyRange);
|
|
3490
|
+
request.onsuccess = (event) => {
|
|
3491
|
+
const cursor = event.target.result;
|
|
3492
|
+
if (cursor) {
|
|
3493
|
+
const record = cursor.value;
|
|
3494
|
+
if (this.matchesPrefixes(record)) {
|
|
3495
|
+
executions.push(record.executed_at);
|
|
3496
|
+
}
|
|
3497
|
+
cursor.continue();
|
|
3498
|
+
}
|
|
3499
|
+
};
|
|
3500
|
+
tx.oncomplete = () => {
|
|
3501
|
+
executions.sort();
|
|
3502
|
+
resolve(executions[offset]);
|
|
3503
|
+
};
|
|
3504
|
+
tx.onerror = () => reject(tx.error);
|
|
3505
|
+
request.onerror = () => reject(request.error);
|
|
3506
|
+
});
|
|
3507
|
+
}
|
|
3508
|
+
async getNextAvailableTime(queueName) {
|
|
3509
|
+
const db = await this.getNextAvailableDb();
|
|
3510
|
+
const tx = db.transaction(this.nextAvailableTableName, "readonly");
|
|
3511
|
+
const store = tx.objectStore(this.nextAvailableTableName);
|
|
3512
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3513
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
3514
|
+
return new Promise((resolve, reject) => {
|
|
3515
|
+
const request = store.get(key);
|
|
3516
|
+
request.onsuccess = () => {
|
|
3517
|
+
const record = request.result;
|
|
3518
|
+
if (record && this.matchesPrefixes(record)) {
|
|
3519
|
+
resolve(record.next_available_at);
|
|
3520
|
+
} else {
|
|
3521
|
+
resolve(undefined);
|
|
3522
|
+
}
|
|
3523
|
+
};
|
|
3524
|
+
request.onerror = () => reject(request.error);
|
|
3525
|
+
tx.onerror = () => reject(tx.error);
|
|
3526
|
+
});
|
|
3527
|
+
}
|
|
3528
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
3529
|
+
const db = await this.getNextAvailableDb();
|
|
3530
|
+
const tx = db.transaction(this.nextAvailableTableName, "readwrite");
|
|
3531
|
+
const store = tx.objectStore(this.nextAvailableTableName);
|
|
3532
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3533
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
3534
|
+
const record = {
|
|
3535
|
+
queue_name: queueName,
|
|
3536
|
+
next_available_at: nextAvailableAt
|
|
3537
|
+
};
|
|
3538
|
+
for (const [k, value] of Object.entries(this.prefixValues)) {
|
|
3539
|
+
record[k] = value;
|
|
3540
|
+
}
|
|
3541
|
+
record[this.getPrefixColumnNames().concat(["queue_name"]).join("_")] = key;
|
|
3542
|
+
return new Promise((resolve, reject) => {
|
|
3543
|
+
const request = store.put(record);
|
|
3544
|
+
tx.oncomplete = () => resolve();
|
|
3545
|
+
tx.onerror = () => reject(tx.error);
|
|
3546
|
+
request.onerror = () => reject(request.error);
|
|
3547
|
+
});
|
|
3548
|
+
}
|
|
3549
|
+
async clear(queueName) {
|
|
3550
|
+
const execDb = await this.getExecutionDb();
|
|
3551
|
+
const execTx = execDb.transaction(this.executionTableName, "readwrite");
|
|
3552
|
+
const execStore = execTx.objectStore(this.executionTableName);
|
|
3553
|
+
const execIndex = execStore.index("queue_executed_at");
|
|
3554
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3555
|
+
await new Promise((resolve, reject) => {
|
|
3556
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, ""]);
|
|
3557
|
+
const request = execIndex.openCursor(keyRange);
|
|
3558
|
+
request.onsuccess = (event) => {
|
|
3559
|
+
const cursor = event.target.result;
|
|
3560
|
+
if (cursor) {
|
|
3561
|
+
const record = cursor.value;
|
|
3562
|
+
if (this.matchesPrefixes(record)) {
|
|
3563
|
+
cursor.delete();
|
|
3564
|
+
}
|
|
3565
|
+
cursor.continue();
|
|
3566
|
+
}
|
|
3567
|
+
};
|
|
3568
|
+
execTx.oncomplete = () => resolve();
|
|
3569
|
+
execTx.onerror = () => reject(execTx.error);
|
|
3570
|
+
request.onerror = () => reject(request.error);
|
|
3571
|
+
});
|
|
3572
|
+
const nextDb = await this.getNextAvailableDb();
|
|
3573
|
+
const nextTx = nextDb.transaction(this.nextAvailableTableName, "readwrite");
|
|
3574
|
+
const nextStore = nextTx.objectStore(this.nextAvailableTableName);
|
|
3575
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
3576
|
+
await new Promise((resolve, reject) => {
|
|
3577
|
+
const request = nextStore.delete(key);
|
|
3578
|
+
nextTx.oncomplete = () => resolve();
|
|
3579
|
+
nextTx.onerror = () => reject(nextTx.error);
|
|
3580
|
+
request.onerror = () => reject(request.error);
|
|
3581
|
+
});
|
|
3582
|
+
}
|
|
3583
|
+
}
|
|
3584
|
+
// src/limiter/SupabaseRateLimiterStorage.ts
|
|
3585
|
+
import { createServiceToken as createServiceToken18 } from "@workglow/util";
|
|
3586
|
+
var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken18("ratelimiter.storage.supabase");
|
|
3587
|
+
|
|
3588
|
+
class SupabaseRateLimiterStorage {
|
|
3589
|
+
client;
|
|
3590
|
+
prefixes;
|
|
3591
|
+
prefixValues;
|
|
3592
|
+
executionTableName;
|
|
3593
|
+
nextAvailableTableName;
|
|
3594
|
+
constructor(client, options) {
|
|
3595
|
+
this.client = client;
|
|
3596
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3597
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3598
|
+
if (this.prefixes.length > 0) {
|
|
3599
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3600
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
3601
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
3602
|
+
} else {
|
|
3603
|
+
this.executionTableName = "rate_limit_executions";
|
|
3604
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
3605
|
+
}
|
|
3606
|
+
}
|
|
3607
|
+
getPrefixColumnType(type) {
|
|
3608
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
3609
|
+
}
|
|
3610
|
+
buildPrefixColumnsSql() {
|
|
3611
|
+
if (this.prefixes.length === 0)
|
|
3612
|
+
return "";
|
|
3613
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
3614
|
+
`) + `,
|
|
3615
|
+
`;
|
|
3616
|
+
}
|
|
3617
|
+
getPrefixColumnNames() {
|
|
3618
|
+
return this.prefixes.map((p) => p.name);
|
|
3619
|
+
}
|
|
3620
|
+
applyPrefixFilters(query) {
|
|
3621
|
+
let result = query;
|
|
3622
|
+
for (const prefix of this.prefixes) {
|
|
3623
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
3624
|
+
}
|
|
3625
|
+
return result;
|
|
3626
|
+
}
|
|
3627
|
+
getPrefixInsertValues() {
|
|
3628
|
+
const values = {};
|
|
3629
|
+
for (const prefix of this.prefixes) {
|
|
3630
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
3631
|
+
}
|
|
3632
|
+
return values;
|
|
3633
|
+
}
|
|
3634
|
+
async setupDatabase() {
|
|
3635
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
3636
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3637
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3638
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
3639
|
+
const createExecTableSql = `
|
|
3640
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
3641
|
+
id SERIAL PRIMARY KEY,
|
|
3642
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3643
|
+
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
3644
|
+
)
|
|
3645
|
+
`;
|
|
3646
|
+
const { error: execTableError } = await this.client.rpc("exec_sql", {
|
|
3647
|
+
query: createExecTableSql
|
|
3648
|
+
});
|
|
3649
|
+
if (execTableError && execTableError.code !== "42P07") {
|
|
3650
|
+
throw execTableError;
|
|
3651
|
+
}
|
|
3652
|
+
const createExecIndexSql = `
|
|
3653
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
3654
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
3655
|
+
`;
|
|
3656
|
+
await this.client.rpc("exec_sql", { query: createExecIndexSql });
|
|
3657
|
+
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
3658
|
+
const createNextTableSql = `
|
|
3659
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
3660
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3661
|
+
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
3662
|
+
PRIMARY KEY (${primaryKeyColumns})
|
|
3663
|
+
)
|
|
3664
|
+
`;
|
|
3665
|
+
const { error: nextTableError } = await this.client.rpc("exec_sql", {
|
|
3666
|
+
query: createNextTableSql
|
|
3667
|
+
});
|
|
3668
|
+
if (nextTableError && nextTableError.code !== "42P07") {
|
|
3669
|
+
throw nextTableError;
|
|
3670
|
+
}
|
|
3671
|
+
}
|
|
3672
|
+
async recordExecution(queueName) {
|
|
3673
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
3674
|
+
const { error } = await this.client.from(this.executionTableName).insert({
|
|
3675
|
+
...prefixInsertValues,
|
|
3676
|
+
queue_name: queueName
|
|
3677
|
+
});
|
|
3678
|
+
if (error)
|
|
3679
|
+
throw error;
|
|
3680
|
+
}
|
|
3681
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
3682
|
+
let query = this.client.from(this.executionTableName).select("*", { count: "exact", head: true }).eq("queue_name", queueName).gt("executed_at", windowStartTime);
|
|
3683
|
+
query = this.applyPrefixFilters(query);
|
|
3684
|
+
const { count, error } = await query;
|
|
3685
|
+
if (error)
|
|
3686
|
+
throw error;
|
|
3687
|
+
return count ?? 0;
|
|
3688
|
+
}
|
|
3689
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
3690
|
+
let query = this.client.from(this.executionTableName).select("executed_at").eq("queue_name", queueName);
|
|
3691
|
+
query = this.applyPrefixFilters(query);
|
|
3692
|
+
const { data, error } = await query.order("executed_at", { ascending: true }).range(offset, offset);
|
|
3693
|
+
if (error)
|
|
3694
|
+
throw error;
|
|
3695
|
+
if (!data || data.length === 0)
|
|
3696
|
+
return;
|
|
3697
|
+
return new Date(data[0].executed_at).toISOString();
|
|
3698
|
+
}
|
|
3699
|
+
async getNextAvailableTime(queueName) {
|
|
3700
|
+
let query = this.client.from(this.nextAvailableTableName).select("next_available_at").eq("queue_name", queueName);
|
|
3701
|
+
query = this.applyPrefixFilters(query);
|
|
3702
|
+
const { data, error } = await query.single();
|
|
3703
|
+
if (error) {
|
|
3704
|
+
if (error.code === "PGRST116")
|
|
3705
|
+
return;
|
|
3706
|
+
throw error;
|
|
3707
|
+
}
|
|
3708
|
+
if (!data?.next_available_at)
|
|
3709
|
+
return;
|
|
3710
|
+
return new Date(data.next_available_at).toISOString();
|
|
3711
|
+
}
|
|
3712
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
3713
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
3714
|
+
const { error } = await this.client.from(this.nextAvailableTableName).upsert({
|
|
3715
|
+
...prefixInsertValues,
|
|
3716
|
+
queue_name: queueName,
|
|
3717
|
+
next_available_at: nextAvailableAt
|
|
3718
|
+
}, {
|
|
3719
|
+
onConflict: this.prefixes.length > 0 ? `${this.getPrefixColumnNames().join(",")},queue_name` : "queue_name"
|
|
3720
|
+
});
|
|
3721
|
+
if (error)
|
|
3722
|
+
throw error;
|
|
3723
|
+
}
|
|
3724
|
+
async clear(queueName) {
|
|
3725
|
+
let execQuery = this.client.from(this.executionTableName).delete().eq("queue_name", queueName);
|
|
3726
|
+
execQuery = this.applyPrefixFilters(execQuery);
|
|
3727
|
+
const { error: execError } = await execQuery;
|
|
3728
|
+
if (execError)
|
|
3729
|
+
throw execError;
|
|
3730
|
+
let nextQuery = this.client.from(this.nextAvailableTableName).delete().eq("queue_name", queueName);
|
|
3731
|
+
nextQuery = this.applyPrefixFilters(nextQuery);
|
|
3732
|
+
const { error: nextError } = await nextQuery;
|
|
3733
|
+
if (nextError)
|
|
3734
|
+
throw nextError;
|
|
3735
|
+
}
|
|
2598
3736
|
}
|
|
2599
3737
|
export {
|
|
2600
3738
|
ensureIndexedDbTable,
|
|
@@ -2602,13 +3740,16 @@ export {
|
|
|
2602
3740
|
TabularRepository,
|
|
2603
3741
|
TABULAR_REPOSITORY,
|
|
2604
3742
|
SupabaseTabularRepository,
|
|
3743
|
+
SupabaseRateLimiterStorage,
|
|
2605
3744
|
SupabaseQueueStorage,
|
|
2606
3745
|
SupabaseKvRepository,
|
|
2607
3746
|
SharedInMemoryTabularRepository,
|
|
2608
3747
|
SUPABASE_TABULAR_REPOSITORY,
|
|
3748
|
+
SUPABASE_RATE_LIMITER_STORAGE,
|
|
2609
3749
|
SUPABASE_QUEUE_STORAGE,
|
|
2610
3750
|
SUPABASE_KV_REPOSITORY,
|
|
2611
3751
|
SHARED_IN_MEMORY_TABULAR_REPOSITORY,
|
|
3752
|
+
RATE_LIMITER_STORAGE,
|
|
2612
3753
|
QUEUE_STORAGE,
|
|
2613
3754
|
MEMORY_TABULAR_REPOSITORY,
|
|
2614
3755
|
MEMORY_KV_REPOSITORY,
|
|
@@ -2617,12 +3758,16 @@ export {
|
|
|
2617
3758
|
KV_REPOSITORY,
|
|
2618
3759
|
JobStatus,
|
|
2619
3760
|
IndexedDbTabularRepository,
|
|
3761
|
+
IndexedDbRateLimiterStorage,
|
|
2620
3762
|
IndexedDbQueueStorage,
|
|
2621
3763
|
IndexedDbKvRepository,
|
|
2622
3764
|
InMemoryTabularRepository,
|
|
3765
|
+
InMemoryRateLimiterStorage,
|
|
2623
3766
|
InMemoryQueueStorage,
|
|
2624
3767
|
InMemoryKvRepository,
|
|
3768
|
+
IN_MEMORY_RATE_LIMITER_STORAGE,
|
|
2625
3769
|
IN_MEMORY_QUEUE_STORAGE,
|
|
3770
|
+
INDEXED_DB_RATE_LIMITER_STORAGE,
|
|
2626
3771
|
INDEXED_DB_QUEUE_STORAGE,
|
|
2627
3772
|
IDB_TABULAR_REPOSITORY,
|
|
2628
3773
|
IDB_KV_REPOSITORY,
|
|
@@ -2632,4 +3777,4 @@ export {
|
|
|
2632
3777
|
CACHED_TABULAR_REPOSITORY
|
|
2633
3778
|
};
|
|
2634
3779
|
|
|
2635
|
-
//# debugId=
|
|
3780
|
+
//# debugId=91B1809D834FE66964756E2164756E21
|