@workglow/storage 0.0.57 → 0.0.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +1266 -121
- package/dist/browser.js.map +15 -10
- package/dist/bun.js +2132 -267
- package/dist/bun.js.map +20 -13
- package/dist/common-server.d.ts +4 -0
- package/dist/common-server.d.ts.map +1 -1
- package/dist/common.d.ts +2 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/limiter/IRateLimiterStorage.d.ts +81 -0
- package/dist/limiter/IRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/InMemoryRateLimiterStorage.d.ts +32 -0
- package/dist/limiter/InMemoryRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/IndexedDbRateLimiterStorage.d.ts +52 -0
- package/dist/limiter/IndexedDbRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/PostgresRateLimiterStorage.d.ts +54 -0
- package/dist/limiter/PostgresRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/SqliteRateLimiterStorage.d.ts +53 -0
- package/dist/limiter/SqliteRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/SupabaseRateLimiterStorage.d.ts +53 -0
- package/dist/limiter/SupabaseRateLimiterStorage.d.ts.map +1 -0
- package/dist/node.js +2132 -267
- package/dist/node.js.map +20 -13
- package/dist/queue/IQueueStorage.d.ts +72 -1
- package/dist/queue/IQueueStorage.d.ts.map +1 -1
- package/dist/queue/InMemoryQueueStorage.d.ts +44 -11
- package/dist/queue/InMemoryQueueStorage.d.ts.map +1 -1
- package/dist/queue/IndexedDbQueueStorage.d.ts +70 -5
- package/dist/queue/IndexedDbQueueStorage.d.ts.map +1 -1
- package/dist/queue/PostgresQueueStorage.d.ts +80 -8
- package/dist/queue/PostgresQueueStorage.d.ts.map +1 -1
- package/dist/queue/SqliteQueueStorage.d.ts +90 -34
- package/dist/queue/SqliteQueueStorage.d.ts.map +1 -1
- package/dist/queue/SupabaseQueueStorage.d.ts +98 -4
- package/dist/queue/SupabaseQueueStorage.d.ts.map +1 -1
- package/dist/tabular/ITabularRepository.d.ts +18 -0
- package/dist/tabular/ITabularRepository.d.ts.map +1 -1
- package/dist/tabular/InMemoryTabularRepository.d.ts +9 -1
- package/dist/tabular/InMemoryTabularRepository.d.ts.map +1 -1
- package/dist/tabular/SqliteTabularRepository.d.ts.map +1 -1
- package/dist/tabular/SupabaseTabularRepository.d.ts +21 -1
- package/dist/tabular/SupabaseTabularRepository.d.ts.map +1 -1
- package/dist/tabular/TabularRepository.d.ts +10 -1
- package/dist/tabular/TabularRepository.d.ts.map +1 -1
- package/dist/util/PollingSubscriptionManager.d.ts +112 -0
- package/dist/util/PollingSubscriptionManager.d.ts.map +1 -0
- package/package.json +5 -5
package/dist/node.js
CHANGED
|
@@ -106,6 +106,9 @@ class TabularRepository {
|
|
|
106
106
|
waitOn(name) {
|
|
107
107
|
return this.events.waitOn(name);
|
|
108
108
|
}
|
|
109
|
+
subscribeToChanges(_callback) {
|
|
110
|
+
throw new Error(`subscribeToChanges is not implemented for ${this.constructor.name}. ` + `Use InMemoryTabularRepository or SupabaseTabularRepository for subscription support.`);
|
|
111
|
+
}
|
|
109
112
|
primaryKeyColumns() {
|
|
110
113
|
const columns = [];
|
|
111
114
|
for (const key of Object.keys(this.primaryKeySchema.properties)) {
|
|
@@ -273,12 +276,30 @@ class InMemoryTabularRepository extends TabularRepository {
|
|
|
273
276
|
return false;
|
|
274
277
|
}
|
|
275
278
|
});
|
|
276
|
-
for (const [id,
|
|
279
|
+
for (const [id, entity] of entriesToDelete) {
|
|
277
280
|
this.values.delete(id);
|
|
281
|
+
const { key } = this.separateKeyValueFromCombined(entity);
|
|
282
|
+
this.events.emit("delete", key);
|
|
278
283
|
}
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
284
|
+
}
|
|
285
|
+
subscribeToChanges(callback) {
|
|
286
|
+
const handlePut = (entity) => {
|
|
287
|
+
callback({ type: "UPDATE", new: entity });
|
|
288
|
+
};
|
|
289
|
+
const handleDelete = (_key) => {
|
|
290
|
+
callback({ type: "DELETE" });
|
|
291
|
+
};
|
|
292
|
+
const handleClearAll = () => {
|
|
293
|
+
callback({ type: "DELETE" });
|
|
294
|
+
};
|
|
295
|
+
this.events.on("put", handlePut);
|
|
296
|
+
this.events.on("delete", handleDelete);
|
|
297
|
+
this.events.on("clearall", handleClearAll);
|
|
298
|
+
return () => {
|
|
299
|
+
this.events.off("put", handlePut);
|
|
300
|
+
this.events.off("delete", handleDelete);
|
|
301
|
+
this.events.off("clearall", handleClearAll);
|
|
302
|
+
};
|
|
282
303
|
}
|
|
283
304
|
destroy() {
|
|
284
305
|
this.values.clear();
|
|
@@ -540,7 +561,7 @@ class InMemoryKvRepository extends KvViaTabularRepository {
|
|
|
540
561
|
}
|
|
541
562
|
}
|
|
542
563
|
// src/queue/InMemoryQueueStorage.ts
|
|
543
|
-
import { createServiceToken as createServiceToken7, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
|
|
564
|
+
import { createServiceToken as createServiceToken7, EventEmitter as EventEmitter3, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
|
|
544
565
|
|
|
545
566
|
// src/queue/IQueueStorage.ts
|
|
546
567
|
import { createServiceToken as createServiceToken6 } from "@workglow/util";
|
|
@@ -560,63 +581,88 @@ var IN_MEMORY_QUEUE_STORAGE = createServiceToken7("jobqueue.storage.inMemory");
|
|
|
560
581
|
|
|
561
582
|
class InMemoryQueueStorage {
|
|
562
583
|
queueName;
|
|
563
|
-
|
|
584
|
+
prefixValues;
|
|
585
|
+
events = new EventEmitter3;
|
|
586
|
+
constructor(queueName, options) {
|
|
564
587
|
this.queueName = queueName;
|
|
565
588
|
this.jobQueue = [];
|
|
589
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
566
590
|
}
|
|
567
591
|
jobQueue;
|
|
592
|
+
matchesPrefixes(job) {
|
|
593
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
594
|
+
if (job[key] !== value) {
|
|
595
|
+
return false;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
return true;
|
|
599
|
+
}
|
|
568
600
|
pendingQueue() {
|
|
569
601
|
const now = new Date().toISOString();
|
|
570
|
-
return this.jobQueue.filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
602
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
571
603
|
}
|
|
572
604
|
async add(job) {
|
|
573
605
|
await sleep(0);
|
|
574
606
|
const now = new Date().toISOString();
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
607
|
+
const jobWithPrefixes = job;
|
|
608
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid4();
|
|
609
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid4();
|
|
610
|
+
jobWithPrefixes.queue = this.queueName;
|
|
611
|
+
jobWithPrefixes.fingerprint = await makeFingerprint4(jobWithPrefixes.input);
|
|
612
|
+
jobWithPrefixes.status = "PENDING" /* PENDING */;
|
|
613
|
+
jobWithPrefixes.progress = 0;
|
|
614
|
+
jobWithPrefixes.progress_message = "";
|
|
615
|
+
jobWithPrefixes.progress_details = null;
|
|
616
|
+
jobWithPrefixes.created_at = now;
|
|
617
|
+
jobWithPrefixes.run_after = now;
|
|
618
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
619
|
+
jobWithPrefixes[key] = value;
|
|
620
|
+
}
|
|
621
|
+
this.jobQueue.push(jobWithPrefixes);
|
|
622
|
+
this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
|
|
623
|
+
return jobWithPrefixes.id;
|
|
587
624
|
}
|
|
588
625
|
async get(id) {
|
|
589
626
|
await sleep(0);
|
|
590
|
-
|
|
627
|
+
const job = this.jobQueue.find((j) => j.id === id);
|
|
628
|
+
if (job && this.matchesPrefixes(job)) {
|
|
629
|
+
return job;
|
|
630
|
+
}
|
|
631
|
+
return;
|
|
591
632
|
}
|
|
592
633
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
593
634
|
await sleep(0);
|
|
594
635
|
num = Number(num) || 100;
|
|
595
|
-
return this.jobQueue.sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
636
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
596
637
|
}
|
|
597
|
-
async next() {
|
|
638
|
+
async next(workerId) {
|
|
598
639
|
await sleep(0);
|
|
599
640
|
const top = this.pendingQueue();
|
|
600
641
|
const job = top[0];
|
|
601
642
|
if (job) {
|
|
643
|
+
const oldJob = { ...job };
|
|
602
644
|
job.status = "PROCESSING" /* PROCESSING */;
|
|
603
645
|
job.last_ran_at = new Date().toISOString();
|
|
646
|
+
job.worker_id = workerId ?? null;
|
|
647
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
604
648
|
return job;
|
|
605
649
|
}
|
|
606
650
|
}
|
|
607
651
|
async size(status = "PENDING" /* PENDING */) {
|
|
608
652
|
await sleep(0);
|
|
609
|
-
return this.jobQueue.filter((j) => j.status === status).length;
|
|
653
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
|
|
610
654
|
}
|
|
611
655
|
async saveProgress(id, progress, message, details) {
|
|
612
656
|
await sleep(0);
|
|
613
|
-
const job = this.jobQueue.find((j) => j.id === id);
|
|
657
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
614
658
|
if (!job) {
|
|
615
659
|
throw new Error(`Job ${id} not found`);
|
|
616
660
|
}
|
|
661
|
+
const oldJob = { ...job };
|
|
617
662
|
job.progress = progress;
|
|
618
663
|
job.progress_message = message;
|
|
619
664
|
job.progress_details = details;
|
|
665
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
620
666
|
}
|
|
621
667
|
async complete(job) {
|
|
622
668
|
await sleep(0);
|
|
@@ -626,44 +672,149 @@ class InMemoryQueueStorage {
|
|
|
626
672
|
const currentAttempts = existing?.run_attempts ?? 0;
|
|
627
673
|
job.run_attempts = currentAttempts + 1;
|
|
628
674
|
this.jobQueue[index] = job;
|
|
675
|
+
this.events.emit("change", { type: "UPDATE", old: existing, new: job });
|
|
629
676
|
}
|
|
630
677
|
}
|
|
631
678
|
async abort(id) {
|
|
632
679
|
await sleep(0);
|
|
633
|
-
const job = this.jobQueue.find((j) => j.id === id);
|
|
680
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
634
681
|
if (job) {
|
|
682
|
+
const oldJob = { ...job };
|
|
635
683
|
job.status = "ABORTING" /* ABORTING */;
|
|
684
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
636
685
|
}
|
|
637
686
|
}
|
|
638
687
|
async getByRunId(runId) {
|
|
639
688
|
await sleep(0);
|
|
640
|
-
return this.jobQueue.filter((job) => job.job_run_id === runId);
|
|
689
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
|
|
641
690
|
}
|
|
642
691
|
async deleteAll() {
|
|
643
692
|
await sleep(0);
|
|
644
|
-
this.jobQueue
|
|
693
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
|
|
694
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
|
|
695
|
+
for (const job of deletedJobs) {
|
|
696
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
697
|
+
}
|
|
645
698
|
}
|
|
646
699
|
async outputForInput(input) {
|
|
647
700
|
await sleep(0);
|
|
648
701
|
const fingerprint = await makeFingerprint4(input);
|
|
649
|
-
return this.jobQueue.find((j) => j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
|
|
702
|
+
return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
|
|
650
703
|
}
|
|
651
704
|
async delete(id) {
|
|
652
705
|
await sleep(0);
|
|
653
|
-
|
|
706
|
+
const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
|
|
707
|
+
this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
|
|
708
|
+
if (deletedJob) {
|
|
709
|
+
this.events.emit("change", { type: "DELETE", old: deletedJob });
|
|
710
|
+
}
|
|
654
711
|
}
|
|
655
712
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
656
713
|
await sleep(0);
|
|
657
714
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
658
|
-
|
|
715
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
|
|
716
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
|
|
717
|
+
for (const job of deletedJobs) {
|
|
718
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
async setupDatabase() {}
|
|
722
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
723
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
724
|
+
return true;
|
|
725
|
+
}
|
|
726
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
727
|
+
if (Object.keys(filterValues).length === 0) {
|
|
728
|
+
return true;
|
|
729
|
+
}
|
|
730
|
+
const jobWithPrefixes = job;
|
|
731
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
732
|
+
if (jobWithPrefixes[key] !== value) {
|
|
733
|
+
return false;
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
return true;
|
|
737
|
+
}
|
|
738
|
+
subscribeToChanges(callback, options) {
|
|
739
|
+
const prefixFilter = options?.prefixFilter;
|
|
740
|
+
const filteredCallback = (change) => {
|
|
741
|
+
const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
|
|
742
|
+
const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
|
|
743
|
+
if (!newMatches && !oldMatches) {
|
|
744
|
+
return;
|
|
745
|
+
}
|
|
746
|
+
callback(change);
|
|
747
|
+
};
|
|
748
|
+
return this.events.subscribe("change", filteredCallback);
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
// src/limiter/IRateLimiterStorage.ts
|
|
752
|
+
import { createServiceToken as createServiceToken8 } from "@workglow/util";
|
|
753
|
+
var RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage");
|
|
754
|
+
// src/limiter/InMemoryRateLimiterStorage.ts
|
|
755
|
+
import { createServiceToken as createServiceToken9, sleep as sleep2 } from "@workglow/util";
|
|
756
|
+
var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken9("ratelimiter.storage.inMemory");
|
|
757
|
+
|
|
758
|
+
class InMemoryRateLimiterStorage {
|
|
759
|
+
prefixValues;
|
|
760
|
+
executions = new Map;
|
|
761
|
+
nextAvailableTimes = new Map;
|
|
762
|
+
constructor(options) {
|
|
763
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
764
|
+
}
|
|
765
|
+
makeKey(queueName) {
|
|
766
|
+
const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
|
|
767
|
+
return prefixPart ? `${prefixPart}|${queueName}` : queueName;
|
|
659
768
|
}
|
|
660
769
|
async setupDatabase() {}
|
|
770
|
+
async recordExecution(queueName) {
|
|
771
|
+
await sleep2(0);
|
|
772
|
+
const key = this.makeKey(queueName);
|
|
773
|
+
const executions = this.executions.get(key) ?? [];
|
|
774
|
+
executions.push({
|
|
775
|
+
queueName,
|
|
776
|
+
executedAt: new Date
|
|
777
|
+
});
|
|
778
|
+
this.executions.set(key, executions);
|
|
779
|
+
}
|
|
780
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
781
|
+
await sleep2(0);
|
|
782
|
+
const key = this.makeKey(queueName);
|
|
783
|
+
const executions = this.executions.get(key) ?? [];
|
|
784
|
+
const windowStart = new Date(windowStartTime);
|
|
785
|
+
return executions.filter((e) => e.executedAt > windowStart).length;
|
|
786
|
+
}
|
|
787
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
788
|
+
await sleep2(0);
|
|
789
|
+
const key = this.makeKey(queueName);
|
|
790
|
+
const executions = this.executions.get(key) ?? [];
|
|
791
|
+
const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
|
|
792
|
+
const execution = sorted[offset];
|
|
793
|
+
return execution?.executedAt.toISOString();
|
|
794
|
+
}
|
|
795
|
+
async getNextAvailableTime(queueName) {
|
|
796
|
+
await sleep2(0);
|
|
797
|
+
const key = this.makeKey(queueName);
|
|
798
|
+
const time = this.nextAvailableTimes.get(key);
|
|
799
|
+
return time?.toISOString();
|
|
800
|
+
}
|
|
801
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
802
|
+
await sleep2(0);
|
|
803
|
+
const key = this.makeKey(queueName);
|
|
804
|
+
this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
|
|
805
|
+
}
|
|
806
|
+
async clear(queueName) {
|
|
807
|
+
await sleep2(0);
|
|
808
|
+
const key = this.makeKey(queueName);
|
|
809
|
+
this.executions.delete(key);
|
|
810
|
+
this.nextAvailableTimes.delete(key);
|
|
811
|
+
}
|
|
661
812
|
}
|
|
662
813
|
// src/tabular/FsFolderTabularRepository.ts
|
|
663
|
-
import { createServiceToken as
|
|
814
|
+
import { createServiceToken as createServiceToken10, sleep as sleep3 } from "@workglow/util";
|
|
664
815
|
import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
665
816
|
import path from "node:path";
|
|
666
|
-
var FS_FOLDER_TABULAR_REPOSITORY =
|
|
817
|
+
var FS_FOLDER_TABULAR_REPOSITORY = createServiceToken10("storage.tabularRepository.fsFolder");
|
|
667
818
|
|
|
668
819
|
class FsFolderTabularRepository extends TabularRepository {
|
|
669
820
|
folderPath;
|
|
@@ -688,7 +839,7 @@ class FsFolderTabularRepository extends TabularRepository {
|
|
|
688
839
|
await writeFile(filePath, JSON.stringify(entity));
|
|
689
840
|
} catch (error) {
|
|
690
841
|
try {
|
|
691
|
-
await
|
|
842
|
+
await sleep3(1);
|
|
692
843
|
await writeFile(filePath, JSON.stringify(entity));
|
|
693
844
|
} catch (error2) {
|
|
694
845
|
console.error("Error writing file", filePath, error2);
|
|
@@ -772,7 +923,7 @@ class FsFolderTabularRepository extends TabularRepository {
|
|
|
772
923
|
}
|
|
773
924
|
}
|
|
774
925
|
// src/tabular/PostgresTabularRepository.ts
|
|
775
|
-
import { createServiceToken as
|
|
926
|
+
import { createServiceToken as createServiceToken11 } from "@workglow/util";
|
|
776
927
|
|
|
777
928
|
// src/tabular/BaseSqlTabularRepository.ts
|
|
778
929
|
class BaseSqlTabularRepository extends TabularRepository {
|
|
@@ -958,7 +1109,7 @@ class BaseSqlTabularRepository extends TabularRepository {
|
|
|
958
1109
|
}
|
|
959
1110
|
|
|
960
1111
|
// src/tabular/PostgresTabularRepository.ts
|
|
961
|
-
var POSTGRES_TABULAR_REPOSITORY =
|
|
1112
|
+
var POSTGRES_TABULAR_REPOSITORY = createServiceToken11("storage.tabularRepository.postgres");
|
|
962
1113
|
|
|
963
1114
|
class PostgresTabularRepository extends BaseSqlTabularRepository {
|
|
964
1115
|
db;
|
|
@@ -1316,8 +1467,8 @@ class PostgresTabularRepository extends BaseSqlTabularRepository {
|
|
|
1316
1467
|
}
|
|
1317
1468
|
// src/tabular/SqliteTabularRepository.ts
|
|
1318
1469
|
import { Sqlite } from "@workglow/sqlite";
|
|
1319
|
-
import { createServiceToken as
|
|
1320
|
-
var SQLITE_TABULAR_REPOSITORY =
|
|
1470
|
+
import { createServiceToken as createServiceToken12 } from "@workglow/util";
|
|
1471
|
+
var SQLITE_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.sqlite");
|
|
1321
1472
|
var Database = Sqlite.Database;
|
|
1322
1473
|
|
|
1323
1474
|
class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
@@ -1368,13 +1519,31 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1368
1519
|
return this.db;
|
|
1369
1520
|
}
|
|
1370
1521
|
jsToSqlValue(column, value) {
|
|
1522
|
+
if (value !== null && value !== undefined && typeof value === "object") {
|
|
1523
|
+
if (value instanceof Date) {
|
|
1524
|
+
return super.jsToSqlValue(column, value);
|
|
1525
|
+
}
|
|
1526
|
+
if (value instanceof Uint8Array) {
|
|
1527
|
+
return super.jsToSqlValue(column, value);
|
|
1528
|
+
}
|
|
1529
|
+
if (typeof Buffer !== "undefined" && value instanceof Buffer) {
|
|
1530
|
+
return super.jsToSqlValue(column, value);
|
|
1531
|
+
}
|
|
1532
|
+
return JSON.stringify(value);
|
|
1533
|
+
}
|
|
1534
|
+
if (value === null) {
|
|
1535
|
+
const typeDef2 = this.schema.properties[column];
|
|
1536
|
+
if (typeDef2 && this.isNullable(typeDef2)) {
|
|
1537
|
+
return null;
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1371
1540
|
const typeDef = this.schema.properties[column];
|
|
1372
1541
|
if (typeDef) {
|
|
1373
1542
|
const actualType = this.getNonNullType(typeDef);
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1543
|
+
const isObject = typeDef === true || typeof actualType !== "boolean" && actualType.type === "object";
|
|
1544
|
+
const isArray = typeDef === true || typeof actualType !== "boolean" && actualType.type === "array";
|
|
1545
|
+
const isBoolean = typeDef === true || typeof actualType !== "boolean" && actualType.type === "boolean";
|
|
1546
|
+
if (isBoolean) {
|
|
1378
1547
|
const v = value;
|
|
1379
1548
|
if (typeof v === "boolean")
|
|
1380
1549
|
return v ? 1 : 0;
|
|
@@ -1383,8 +1552,22 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1383
1552
|
if (typeof v === "string")
|
|
1384
1553
|
return v === "1" || v.toLowerCase() === "true" ? 1 : 0;
|
|
1385
1554
|
}
|
|
1555
|
+
if ((isObject || isArray) && value !== null && typeof value === "object") {
|
|
1556
|
+
if (!(value instanceof Date) && !(value instanceof Uint8Array) && (typeof Buffer === "undefined" || !(value instanceof Buffer))) {
|
|
1557
|
+
if (Array.isArray(value) || Object.getPrototypeOf(value) === Object.prototype) {
|
|
1558
|
+
return JSON.stringify(value);
|
|
1559
|
+
}
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
const result = super.jsToSqlValue(column, value);
|
|
1564
|
+
if (result !== null && typeof result === "object") {
|
|
1565
|
+
const resultObj = result;
|
|
1566
|
+
if (!(resultObj instanceof Uint8Array) && (typeof Buffer === "undefined" || !(resultObj instanceof Buffer))) {
|
|
1567
|
+
return JSON.stringify(resultObj);
|
|
1568
|
+
}
|
|
1386
1569
|
}
|
|
1387
|
-
return
|
|
1570
|
+
return result;
|
|
1388
1571
|
}
|
|
1389
1572
|
sqlToJsValue(column, value) {
|
|
1390
1573
|
const typeDef = this.schema.properties[column];
|
|
@@ -1393,14 +1576,27 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1393
1576
|
return null;
|
|
1394
1577
|
}
|
|
1395
1578
|
const actualType = this.getNonNullType(typeDef);
|
|
1396
|
-
|
|
1579
|
+
const isObject = typeDef === true || typeof actualType !== "boolean" && actualType.type === "object";
|
|
1580
|
+
const isArray = typeDef === true || typeof actualType !== "boolean" && actualType.type === "array";
|
|
1581
|
+
const isBoolean = typeDef === true || typeof actualType !== "boolean" && actualType.type === "boolean";
|
|
1582
|
+
if (isBoolean) {
|
|
1397
1583
|
const v = value;
|
|
1398
1584
|
if (typeof v === "boolean")
|
|
1399
1585
|
return v;
|
|
1400
1586
|
if (typeof v === "number")
|
|
1401
|
-
return v !== 0;
|
|
1587
|
+
return v !== 0 ? true : false;
|
|
1402
1588
|
if (typeof v === "string")
|
|
1403
|
-
return v === "1" || v.toLowerCase() === "true";
|
|
1589
|
+
return v === "1" || v.toLowerCase() === "true" ? true : false;
|
|
1590
|
+
}
|
|
1591
|
+
if (isArray || isObject) {
|
|
1592
|
+
if (typeof value === "string") {
|
|
1593
|
+
try {
|
|
1594
|
+
return JSON.parse(value);
|
|
1595
|
+
} catch (e) {
|
|
1596
|
+
return value;
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
return value;
|
|
1404
1600
|
}
|
|
1405
1601
|
}
|
|
1406
1602
|
return super.sqlToJsValue(column, value);
|
|
@@ -1452,6 +1648,50 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1452
1648
|
const primaryKeyParams = this.getPrimaryKeyAsOrderedArray(key);
|
|
1453
1649
|
const valueParams = this.getValueAsOrderedArray(value);
|
|
1454
1650
|
const params = [...primaryKeyParams, ...valueParams];
|
|
1651
|
+
for (let i = 0;i < params.length; i++) {
|
|
1652
|
+
let param = params[i];
|
|
1653
|
+
if (param === undefined) {
|
|
1654
|
+
params[i] = null;
|
|
1655
|
+
continue;
|
|
1656
|
+
}
|
|
1657
|
+
if (param !== null && typeof param === "object") {
|
|
1658
|
+
const paramObj = param;
|
|
1659
|
+
if (paramObj instanceof Uint8Array) {
|
|
1660
|
+
continue;
|
|
1661
|
+
}
|
|
1662
|
+
if (typeof Buffer !== "undefined" && paramObj instanceof Buffer) {
|
|
1663
|
+
params[i] = new Uint8Array(paramObj);
|
|
1664
|
+
continue;
|
|
1665
|
+
}
|
|
1666
|
+
try {
|
|
1667
|
+
params[i] = JSON.stringify(paramObj);
|
|
1668
|
+
} catch (e) {
|
|
1669
|
+
throw new Error(`Failed to stringify param at index ${i} for column binding: ${String(e)}`);
|
|
1670
|
+
}
|
|
1671
|
+
continue;
|
|
1672
|
+
}
|
|
1673
|
+
}
|
|
1674
|
+
const invalidParams = [];
|
|
1675
|
+
for (let i = 0;i < params.length; i++) {
|
|
1676
|
+
const param = params[i];
|
|
1677
|
+
if (param === null || param === undefined || typeof param === "string" || typeof param === "number" || typeof param === "boolean" || typeof param === "bigint") {
|
|
1678
|
+
continue;
|
|
1679
|
+
}
|
|
1680
|
+
if (typeof param === "object") {
|
|
1681
|
+
const paramObj = param;
|
|
1682
|
+
if (paramObj instanceof Uint8Array || typeof Buffer !== "undefined" && paramObj instanceof Buffer) {
|
|
1683
|
+
continue;
|
|
1684
|
+
}
|
|
1685
|
+
invalidParams.push({ index: i, type: typeof param, value: param });
|
|
1686
|
+
} else {
|
|
1687
|
+
invalidParams.push({ index: i, type: typeof param, value: param });
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
if (invalidParams.length > 0) {
|
|
1691
|
+
console.error("Invalid params detected:", invalidParams);
|
|
1692
|
+
console.error("All params:", params.map((p, i) => ({ i, type: typeof p, value: p, isArray: Array.isArray(p) })));
|
|
1693
|
+
throw new Error(`Invalid SQLite params detected at indices: ${invalidParams.map((p) => p.index).join(", ")}`);
|
|
1694
|
+
}
|
|
1455
1695
|
const updatedEntity = stmt.get(...params);
|
|
1456
1696
|
for (const k in this.schema.properties) {
|
|
1457
1697
|
updatedEntity[k] = this.sqlToJsValue(k, updatedEntity[k]);
|
|
@@ -1479,6 +1719,17 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1479
1719
|
const primaryKeyParams = this.getPrimaryKeyAsOrderedArray(key);
|
|
1480
1720
|
const valueParams = this.getValueAsOrderedArray(value);
|
|
1481
1721
|
const params = [...primaryKeyParams, ...valueParams];
|
|
1722
|
+
for (let i = 0;i < params.length; i++) {
|
|
1723
|
+
let param = params[i];
|
|
1724
|
+
if (param === undefined) {
|
|
1725
|
+
params[i] = null;
|
|
1726
|
+
} else if (param !== null && typeof param === "object") {
|
|
1727
|
+
const paramObj = param;
|
|
1728
|
+
if (!(paramObj instanceof Uint8Array) && (typeof Buffer === "undefined" || !(paramObj instanceof Buffer))) {
|
|
1729
|
+
params[i] = JSON.stringify(paramObj);
|
|
1730
|
+
}
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1482
1733
|
const updatedEntity = stmt.get(...params);
|
|
1483
1734
|
for (const k in this.schema.properties) {
|
|
1484
1735
|
updatedEntity[k] = this.sqlToJsValue(k, updatedEntity[k]);
|
|
@@ -1594,11 +1845,12 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1594
1845
|
}
|
|
1595
1846
|
}
|
|
1596
1847
|
// src/tabular/SupabaseTabularRepository.ts
|
|
1597
|
-
import { createServiceToken as
|
|
1598
|
-
var SUPABASE_TABULAR_REPOSITORY =
|
|
1848
|
+
import { createServiceToken as createServiceToken13 } from "@workglow/util";
|
|
1849
|
+
var SUPABASE_TABULAR_REPOSITORY = createServiceToken13("storage.tabularRepository.supabase");
|
|
1599
1850
|
|
|
1600
1851
|
class SupabaseTabularRepository extends BaseSqlTabularRepository {
|
|
1601
1852
|
client;
|
|
1853
|
+
realtimeChannel = null;
|
|
1602
1854
|
constructor(client, table = "tabular_store", schema, primaryKeyNames, indexes = []) {
|
|
1603
1855
|
super(table, schema, primaryKeyNames, indexes);
|
|
1604
1856
|
this.client = client;
|
|
@@ -1971,10 +2223,44 @@ class SupabaseTabularRepository extends BaseSqlTabularRepository {
|
|
|
1971
2223
|
throw error;
|
|
1972
2224
|
this.events.emit("delete", column);
|
|
1973
2225
|
}
|
|
2226
|
+
convertRealtimeRow(row) {
|
|
2227
|
+
const entity = { ...row };
|
|
2228
|
+
for (const key in this.schema.properties) {
|
|
2229
|
+
entity[key] = this.sqlToJsValue(key, row[key]);
|
|
2230
|
+
}
|
|
2231
|
+
return entity;
|
|
2232
|
+
}
|
|
2233
|
+
subscribeToChanges(callback) {
|
|
2234
|
+
const channelName = `tabular-${this.table}-${Date.now()}`;
|
|
2235
|
+
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
2236
|
+
event: "*",
|
|
2237
|
+
schema: "public",
|
|
2238
|
+
table: this.table
|
|
2239
|
+
}, (payload) => {
|
|
2240
|
+
const change = {
|
|
2241
|
+
type: payload.eventType.toUpperCase(),
|
|
2242
|
+
old: payload.old && Object.keys(payload.old).length > 0 ? this.convertRealtimeRow(payload.old) : undefined,
|
|
2243
|
+
new: payload.new && Object.keys(payload.new).length > 0 ? this.convertRealtimeRow(payload.new) : undefined
|
|
2244
|
+
};
|
|
2245
|
+
callback(change);
|
|
2246
|
+
}).subscribe();
|
|
2247
|
+
return () => {
|
|
2248
|
+
if (this.realtimeChannel) {
|
|
2249
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
2250
|
+
this.realtimeChannel = null;
|
|
2251
|
+
}
|
|
2252
|
+
};
|
|
2253
|
+
}
|
|
2254
|
+
destroy() {
|
|
2255
|
+
if (this.realtimeChannel) {
|
|
2256
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
2257
|
+
this.realtimeChannel = null;
|
|
2258
|
+
}
|
|
2259
|
+
}
|
|
1974
2260
|
}
|
|
1975
2261
|
// src/kv/FsFolderJsonKvRepository.ts
|
|
1976
|
-
import { createServiceToken as
|
|
1977
|
-
var FS_FOLDER_JSON_KV_REPOSITORY =
|
|
2262
|
+
import { createServiceToken as createServiceToken14 } from "@workglow/util";
|
|
2263
|
+
var FS_FOLDER_JSON_KV_REPOSITORY = createServiceToken14("storage.kvRepository.fsFolderJson");
|
|
1978
2264
|
|
|
1979
2265
|
class FsFolderJsonKvRepository extends KvViaTabularRepository {
|
|
1980
2266
|
folderPath;
|
|
@@ -1986,10 +2272,10 @@ class FsFolderJsonKvRepository extends KvViaTabularRepository {
|
|
|
1986
2272
|
}
|
|
1987
2273
|
}
|
|
1988
2274
|
// src/kv/FsFolderKvRepository.ts
|
|
1989
|
-
import { createServiceToken as
|
|
2275
|
+
import { createServiceToken as createServiceToken15 } from "@workglow/util";
|
|
1990
2276
|
import { mkdir as mkdir2, readFile as readFile2, rm as rm2, unlink, writeFile as writeFile2 } from "fs/promises";
|
|
1991
2277
|
import path2 from "path";
|
|
1992
|
-
var FS_FOLDER_KV_REPOSITORY =
|
|
2278
|
+
var FS_FOLDER_KV_REPOSITORY = createServiceToken15("storage.kvRepository.fsFolder");
|
|
1993
2279
|
|
|
1994
2280
|
class FsFolderKvRepository extends KvRepository {
|
|
1995
2281
|
folderPath;
|
|
@@ -2066,8 +2352,8 @@ class FsFolderKvRepository extends KvRepository {
|
|
|
2066
2352
|
}
|
|
2067
2353
|
}
|
|
2068
2354
|
// src/kv/PostgresKvRepository.ts
|
|
2069
|
-
import { createServiceToken as
|
|
2070
|
-
var POSTGRES_KV_REPOSITORY =
|
|
2355
|
+
import { createServiceToken as createServiceToken16 } from "@workglow/util";
|
|
2356
|
+
var POSTGRES_KV_REPOSITORY = createServiceToken16("storage.kvRepository.postgres");
|
|
2071
2357
|
|
|
2072
2358
|
class PostgresKvRepository extends KvViaTabularRepository {
|
|
2073
2359
|
db;
|
|
@@ -2081,8 +2367,8 @@ class PostgresKvRepository extends KvViaTabularRepository {
|
|
|
2081
2367
|
}
|
|
2082
2368
|
}
|
|
2083
2369
|
// src/kv/SqliteKvRepository.ts
|
|
2084
|
-
import { createServiceToken as
|
|
2085
|
-
var SQLITE_KV_REPOSITORY =
|
|
2370
|
+
import { createServiceToken as createServiceToken17 } from "@workglow/util";
|
|
2371
|
+
var SQLITE_KV_REPOSITORY = createServiceToken17("storage.kvRepository.sqlite");
|
|
2086
2372
|
|
|
2087
2373
|
class SqliteKvRepository extends KvViaTabularRepository {
|
|
2088
2374
|
db;
|
|
@@ -2096,8 +2382,8 @@ class SqliteKvRepository extends KvViaTabularRepository {
|
|
|
2096
2382
|
}
|
|
2097
2383
|
}
|
|
2098
2384
|
// src/kv/SupabaseKvRepository.ts
|
|
2099
|
-
import { createServiceToken as
|
|
2100
|
-
var SUPABASE_KV_REPOSITORY =
|
|
2385
|
+
import { createServiceToken as createServiceToken18 } from "@workglow/util";
|
|
2386
|
+
var SUPABASE_KV_REPOSITORY = createServiceToken18("storage.kvRepository.supabase");
|
|
2101
2387
|
|
|
2102
2388
|
class SupabaseKvRepository extends KvViaTabularRepository {
|
|
2103
2389
|
client;
|
|
@@ -2111,15 +2397,169 @@ class SupabaseKvRepository extends KvViaTabularRepository {
|
|
|
2111
2397
|
}
|
|
2112
2398
|
}
|
|
2113
2399
|
// src/queue/PostgresQueueStorage.ts
|
|
2114
|
-
import { createServiceToken as
|
|
2115
|
-
|
|
2400
|
+
import { createServiceToken as createServiceToken19, makeFingerprint as makeFingerprint5, uuid4 as uuid42 } from "@workglow/util";
|
|
2401
|
+
|
|
2402
|
+
// src/util/PollingSubscriptionManager.ts
|
|
2403
|
+
class PollingSubscriptionManager {
|
|
2404
|
+
intervals = new Map;
|
|
2405
|
+
lastKnownState = new Map;
|
|
2406
|
+
initialized = false;
|
|
2407
|
+
fetchState;
|
|
2408
|
+
compareItems;
|
|
2409
|
+
payloadFactory;
|
|
2410
|
+
defaultIntervalMs;
|
|
2411
|
+
constructor(fetchState, compareItems, payloadFactory, options) {
|
|
2412
|
+
this.fetchState = fetchState;
|
|
2413
|
+
this.compareItems = compareItems;
|
|
2414
|
+
this.payloadFactory = payloadFactory;
|
|
2415
|
+
this.defaultIntervalMs = options?.defaultIntervalMs ?? 1000;
|
|
2416
|
+
}
|
|
2417
|
+
subscribe(callback, options) {
|
|
2418
|
+
const interval = options?.intervalMs ?? this.defaultIntervalMs;
|
|
2419
|
+
const subscription = {
|
|
2420
|
+
callback,
|
|
2421
|
+
intervalMs: interval
|
|
2422
|
+
};
|
|
2423
|
+
let intervalGroup = this.intervals.get(interval);
|
|
2424
|
+
if (!intervalGroup) {
|
|
2425
|
+
const subscribers = new Set;
|
|
2426
|
+
const intervalId = setInterval(() => this.poll(subscribers), interval);
|
|
2427
|
+
intervalGroup = { intervalId, subscribers };
|
|
2428
|
+
this.intervals.set(interval, intervalGroup);
|
|
2429
|
+
if (!this.initialized) {
|
|
2430
|
+
this.initialized = true;
|
|
2431
|
+
this.initAndPoll(subscribers, subscription);
|
|
2432
|
+
} else {
|
|
2433
|
+
this.pollForNewSubscriber(subscription);
|
|
2434
|
+
}
|
|
2435
|
+
} else {
|
|
2436
|
+
this.pollForNewSubscriber(subscription);
|
|
2437
|
+
}
|
|
2438
|
+
intervalGroup.subscribers.add(subscription);
|
|
2439
|
+
return () => {
|
|
2440
|
+
const group = this.intervals.get(interval);
|
|
2441
|
+
if (group) {
|
|
2442
|
+
group.subscribers.delete(subscription);
|
|
2443
|
+
if (group.subscribers.size === 0) {
|
|
2444
|
+
clearInterval(group.intervalId);
|
|
2445
|
+
this.intervals.delete(interval);
|
|
2446
|
+
}
|
|
2447
|
+
}
|
|
2448
|
+
};
|
|
2449
|
+
}
|
|
2450
|
+
async initAndPoll(subscribers, newSubscription) {
|
|
2451
|
+
try {
|
|
2452
|
+
this.lastKnownState = await this.fetchState();
|
|
2453
|
+
for (const [, item] of this.lastKnownState) {
|
|
2454
|
+
const payload = this.payloadFactory.insert(item);
|
|
2455
|
+
try {
|
|
2456
|
+
newSubscription.callback(payload);
|
|
2457
|
+
} catch {}
|
|
2458
|
+
}
|
|
2459
|
+
} catch {}
|
|
2460
|
+
}
|
|
2461
|
+
pollForNewSubscriber(subscription) {
|
|
2462
|
+
for (const [, item] of this.lastKnownState) {
|
|
2463
|
+
const payload = this.payloadFactory.insert(item);
|
|
2464
|
+
try {
|
|
2465
|
+
subscription.callback(payload);
|
|
2466
|
+
} catch {}
|
|
2467
|
+
}
|
|
2468
|
+
}
|
|
2469
|
+
async poll(subscribers) {
|
|
2470
|
+
if (subscribers.size === 0)
|
|
2471
|
+
return;
|
|
2472
|
+
try {
|
|
2473
|
+
const currentState = await this.fetchState();
|
|
2474
|
+
const changes = [];
|
|
2475
|
+
for (const [key, item] of currentState) {
|
|
2476
|
+
const oldItem = this.lastKnownState.get(key);
|
|
2477
|
+
if (!oldItem) {
|
|
2478
|
+
changes.push(this.payloadFactory.insert(item));
|
|
2479
|
+
} else if (!this.compareItems(oldItem, item)) {
|
|
2480
|
+
changes.push(this.payloadFactory.update(oldItem, item));
|
|
2481
|
+
}
|
|
2482
|
+
}
|
|
2483
|
+
for (const [key, item] of this.lastKnownState) {
|
|
2484
|
+
if (!currentState.has(key)) {
|
|
2485
|
+
changes.push(this.payloadFactory.delete(item));
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
this.lastKnownState = currentState;
|
|
2489
|
+
for (const change of changes) {
|
|
2490
|
+
for (const sub of subscribers) {
|
|
2491
|
+
try {
|
|
2492
|
+
sub.callback(change);
|
|
2493
|
+
} catch {}
|
|
2494
|
+
}
|
|
2495
|
+
}
|
|
2496
|
+
} catch {}
|
|
2497
|
+
}
|
|
2498
|
+
get subscriptionCount() {
|
|
2499
|
+
let count = 0;
|
|
2500
|
+
for (const group of this.intervals.values()) {
|
|
2501
|
+
count += group.subscribers.size;
|
|
2502
|
+
}
|
|
2503
|
+
return count;
|
|
2504
|
+
}
|
|
2505
|
+
get hasSubscriptions() {
|
|
2506
|
+
return this.intervals.size > 0;
|
|
2507
|
+
}
|
|
2508
|
+
destroy() {
|
|
2509
|
+
for (const group of this.intervals.values()) {
|
|
2510
|
+
clearInterval(group.intervalId);
|
|
2511
|
+
}
|
|
2512
|
+
this.intervals.clear();
|
|
2513
|
+
this.lastKnownState.clear();
|
|
2514
|
+
this.initialized = false;
|
|
2515
|
+
}
|
|
2516
|
+
}
|
|
2517
|
+
|
|
2518
|
+
// src/queue/PostgresQueueStorage.ts
|
|
2519
|
+
var POSTGRES_QUEUE_STORAGE = createServiceToken19("jobqueue.storage.postgres");
|
|
2116
2520
|
|
|
2117
2521
|
class PostgresQueueStorage {
|
|
2118
2522
|
db;
|
|
2119
2523
|
queueName;
|
|
2120
|
-
|
|
2524
|
+
prefixes;
|
|
2525
|
+
prefixValues;
|
|
2526
|
+
tableName;
|
|
2527
|
+
pollingManager = null;
|
|
2528
|
+
constructor(db, queueName, options) {
|
|
2121
2529
|
this.db = db;
|
|
2122
2530
|
this.queueName = queueName;
|
|
2531
|
+
this.prefixes = options?.prefixes ?? [];
|
|
2532
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
2533
|
+
if (this.prefixes.length > 0) {
|
|
2534
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
2535
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
2536
|
+
} else {
|
|
2537
|
+
this.tableName = "job_queue";
|
|
2538
|
+
}
|
|
2539
|
+
}
|
|
2540
|
+
getPrefixColumnType(type) {
|
|
2541
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
2542
|
+
}
|
|
2543
|
+
buildPrefixColumnsSql() {
|
|
2544
|
+
if (this.prefixes.length === 0)
|
|
2545
|
+
return "";
|
|
2546
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
2547
|
+
`) + `,
|
|
2548
|
+
`;
|
|
2549
|
+
}
|
|
2550
|
+
getPrefixColumnNames() {
|
|
2551
|
+
return this.prefixes.map((p) => p.name);
|
|
2552
|
+
}
|
|
2553
|
+
buildPrefixWhereClause(startParam) {
|
|
2554
|
+
if (this.prefixes.length === 0) {
|
|
2555
|
+
return { conditions: "", params: [] };
|
|
2556
|
+
}
|
|
2557
|
+
const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
|
|
2558
|
+
const params = this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2559
|
+
return { conditions: " AND " + conditions, params };
|
|
2560
|
+
}
|
|
2561
|
+
getPrefixParamValues() {
|
|
2562
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2123
2563
|
}
|
|
2124
2564
|
async setupDatabase() {
|
|
2125
2565
|
let sql;
|
|
@@ -2130,10 +2570,13 @@ class PostgresQueueStorage {
|
|
|
2130
2570
|
if (e.code !== "42710")
|
|
2131
2571
|
throw e;
|
|
2132
2572
|
}
|
|
2573
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
2574
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2575
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2133
2576
|
sql = `
|
|
2134
|
-
CREATE TABLE IF NOT EXISTS
|
|
2577
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
2135
2578
|
id SERIAL NOT NULL,
|
|
2136
|
-
fingerprint text NOT NULL,
|
|
2579
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
2137
2580
|
queue text NOT NULL,
|
|
2138
2581
|
job_run_id text NOT NULL,
|
|
2139
2582
|
status job_status NOT NULL default 'PENDING',
|
|
@@ -2150,20 +2593,22 @@ class PostgresQueueStorage {
|
|
|
2150
2593
|
error_code text,
|
|
2151
2594
|
progress real DEFAULT 0,
|
|
2152
2595
|
progress_message text DEFAULT '',
|
|
2153
|
-
progress_details jsonb
|
|
2596
|
+
progress_details jsonb,
|
|
2597
|
+
worker_id text
|
|
2154
2598
|
)`;
|
|
2155
2599
|
await this.db.query(sql);
|
|
2600
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
2156
2601
|
sql = `
|
|
2157
|
-
CREATE INDEX IF NOT EXISTS
|
|
2158
|
-
ON
|
|
2602
|
+
CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx
|
|
2603
|
+
ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`;
|
|
2159
2604
|
await this.db.query(sql);
|
|
2160
2605
|
sql = `
|
|
2161
|
-
CREATE INDEX IF NOT EXISTS
|
|
2162
|
-
ON
|
|
2606
|
+
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx
|
|
2607
|
+
ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`;
|
|
2163
2608
|
await this.db.query(sql);
|
|
2164
2609
|
sql = `
|
|
2165
|
-
CREATE INDEX IF NOT EXISTS
|
|
2166
|
-
ON
|
|
2610
|
+
CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx
|
|
2611
|
+
ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`;
|
|
2167
2612
|
await this.db.query(sql);
|
|
2168
2613
|
}
|
|
2169
2614
|
async add(job) {
|
|
@@ -2177,9 +2622,14 @@ class PostgresQueueStorage {
|
|
|
2177
2622
|
job.progress_details = null;
|
|
2178
2623
|
job.created_at = now;
|
|
2179
2624
|
job.run_after = now;
|
|
2625
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2626
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2627
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
2628
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(",") + "," : "";
|
|
2629
|
+
const baseParamStart = prefixColumnNames.length + 1;
|
|
2180
2630
|
const sql = `
|
|
2181
|
-
INSERT INTO
|
|
2182
|
-
queue,
|
|
2631
|
+
INSERT INTO ${this.tableName}(
|
|
2632
|
+
${prefixColumnsInsert}queue,
|
|
2183
2633
|
fingerprint,
|
|
2184
2634
|
input,
|
|
2185
2635
|
run_after,
|
|
@@ -2192,9 +2642,10 @@ class PostgresQueueStorage {
|
|
|
2192
2642
|
progress_details
|
|
2193
2643
|
)
|
|
2194
2644
|
VALUES
|
|
2195
|
-
($1
|
|
2645
|
+
(${prefixParamPlaceholders}$${baseParamStart},$${baseParamStart + 1},$${baseParamStart + 2},$${baseParamStart + 3},$${baseParamStart + 4},$${baseParamStart + 5},$${baseParamStart + 6},$${baseParamStart + 7},$${baseParamStart + 8},$${baseParamStart + 9},$${baseParamStart + 10})
|
|
2196
2646
|
RETURNING id`;
|
|
2197
2647
|
const params = [
|
|
2648
|
+
...prefixParamValues,
|
|
2198
2649
|
job.queue,
|
|
2199
2650
|
job.fingerprint,
|
|
2200
2651
|
JSON.stringify(job.input),
|
|
@@ -2214,68 +2665,75 @@ class PostgresQueueStorage {
|
|
|
2214
2665
|
return job.id;
|
|
2215
2666
|
}
|
|
2216
2667
|
async get(id) {
|
|
2668
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2217
2669
|
const result = await this.db.query(`SELECT *
|
|
2218
|
-
FROM
|
|
2219
|
-
WHERE id = $1 AND queue = $2
|
|
2670
|
+
FROM ${this.tableName}
|
|
2671
|
+
WHERE id = $1 AND queue = $2${prefixConditions}
|
|
2220
2672
|
FOR UPDATE SKIP LOCKED
|
|
2221
|
-
LIMIT 1`, [id, this.queueName]);
|
|
2673
|
+
LIMIT 1`, [id, this.queueName, ...prefixParams]);
|
|
2222
2674
|
if (!result || result.rows.length === 0)
|
|
2223
2675
|
return;
|
|
2224
2676
|
return result.rows[0];
|
|
2225
2677
|
}
|
|
2226
2678
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2227
2679
|
num = Number(num) || 100;
|
|
2680
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
2228
2681
|
const result = await this.db.query(`
|
|
2229
2682
|
SELECT *
|
|
2230
|
-
FROM
|
|
2683
|
+
FROM ${this.tableName}
|
|
2231
2684
|
WHERE queue = $1
|
|
2232
|
-
AND status = $2
|
|
2685
|
+
AND status = $2${prefixConditions}
|
|
2233
2686
|
ORDER BY run_after ASC
|
|
2234
2687
|
LIMIT $3
|
|
2235
|
-
FOR UPDATE SKIP LOCKED`, [this.queueName, status, num]);
|
|
2688
|
+
FOR UPDATE SKIP LOCKED`, [this.queueName, status, num, ...prefixParams]);
|
|
2236
2689
|
if (!result)
|
|
2237
2690
|
return [];
|
|
2238
2691
|
return result.rows;
|
|
2239
2692
|
}
|
|
2240
|
-
async next() {
|
|
2693
|
+
async next(workerId) {
|
|
2694
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
|
|
2241
2695
|
const result = await this.db.query(`
|
|
2242
|
-
UPDATE
|
|
2243
|
-
SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2696
|
+
UPDATE ${this.tableName}
|
|
2697
|
+
SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = $4
|
|
2244
2698
|
WHERE id = (
|
|
2245
2699
|
SELECT id
|
|
2246
|
-
FROM
|
|
2700
|
+
FROM ${this.tableName}
|
|
2247
2701
|
WHERE queue = $2
|
|
2248
|
-
AND status = $3
|
|
2702
|
+
AND status = $3${prefixConditions}
|
|
2249
2703
|
AND run_after <= NOW() AT TIME ZONE 'UTC'
|
|
2250
2704
|
ORDER BY run_after ASC
|
|
2251
2705
|
FOR UPDATE SKIP LOCKED
|
|
2252
2706
|
LIMIT 1
|
|
2253
2707
|
)
|
|
2254
|
-
RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING
|
|
2708
|
+
RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING */, workerId ?? null, ...prefixParams]);
|
|
2255
2709
|
return result?.rows?.[0] ?? undefined;
|
|
2256
2710
|
}
|
|
2257
2711
|
async size(status = "PENDING" /* PENDING */) {
|
|
2712
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2258
2713
|
const result = await this.db.query(`
|
|
2259
2714
|
SELECT COUNT(*) as count
|
|
2260
|
-
FROM
|
|
2715
|
+
FROM ${this.tableName}
|
|
2261
2716
|
WHERE queue = $1
|
|
2262
|
-
AND status = $2`, [this.queueName, status]);
|
|
2717
|
+
AND status = $2${prefixConditions}`, [this.queueName, status, ...prefixParams]);
|
|
2263
2718
|
if (!result)
|
|
2264
2719
|
return 0;
|
|
2265
2720
|
return parseInt(result.rows[0].count, 10);
|
|
2266
2721
|
}
|
|
2267
2722
|
async complete(jobDetails) {
|
|
2723
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2268
2724
|
if (jobDetails.status === "DISABLED" /* DISABLED */) {
|
|
2269
|
-
|
|
2725
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(4);
|
|
2726
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
2270
2727
|
SET
|
|
2271
2728
|
status = $1,
|
|
2272
2729
|
progress = 100,
|
|
2273
2730
|
progress_message = '',
|
|
2274
2731
|
progress_details = NULL,
|
|
2275
2732
|
completed_at = NOW() AT TIME ZONE 'UTC'
|
|
2276
|
-
WHERE id = $2 AND queue = $3`, [jobDetails.status, jobDetails.id, this.queueName]);
|
|
2733
|
+
WHERE id = $2 AND queue = $3${prefixConditions}`, [jobDetails.status, jobDetails.id, this.queueName, ...prefixParams]);
|
|
2277
2734
|
} else if (jobDetails.status === "PENDING" /* PENDING */) {
|
|
2278
|
-
|
|
2735
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
|
|
2736
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
2279
2737
|
SET
|
|
2280
2738
|
error = $1,
|
|
2281
2739
|
error_code = $2,
|
|
@@ -2286,17 +2744,19 @@ class PostgresQueueStorage {
|
|
|
2286
2744
|
progress_details = NULL,
|
|
2287
2745
|
run_attempts = run_attempts + 1,
|
|
2288
2746
|
last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2289
|
-
WHERE id = $5 AND queue = $6`, [
|
|
2747
|
+
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
2290
2748
|
jobDetails.error,
|
|
2291
2749
|
jobDetails.error_code,
|
|
2292
2750
|
jobDetails.status,
|
|
2293
2751
|
jobDetails.run_after,
|
|
2294
2752
|
jobDetails.id,
|
|
2295
|
-
this.queueName
|
|
2753
|
+
this.queueName,
|
|
2754
|
+
...prefixParams
|
|
2296
2755
|
]);
|
|
2297
2756
|
} else {
|
|
2757
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
|
|
2298
2758
|
await this.db.query(`
|
|
2299
|
-
UPDATE
|
|
2759
|
+
UPDATE ${this.tableName}
|
|
2300
2760
|
SET
|
|
2301
2761
|
output = $1,
|
|
2302
2762
|
error = $2,
|
|
@@ -2308,86 +2768,231 @@ class PostgresQueueStorage {
|
|
|
2308
2768
|
run_attempts = run_attempts + 1,
|
|
2309
2769
|
completed_at = NOW() AT TIME ZONE 'UTC',
|
|
2310
2770
|
last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2311
|
-
WHERE id = $5 AND queue = $6`, [
|
|
2771
|
+
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
2312
2772
|
jobDetails.output ? JSON.stringify(jobDetails.output) : null,
|
|
2313
2773
|
jobDetails.error ?? null,
|
|
2314
2774
|
jobDetails.error_code ?? null,
|
|
2315
2775
|
jobDetails.status,
|
|
2316
2776
|
jobDetails.id,
|
|
2317
|
-
this.queueName
|
|
2777
|
+
this.queueName,
|
|
2778
|
+
...prefixParams
|
|
2318
2779
|
]);
|
|
2319
2780
|
}
|
|
2320
2781
|
}
|
|
2321
2782
|
async deleteAll() {
|
|
2783
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
2322
2784
|
await this.db.query(`
|
|
2323
|
-
DELETE FROM
|
|
2324
|
-
WHERE queue = $1`, [this.queueName]);
|
|
2785
|
+
DELETE FROM ${this.tableName}
|
|
2786
|
+
WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
|
|
2325
2787
|
}
|
|
2326
2788
|
async outputForInput(input) {
|
|
2327
2789
|
const fingerprint = await makeFingerprint5(input);
|
|
2790
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2328
2791
|
const result = await this.db.query(`
|
|
2329
2792
|
SELECT output
|
|
2330
|
-
FROM
|
|
2331
|
-
WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'`, [fingerprint, this.queueName]);
|
|
2332
|
-
if (!result)
|
|
2793
|
+
FROM ${this.tableName}
|
|
2794
|
+
WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'${prefixConditions}`, [fingerprint, this.queueName, ...prefixParams]);
|
|
2795
|
+
if (!result || result.rows.length === 0)
|
|
2333
2796
|
return null;
|
|
2334
2797
|
return result.rows[0].output;
|
|
2335
2798
|
}
|
|
2336
2799
|
async abort(jobId) {
|
|
2337
|
-
const
|
|
2338
|
-
|
|
2800
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2801
|
+
await this.db.query(`
|
|
2802
|
+
UPDATE ${this.tableName}
|
|
2339
2803
|
SET status = 'ABORTING'
|
|
2340
|
-
WHERE id = $1 AND queue = $2`, [jobId, this.queueName]);
|
|
2804
|
+
WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
|
|
2341
2805
|
}
|
|
2342
2806
|
async getByRunId(job_run_id) {
|
|
2807
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2343
2808
|
const result = await this.db.query(`
|
|
2344
|
-
SELECT * FROM
|
|
2809
|
+
SELECT * FROM ${this.tableName} WHERE job_run_id = $1 AND queue = $2${prefixConditions}`, [job_run_id, this.queueName, ...prefixParams]);
|
|
2345
2810
|
if (!result)
|
|
2346
2811
|
return [];
|
|
2347
2812
|
return result.rows;
|
|
2348
2813
|
}
|
|
2349
2814
|
async saveProgress(jobId, progress, message, details) {
|
|
2815
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(6);
|
|
2350
2816
|
await this.db.query(`
|
|
2351
|
-
UPDATE
|
|
2817
|
+
UPDATE ${this.tableName}
|
|
2352
2818
|
SET progress = $1,
|
|
2353
2819
|
progress_message = $2,
|
|
2354
2820
|
progress_details = $3
|
|
2355
|
-
WHERE id = $4 AND queue = $5`, [
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2821
|
+
WHERE id = $4 AND queue = $5${prefixConditions}`, [
|
|
2822
|
+
progress,
|
|
2823
|
+
message,
|
|
2824
|
+
details ? JSON.stringify(details) : null,
|
|
2359
2825
|
jobId,
|
|
2360
|
-
this.queueName
|
|
2826
|
+
this.queueName,
|
|
2827
|
+
...prefixParams
|
|
2361
2828
|
]);
|
|
2362
2829
|
}
|
|
2830
|
+
async delete(jobId) {
|
|
2831
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2832
|
+
await this.db.query(`DELETE FROM ${this.tableName} WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
|
|
2833
|
+
}
|
|
2363
2834
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2364
2835
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
2365
|
-
|
|
2836
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
2837
|
+
await this.db.query(`DELETE FROM ${this.tableName}
|
|
2366
2838
|
WHERE queue = $1
|
|
2367
2839
|
AND status = $2
|
|
2368
2840
|
AND completed_at IS NOT NULL
|
|
2369
|
-
AND completed_at <= $3`, [this.queueName, status, cutoffDate]);
|
|
2841
|
+
AND completed_at <= $3${prefixConditions}`, [this.queueName, status, cutoffDate, ...prefixParams]);
|
|
2842
|
+
}
|
|
2843
|
+
async getAllJobs() {
|
|
2844
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
2845
|
+
const result = await this.db.query(`SELECT * FROM ${this.tableName} WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
|
|
2846
|
+
if (!result)
|
|
2847
|
+
return [];
|
|
2848
|
+
return result.rows;
|
|
2849
|
+
}
|
|
2850
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
2851
|
+
const filterEntries = Object.entries(prefixFilter);
|
|
2852
|
+
let query = `SELECT * FROM ${this.tableName} WHERE queue = $1`;
|
|
2853
|
+
const params = [this.queueName];
|
|
2854
|
+
filterEntries.forEach(([key, value], index) => {
|
|
2855
|
+
query += ` AND ${key} = $${index + 2}`;
|
|
2856
|
+
params.push(value);
|
|
2857
|
+
});
|
|
2858
|
+
const result = await this.db.query(query, params);
|
|
2859
|
+
if (!result)
|
|
2860
|
+
return [];
|
|
2861
|
+
return result.rows;
|
|
2862
|
+
}
|
|
2863
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
2864
|
+
if (prefixFilter === undefined) {
|
|
2865
|
+
return false;
|
|
2866
|
+
}
|
|
2867
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
2868
|
+
return true;
|
|
2869
|
+
}
|
|
2870
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
2871
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
2872
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
2873
|
+
return true;
|
|
2874
|
+
}
|
|
2875
|
+
for (const key of instanceKeys) {
|
|
2876
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
2877
|
+
return true;
|
|
2878
|
+
}
|
|
2879
|
+
}
|
|
2880
|
+
return false;
|
|
2881
|
+
}
|
|
2882
|
+
getPollingManager() {
|
|
2883
|
+
if (!this.pollingManager) {
|
|
2884
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
2885
|
+
const jobs = await this.getAllJobs();
|
|
2886
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
2887
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
2888
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
2889
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
2890
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
2891
|
+
});
|
|
2892
|
+
}
|
|
2893
|
+
return this.pollingManager;
|
|
2894
|
+
}
|
|
2895
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
2896
|
+
let lastKnownJobs = new Map;
|
|
2897
|
+
let cancelled = false;
|
|
2898
|
+
const poll = async () => {
|
|
2899
|
+
if (cancelled)
|
|
2900
|
+
return;
|
|
2901
|
+
try {
|
|
2902
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
2903
|
+
if (cancelled)
|
|
2904
|
+
return;
|
|
2905
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
2906
|
+
for (const [id, job] of currentMap) {
|
|
2907
|
+
const old = lastKnownJobs.get(id);
|
|
2908
|
+
if (!old) {
|
|
2909
|
+
callback({ type: "INSERT", new: job });
|
|
2910
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
2911
|
+
callback({ type: "UPDATE", old, new: job });
|
|
2912
|
+
}
|
|
2913
|
+
}
|
|
2914
|
+
for (const [id, job] of lastKnownJobs) {
|
|
2915
|
+
if (!currentMap.has(id)) {
|
|
2916
|
+
callback({ type: "DELETE", old: job });
|
|
2917
|
+
}
|
|
2918
|
+
}
|
|
2919
|
+
lastKnownJobs = currentMap;
|
|
2920
|
+
} catch {}
|
|
2921
|
+
};
|
|
2922
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
2923
|
+
poll();
|
|
2924
|
+
return () => {
|
|
2925
|
+
cancelled = true;
|
|
2926
|
+
clearInterval(intervalId);
|
|
2927
|
+
};
|
|
2928
|
+
}
|
|
2929
|
+
subscribeToChanges(callback, options) {
|
|
2930
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
2931
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
2932
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
2933
|
+
}
|
|
2934
|
+
const manager = this.getPollingManager();
|
|
2935
|
+
return manager.subscribe(callback, { intervalMs });
|
|
2370
2936
|
}
|
|
2371
2937
|
}
|
|
2372
2938
|
// src/queue/SqliteQueueStorage.ts
|
|
2373
|
-
import { createServiceToken as
|
|
2374
|
-
var SQLITE_QUEUE_STORAGE =
|
|
2939
|
+
import { createServiceToken as createServiceToken20, makeFingerprint as makeFingerprint6, sleep as sleep4, uuid4 as uuid43 } from "@workglow/util";
|
|
2940
|
+
var SQLITE_QUEUE_STORAGE = createServiceToken20("jobqueue.storage.sqlite");
|
|
2375
2941
|
|
|
2376
2942
|
class SqliteQueueStorage {
|
|
2377
2943
|
db;
|
|
2378
2944
|
queueName;
|
|
2379
2945
|
options;
|
|
2946
|
+
prefixes;
|
|
2947
|
+
prefixValues;
|
|
2948
|
+
tableName;
|
|
2949
|
+
pollingManager = null;
|
|
2380
2950
|
constructor(db, queueName, options) {
|
|
2381
2951
|
this.db = db;
|
|
2382
2952
|
this.queueName = queueName;
|
|
2383
2953
|
this.options = options;
|
|
2954
|
+
this.prefixes = options?.prefixes ?? [];
|
|
2955
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
2956
|
+
if (this.prefixes.length > 0) {
|
|
2957
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
2958
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
2959
|
+
} else {
|
|
2960
|
+
this.tableName = "job_queue";
|
|
2961
|
+
}
|
|
2962
|
+
}
|
|
2963
|
+
getPrefixColumnType(type) {
|
|
2964
|
+
return type === "uuid" ? "TEXT" : "INTEGER";
|
|
2965
|
+
}
|
|
2966
|
+
buildPrefixColumnsSql() {
|
|
2967
|
+
if (this.prefixes.length === 0)
|
|
2968
|
+
return "";
|
|
2969
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
2970
|
+
`) + `,
|
|
2971
|
+
`;
|
|
2972
|
+
}
|
|
2973
|
+
getPrefixColumnNames() {
|
|
2974
|
+
return this.prefixes.map((p) => p.name);
|
|
2975
|
+
}
|
|
2976
|
+
buildPrefixWhereClause() {
|
|
2977
|
+
if (this.prefixes.length === 0) {
|
|
2978
|
+
return "";
|
|
2979
|
+
}
|
|
2980
|
+
const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
|
|
2981
|
+
return " AND " + conditions;
|
|
2982
|
+
}
|
|
2983
|
+
getPrefixParamValues() {
|
|
2984
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2384
2985
|
}
|
|
2385
2986
|
async setupDatabase() {
|
|
2386
|
-
await
|
|
2987
|
+
await sleep4(0);
|
|
2988
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
2989
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2990
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2991
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
2387
2992
|
this.db.exec(`
|
|
2388
|
-
CREATE TABLE IF NOT EXISTS
|
|
2993
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
2389
2994
|
id INTEGER PRIMARY KEY,
|
|
2390
|
-
fingerprint text NOT NULL,
|
|
2995
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
2391
2996
|
queue text NOT NULL,
|
|
2392
2997
|
job_run_id text NOT NULL,
|
|
2393
2998
|
status TEXT NOT NULL default 'PENDING',
|
|
@@ -2404,12 +3009,13 @@ class SqliteQueueStorage {
|
|
|
2404
3009
|
error_code TEXT,
|
|
2405
3010
|
progress REAL DEFAULT 0,
|
|
2406
3011
|
progress_message TEXT DEFAULT '',
|
|
2407
|
-
progress_details TEXT NULL
|
|
3012
|
+
progress_details TEXT NULL,
|
|
3013
|
+
worker_id TEXT
|
|
2408
3014
|
);
|
|
2409
3015
|
|
|
2410
|
-
CREATE INDEX IF NOT EXISTS
|
|
2411
|
-
CREATE INDEX IF NOT EXISTS
|
|
2412
|
-
CREATE INDEX IF NOT EXISTS
|
|
3016
|
+
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after);
|
|
3017
|
+
CREATE INDEX IF NOT EXISTS job_queue_fingerprint${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status);
|
|
3018
|
+
CREATE INDEX IF NOT EXISTS job_queue_job_run_id${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, job_run_id);
|
|
2413
3019
|
`);
|
|
2414
3020
|
}
|
|
2415
3021
|
async add(job) {
|
|
@@ -2423,9 +3029,13 @@ class SqliteQueueStorage {
|
|
|
2423
3029
|
job.progress_details = null;
|
|
2424
3030
|
job.created_at = now;
|
|
2425
3031
|
job.run_after = now;
|
|
3032
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3033
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3034
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
3035
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
2426
3036
|
const AddQuery = `
|
|
2427
|
-
INSERT INTO
|
|
2428
|
-
queue,
|
|
3037
|
+
INSERT INTO ${this.tableName}(
|
|
3038
|
+
${prefixColumnsInsert}queue,
|
|
2429
3039
|
fingerprint,
|
|
2430
3040
|
input,
|
|
2431
3041
|
run_after,
|
|
@@ -2437,21 +3047,23 @@ class SqliteQueueStorage {
|
|
|
2437
3047
|
progress_details,
|
|
2438
3048
|
created_at
|
|
2439
3049
|
)
|
|
2440
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
3050
|
+
VALUES (${prefixPlaceholders}?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2441
3051
|
RETURNING id`;
|
|
2442
3052
|
const stmt = this.db.prepare(AddQuery);
|
|
2443
|
-
const result = stmt.get(job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
|
|
3053
|
+
const result = stmt.get(...prefixParamValues, job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
|
|
2444
3054
|
job.id = result?.id;
|
|
2445
3055
|
return result?.id;
|
|
2446
3056
|
}
|
|
2447
3057
|
async get(id) {
|
|
3058
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3059
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2448
3060
|
const JobQuery = `
|
|
2449
3061
|
SELECT *
|
|
2450
|
-
FROM
|
|
2451
|
-
WHERE id = ? AND queue =
|
|
3062
|
+
FROM ${this.tableName}
|
|
3063
|
+
WHERE id = ? AND queue = ?${prefixConditions}
|
|
2452
3064
|
LIMIT 1`;
|
|
2453
3065
|
const stmt = this.db.prepare(JobQuery);
|
|
2454
|
-
const result = stmt.get(id, this.queueName);
|
|
3066
|
+
const result = stmt.get(String(id), this.queueName, ...prefixParams);
|
|
2455
3067
|
if (!result)
|
|
2456
3068
|
return;
|
|
2457
3069
|
if (result.input)
|
|
@@ -2464,15 +3076,17 @@ class SqliteQueueStorage {
|
|
|
2464
3076
|
}
|
|
2465
3077
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2466
3078
|
num = Number(num) || 100;
|
|
3079
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3080
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2467
3081
|
const FutureJobQuery = `
|
|
2468
3082
|
SELECT *
|
|
2469
|
-
FROM
|
|
3083
|
+
FROM ${this.tableName}
|
|
2470
3084
|
WHERE queue = ?
|
|
2471
|
-
AND status =
|
|
3085
|
+
AND status = ?${prefixConditions}
|
|
2472
3086
|
ORDER BY run_after ASC
|
|
2473
3087
|
LIMIT ${num}`;
|
|
2474
3088
|
const stmt = this.db.prepare(FutureJobQuery);
|
|
2475
|
-
const result = stmt.all(this.queueName, status);
|
|
3089
|
+
const result = stmt.all(this.queueName, status, ...prefixParams);
|
|
2476
3090
|
return (result || []).map((details) => {
|
|
2477
3091
|
if (details.input)
|
|
2478
3092
|
details.input = JSON.parse(details.input);
|
|
@@ -2484,20 +3098,24 @@ class SqliteQueueStorage {
|
|
|
2484
3098
|
});
|
|
2485
3099
|
}
|
|
2486
3100
|
async abort(jobId) {
|
|
3101
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3102
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2487
3103
|
const AbortQuery = `
|
|
2488
|
-
UPDATE
|
|
3104
|
+
UPDATE ${this.tableName}
|
|
2489
3105
|
SET status = ?
|
|
2490
|
-
WHERE id = ? AND queue =
|
|
3106
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2491
3107
|
const stmt = this.db.prepare(AbortQuery);
|
|
2492
|
-
stmt.run("ABORTING" /* ABORTING */, jobId, this.queueName);
|
|
3108
|
+
stmt.run("ABORTING" /* ABORTING */, String(jobId), this.queueName, ...prefixParams);
|
|
2493
3109
|
}
|
|
2494
3110
|
async getByRunId(job_run_id) {
|
|
3111
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3112
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2495
3113
|
const JobsByRunIdQuery = `
|
|
2496
3114
|
SELECT *
|
|
2497
|
-
FROM
|
|
2498
|
-
WHERE job_run_id = ? AND queue =
|
|
3115
|
+
FROM ${this.tableName}
|
|
3116
|
+
WHERE job_run_id = ? AND queue = ?${prefixConditions}`;
|
|
2499
3117
|
const stmt = this.db.prepare(JobsByRunIdQuery);
|
|
2500
|
-
const result = stmt.all(job_run_id, this.queueName);
|
|
3118
|
+
const result = stmt.all(job_run_id, this.queueName, ...prefixParams);
|
|
2501
3119
|
return (result || []).map((details) => {
|
|
2502
3120
|
if (details.input)
|
|
2503
3121
|
details.input = JSON.parse(details.input);
|
|
@@ -2508,22 +3126,24 @@ class SqliteQueueStorage {
|
|
|
2508
3126
|
return details;
|
|
2509
3127
|
});
|
|
2510
3128
|
}
|
|
2511
|
-
async next() {
|
|
3129
|
+
async next(workerId) {
|
|
2512
3130
|
const now = new Date().toISOString();
|
|
3131
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3132
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2513
3133
|
const stmt = this.db.prepare(`
|
|
2514
|
-
UPDATE
|
|
2515
|
-
SET status = ?, last_ran_at = ?
|
|
3134
|
+
UPDATE ${this.tableName}
|
|
3135
|
+
SET status = ?, last_ran_at = ?, worker_id = ?
|
|
2516
3136
|
WHERE id = (
|
|
2517
3137
|
SELECT id
|
|
2518
|
-
FROM
|
|
3138
|
+
FROM ${this.tableName}
|
|
2519
3139
|
WHERE queue = ?
|
|
2520
|
-
AND status =
|
|
3140
|
+
AND status = ?${prefixConditions}
|
|
2521
3141
|
AND run_after <= ?
|
|
2522
3142
|
ORDER BY run_after ASC
|
|
2523
3143
|
LIMIT 1
|
|
2524
3144
|
)
|
|
2525
3145
|
RETURNING *`);
|
|
2526
|
-
const result = stmt.get("PROCESSING" /* PROCESSING */, now, this.queueName, "PENDING" /* PENDING */, now);
|
|
3146
|
+
const result = stmt.get("PROCESSING" /* PROCESSING */, now, workerId ?? null, this.queueName, "PENDING" /* PENDING */, ...prefixParams, now);
|
|
2527
3147
|
if (!result)
|
|
2528
3148
|
return;
|
|
2529
3149
|
if (result.input)
|
|
@@ -2535,33 +3155,37 @@ class SqliteQueueStorage {
|
|
|
2535
3155
|
return result;
|
|
2536
3156
|
}
|
|
2537
3157
|
async size(status = "PENDING" /* PENDING */) {
|
|
3158
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3159
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2538
3160
|
const sizeQuery = `
|
|
2539
3161
|
SELECT COUNT(*) as count
|
|
2540
|
-
FROM
|
|
3162
|
+
FROM ${this.tableName}
|
|
2541
3163
|
WHERE queue = ?
|
|
2542
|
-
AND status =
|
|
3164
|
+
AND status = ?${prefixConditions}`;
|
|
2543
3165
|
const stmt = this.db.prepare(sizeQuery);
|
|
2544
|
-
const result = stmt.get(this.queueName, status);
|
|
3166
|
+
const result = stmt.get(this.queueName, status, ...prefixParams);
|
|
2545
3167
|
return result.count;
|
|
2546
3168
|
}
|
|
2547
3169
|
async complete(job) {
|
|
2548
3170
|
const now = new Date().toISOString();
|
|
3171
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3172
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2549
3173
|
let updateQuery;
|
|
2550
3174
|
let params;
|
|
2551
3175
|
if (job.status === "DISABLED" /* DISABLED */) {
|
|
2552
3176
|
updateQuery = `
|
|
2553
|
-
UPDATE
|
|
3177
|
+
UPDATE ${this.tableName}
|
|
2554
3178
|
SET
|
|
2555
3179
|
status = ?,
|
|
2556
3180
|
progress = 100,
|
|
2557
3181
|
progress_message = '',
|
|
2558
3182
|
progress_details = NULL,
|
|
2559
3183
|
completed_at = ?
|
|
2560
|
-
WHERE id = ? AND queue =
|
|
2561
|
-
params = [job.status, now, job.id, this.queueName];
|
|
3184
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
3185
|
+
params = [job.status, now, job.id, this.queueName, ...prefixParams];
|
|
2562
3186
|
} else {
|
|
2563
3187
|
updateQuery = `
|
|
2564
|
-
UPDATE
|
|
3188
|
+
UPDATE ${this.tableName}
|
|
2565
3189
|
SET
|
|
2566
3190
|
output = ?,
|
|
2567
3191
|
error = ?,
|
|
@@ -2573,7 +3197,7 @@ class SqliteQueueStorage {
|
|
|
2573
3197
|
last_ran_at = ?,
|
|
2574
3198
|
completed_at = ?,
|
|
2575
3199
|
run_attempts = run_attempts + 1
|
|
2576
|
-
WHERE id = ? AND queue =
|
|
3200
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2577
3201
|
params = [
|
|
2578
3202
|
job.output ? JSON.stringify(job.output) : null,
|
|
2579
3203
|
job.error ?? null,
|
|
@@ -2582,80 +3206,243 @@ class SqliteQueueStorage {
|
|
|
2582
3206
|
now,
|
|
2583
3207
|
now,
|
|
2584
3208
|
job.id,
|
|
2585
|
-
this.queueName
|
|
3209
|
+
this.queueName,
|
|
3210
|
+
...prefixParams
|
|
2586
3211
|
];
|
|
2587
3212
|
}
|
|
2588
3213
|
const stmt = this.db.prepare(updateQuery);
|
|
2589
3214
|
stmt.run(...params);
|
|
2590
3215
|
}
|
|
2591
3216
|
async deleteAll() {
|
|
3217
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3218
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2592
3219
|
const ClearQuery = `
|
|
2593
|
-
DELETE FROM
|
|
2594
|
-
WHERE queue =
|
|
3220
|
+
DELETE FROM ${this.tableName}
|
|
3221
|
+
WHERE queue = ?${prefixConditions}`;
|
|
2595
3222
|
const stmt = this.db.prepare(ClearQuery);
|
|
2596
|
-
stmt.run(this.queueName);
|
|
3223
|
+
stmt.run(this.queueName, ...prefixParams);
|
|
2597
3224
|
}
|
|
2598
3225
|
async outputForInput(input) {
|
|
2599
3226
|
const fingerprint = await makeFingerprint6(input);
|
|
3227
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3228
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2600
3229
|
const OutputQuery = `
|
|
2601
3230
|
SELECT output
|
|
2602
|
-
FROM
|
|
2603
|
-
WHERE queue = ? AND fingerprint = ? AND status =
|
|
3231
|
+
FROM ${this.tableName}
|
|
3232
|
+
WHERE queue = ? AND fingerprint = ? AND status = ?${prefixConditions}`;
|
|
2604
3233
|
const stmt = this.db.prepare(OutputQuery);
|
|
2605
|
-
const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED
|
|
3234
|
+
const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED */, ...prefixParams);
|
|
2606
3235
|
return result?.output ? JSON.parse(result.output) : null;
|
|
2607
3236
|
}
|
|
2608
3237
|
async saveProgress(jobId, progress, message, details) {
|
|
3238
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3239
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2609
3240
|
const UpdateProgressQuery = `
|
|
2610
|
-
UPDATE
|
|
3241
|
+
UPDATE ${this.tableName}
|
|
2611
3242
|
SET progress = ?,
|
|
2612
3243
|
progress_message = ?,
|
|
2613
3244
|
progress_details = ?
|
|
2614
|
-
WHERE id = ? AND queue =
|
|
3245
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2615
3246
|
const stmt = this.db.prepare(UpdateProgressQuery);
|
|
2616
|
-
stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName);
|
|
3247
|
+
stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName, ...prefixParams);
|
|
2617
3248
|
}
|
|
2618
3249
|
async delete(jobId) {
|
|
3250
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3251
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2619
3252
|
const DeleteQuery = `
|
|
2620
|
-
DELETE FROM
|
|
2621
|
-
WHERE id = ? AND queue =
|
|
3253
|
+
DELETE FROM ${this.tableName}
|
|
3254
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2622
3255
|
const stmt = this.db.prepare(DeleteQuery);
|
|
2623
|
-
stmt.run(String(jobId), this.queueName);
|
|
3256
|
+
stmt.run(String(jobId), this.queueName, ...prefixParams);
|
|
2624
3257
|
}
|
|
2625
3258
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2626
3259
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
3260
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3261
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2627
3262
|
const DeleteQuery = `
|
|
2628
|
-
DELETE FROM
|
|
3263
|
+
DELETE FROM ${this.tableName}
|
|
2629
3264
|
WHERE queue = ?
|
|
2630
3265
|
AND status = ?
|
|
2631
3266
|
AND completed_at IS NOT NULL
|
|
2632
|
-
AND completed_at <=
|
|
3267
|
+
AND completed_at <= ?${prefixConditions}`;
|
|
2633
3268
|
const stmt = this.db.prepare(DeleteQuery);
|
|
2634
|
-
stmt.run(this.queueName, status, cutoffDate);
|
|
3269
|
+
stmt.run(this.queueName, status, cutoffDate, ...prefixParams);
|
|
2635
3270
|
}
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
3271
|
+
getAllJobs() {
|
|
3272
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3273
|
+
const prefixParams = this.getPrefixParamValues();
|
|
3274
|
+
const AllJobsQuery = `
|
|
3275
|
+
SELECT *
|
|
3276
|
+
FROM ${this.tableName}
|
|
3277
|
+
WHERE queue = ?${prefixConditions}`;
|
|
3278
|
+
const stmt = this.db.prepare(AllJobsQuery);
|
|
3279
|
+
const result = stmt.all(this.queueName, ...prefixParams);
|
|
3280
|
+
return (result || []).map((details) => {
|
|
3281
|
+
if (details.input)
|
|
3282
|
+
details.input = JSON.parse(details.input);
|
|
3283
|
+
if (details.output)
|
|
3284
|
+
details.output = JSON.parse(details.output);
|
|
3285
|
+
if (details.progress_details)
|
|
3286
|
+
details.progress_details = JSON.parse(details.progress_details);
|
|
3287
|
+
return details;
|
|
3288
|
+
});
|
|
2647
3289
|
}
|
|
2648
|
-
|
|
2649
|
-
const
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
3290
|
+
getAllJobsWithFilter(prefixFilter) {
|
|
3291
|
+
const filterEntries = Object.entries(prefixFilter);
|
|
3292
|
+
let whereClause = "WHERE queue = ?";
|
|
3293
|
+
const params = [this.queueName];
|
|
3294
|
+
for (const [key, value] of filterEntries) {
|
|
3295
|
+
whereClause += ` AND ${key} = ?`;
|
|
3296
|
+
params.push(value);
|
|
3297
|
+
}
|
|
3298
|
+
const AllJobsQuery = `SELECT * FROM ${this.tableName} ${whereClause}`;
|
|
3299
|
+
const stmt = this.db.prepare(AllJobsQuery);
|
|
3300
|
+
const result = stmt.all(...params);
|
|
3301
|
+
return (result || []).map((details) => {
|
|
3302
|
+
if (details.input)
|
|
3303
|
+
details.input = JSON.parse(details.input);
|
|
3304
|
+
if (details.output)
|
|
3305
|
+
details.output = JSON.parse(details.output);
|
|
3306
|
+
if (details.progress_details)
|
|
3307
|
+
details.progress_details = JSON.parse(details.progress_details);
|
|
3308
|
+
return details;
|
|
3309
|
+
});
|
|
3310
|
+
}
|
|
3311
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
3312
|
+
if (prefixFilter === undefined) {
|
|
3313
|
+
return false;
|
|
2653
3314
|
}
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
3315
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
3316
|
+
return true;
|
|
3317
|
+
}
|
|
3318
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
3319
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
3320
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
3321
|
+
return true;
|
|
3322
|
+
}
|
|
3323
|
+
for (const key of instanceKeys) {
|
|
3324
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
3325
|
+
return true;
|
|
3326
|
+
}
|
|
3327
|
+
}
|
|
3328
|
+
return false;
|
|
3329
|
+
}
|
|
3330
|
+
getPollingManager() {
|
|
3331
|
+
if (!this.pollingManager) {
|
|
3332
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
3333
|
+
const jobs = this.getAllJobs();
|
|
3334
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
3335
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
3336
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
3337
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
3338
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
3339
|
+
});
|
|
3340
|
+
}
|
|
3341
|
+
return this.pollingManager;
|
|
3342
|
+
}
|
|
3343
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
3344
|
+
let lastKnownJobs = new Map;
|
|
3345
|
+
const poll = () => {
|
|
3346
|
+
try {
|
|
3347
|
+
const currentJobs = this.getAllJobsWithFilter(prefixFilter);
|
|
3348
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
3349
|
+
for (const [id, job] of currentMap) {
|
|
3350
|
+
const old = lastKnownJobs.get(id);
|
|
3351
|
+
if (!old) {
|
|
3352
|
+
callback({ type: "INSERT", new: job });
|
|
3353
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
3354
|
+
callback({ type: "UPDATE", old, new: job });
|
|
3355
|
+
}
|
|
3356
|
+
}
|
|
3357
|
+
for (const [id, job] of lastKnownJobs) {
|
|
3358
|
+
if (!currentMap.has(id)) {
|
|
3359
|
+
callback({ type: "DELETE", old: job });
|
|
3360
|
+
}
|
|
3361
|
+
}
|
|
3362
|
+
lastKnownJobs = currentMap;
|
|
3363
|
+
} catch {}
|
|
3364
|
+
};
|
|
3365
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
3366
|
+
poll();
|
|
3367
|
+
return () => {
|
|
3368
|
+
clearInterval(intervalId);
|
|
3369
|
+
};
|
|
3370
|
+
}
|
|
3371
|
+
subscribeToChanges(callback, options) {
|
|
3372
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
3373
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
3374
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
3375
|
+
}
|
|
3376
|
+
const manager = this.getPollingManager();
|
|
3377
|
+
return manager.subscribe(callback, { intervalMs });
|
|
3378
|
+
}
|
|
3379
|
+
}
|
|
3380
|
+
// src/queue/SupabaseQueueStorage.ts
|
|
3381
|
+
import { createServiceToken as createServiceToken21, makeFingerprint as makeFingerprint7, uuid4 as uuid44 } from "@workglow/util";
|
|
3382
|
+
var SUPABASE_QUEUE_STORAGE = createServiceToken21("jobqueue.storage.supabase");
|
|
3383
|
+
|
|
3384
|
+
class SupabaseQueueStorage {
|
|
3385
|
+
client;
|
|
3386
|
+
queueName;
|
|
3387
|
+
prefixes;
|
|
3388
|
+
prefixValues;
|
|
3389
|
+
tableName;
|
|
3390
|
+
realtimeChannel = null;
|
|
3391
|
+
pollingManager = null;
|
|
3392
|
+
constructor(client, queueName, options) {
|
|
3393
|
+
this.client = client;
|
|
3394
|
+
this.queueName = queueName;
|
|
3395
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3396
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3397
|
+
if (this.prefixes.length > 0) {
|
|
3398
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3399
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
3400
|
+
} else {
|
|
3401
|
+
this.tableName = "job_queue";
|
|
3402
|
+
}
|
|
3403
|
+
}
|
|
3404
|
+
getPrefixColumnType(type) {
|
|
3405
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
3406
|
+
}
|
|
3407
|
+
buildPrefixColumnsSql() {
|
|
3408
|
+
if (this.prefixes.length === 0)
|
|
3409
|
+
return "";
|
|
3410
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
3411
|
+
`) + `,
|
|
3412
|
+
`;
|
|
3413
|
+
}
|
|
3414
|
+
getPrefixColumnNames() {
|
|
3415
|
+
return this.prefixes.map((p) => p.name);
|
|
3416
|
+
}
|
|
3417
|
+
applyPrefixFilters(query) {
|
|
3418
|
+
let result = query;
|
|
3419
|
+
for (const prefix of this.prefixes) {
|
|
3420
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
3421
|
+
}
|
|
3422
|
+
return result;
|
|
3423
|
+
}
|
|
3424
|
+
getPrefixInsertValues() {
|
|
3425
|
+
const values = {};
|
|
3426
|
+
for (const prefix of this.prefixes) {
|
|
3427
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
3428
|
+
}
|
|
3429
|
+
return values;
|
|
3430
|
+
}
|
|
3431
|
+
async setupDatabase() {
|
|
3432
|
+
const createTypeSql = `CREATE TYPE job_status AS ENUM (${Object.values(JobStatus).map((v) => `'${v}'`).join(",")})`;
|
|
3433
|
+
const { error: typeError } = await this.client.rpc("exec_sql", { query: createTypeSql });
|
|
3434
|
+
if (typeError && typeError.code !== "42710") {
|
|
3435
|
+
throw typeError;
|
|
3436
|
+
}
|
|
3437
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
3438
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3439
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3440
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
3441
|
+
const createTableSql = `
|
|
3442
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
3443
|
+
id SERIAL NOT NULL,
|
|
3444
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
3445
|
+
queue text NOT NULL,
|
|
2659
3446
|
job_run_id text NOT NULL,
|
|
2660
3447
|
status job_status NOT NULL default 'PENDING',
|
|
2661
3448
|
input jsonb NOT NULL,
|
|
@@ -2671,7 +3458,8 @@ class SupabaseQueueStorage {
|
|
|
2671
3458
|
error_code text,
|
|
2672
3459
|
progress real DEFAULT 0,
|
|
2673
3460
|
progress_message text DEFAULT '',
|
|
2674
|
-
progress_details jsonb
|
|
3461
|
+
progress_details jsonb,
|
|
3462
|
+
worker_id text
|
|
2675
3463
|
)`;
|
|
2676
3464
|
const { error: tableError } = await this.client.rpc("exec_sql", { query: createTableSql });
|
|
2677
3465
|
if (tableError) {
|
|
@@ -2680,12 +3468,12 @@ class SupabaseQueueStorage {
|
|
|
2680
3468
|
}
|
|
2681
3469
|
}
|
|
2682
3470
|
const indexes = [
|
|
2683
|
-
`CREATE INDEX IF NOT EXISTS
|
|
2684
|
-
`CREATE INDEX IF NOT EXISTS
|
|
2685
|
-
`CREATE INDEX IF NOT EXISTS
|
|
3471
|
+
`CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`,
|
|
3472
|
+
`CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`,
|
|
3473
|
+
`CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`
|
|
2686
3474
|
];
|
|
2687
3475
|
for (const indexSql of indexes) {
|
|
2688
|
-
|
|
3476
|
+
await this.client.rpc("exec_sql", { query: indexSql });
|
|
2689
3477
|
}
|
|
2690
3478
|
}
|
|
2691
3479
|
async add(job) {
|
|
@@ -2699,7 +3487,9 @@ class SupabaseQueueStorage {
|
|
|
2699
3487
|
job.progress_details = null;
|
|
2700
3488
|
job.created_at = now;
|
|
2701
3489
|
job.run_after = now;
|
|
2702
|
-
const
|
|
3490
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
3491
|
+
const { data, error } = await this.client.from(this.tableName).insert({
|
|
3492
|
+
...prefixInsertValues,
|
|
2703
3493
|
queue: job.queue,
|
|
2704
3494
|
fingerprint: job.fingerprint,
|
|
2705
3495
|
input: job.input,
|
|
@@ -2720,7 +3510,9 @@ class SupabaseQueueStorage {
|
|
|
2720
3510
|
return job.id;
|
|
2721
3511
|
}
|
|
2722
3512
|
async get(id) {
|
|
2723
|
-
|
|
3513
|
+
let query = this.client.from(this.tableName).select("*").eq("id", id).eq("queue", this.queueName);
|
|
3514
|
+
query = this.applyPrefixFilters(query);
|
|
3515
|
+
const { data, error } = await query.single();
|
|
2724
3516
|
if (error) {
|
|
2725
3517
|
if (error.code === "PGRST116")
|
|
2726
3518
|
return;
|
|
@@ -2730,36 +3522,53 @@ class SupabaseQueueStorage {
|
|
|
2730
3522
|
}
|
|
2731
3523
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2732
3524
|
num = Number(num) || 100;
|
|
2733
|
-
|
|
3525
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", status);
|
|
3526
|
+
query = this.applyPrefixFilters(query);
|
|
3527
|
+
const { data, error } = await query.order("run_after", { ascending: true }).limit(num);
|
|
2734
3528
|
if (error)
|
|
2735
3529
|
throw error;
|
|
2736
3530
|
return data ?? [];
|
|
2737
3531
|
}
|
|
2738
|
-
async next() {
|
|
2739
|
-
|
|
3532
|
+
async next(workerId) {
|
|
3533
|
+
let selectQuery = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", "PENDING" /* PENDING */).lte("run_after", new Date().toISOString());
|
|
3534
|
+
selectQuery = this.applyPrefixFilters(selectQuery);
|
|
3535
|
+
const { data: jobs, error: selectError } = await selectQuery.order("run_after", { ascending: true }).limit(1);
|
|
2740
3536
|
if (selectError)
|
|
2741
3537
|
throw selectError;
|
|
2742
3538
|
if (!jobs || jobs.length === 0)
|
|
2743
3539
|
return;
|
|
2744
3540
|
const job = jobs[0];
|
|
2745
|
-
|
|
3541
|
+
let updateQuery = this.client.from(this.tableName).update({
|
|
2746
3542
|
status: "PROCESSING" /* PROCESSING */,
|
|
2747
|
-
last_ran_at: new Date().toISOString()
|
|
2748
|
-
|
|
3543
|
+
last_ran_at: new Date().toISOString(),
|
|
3544
|
+
worker_id: workerId ?? null
|
|
3545
|
+
}).eq("id", job.id).eq("queue", this.queueName);
|
|
3546
|
+
updateQuery = this.applyPrefixFilters(updateQuery);
|
|
3547
|
+
const { data: updatedJob, error: updateError } = await updateQuery.select().single();
|
|
2749
3548
|
if (updateError)
|
|
2750
3549
|
throw updateError;
|
|
2751
3550
|
return updatedJob;
|
|
2752
3551
|
}
|
|
2753
3552
|
async size(status = "PENDING" /* PENDING */) {
|
|
2754
|
-
|
|
3553
|
+
let query = this.client.from(this.tableName).select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
|
|
3554
|
+
query = this.applyPrefixFilters(query);
|
|
3555
|
+
const { count, error } = await query;
|
|
2755
3556
|
if (error)
|
|
2756
3557
|
throw error;
|
|
2757
3558
|
return count ?? 0;
|
|
2758
3559
|
}
|
|
3560
|
+
async getAllJobs() {
|
|
3561
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
3562
|
+
query = this.applyPrefixFilters(query);
|
|
3563
|
+
const { data, error } = await query;
|
|
3564
|
+
if (error)
|
|
3565
|
+
throw error;
|
|
3566
|
+
return data ?? [];
|
|
3567
|
+
}
|
|
2759
3568
|
async complete(jobDetails) {
|
|
2760
3569
|
const now = new Date().toISOString();
|
|
2761
3570
|
if (jobDetails.status === "DISABLED" /* DISABLED */) {
|
|
2762
|
-
|
|
3571
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2763
3572
|
status: jobDetails.status,
|
|
2764
3573
|
progress: 100,
|
|
2765
3574
|
progress_message: "",
|
|
@@ -2767,16 +3576,20 @@ class SupabaseQueueStorage {
|
|
|
2767
3576
|
completed_at: now,
|
|
2768
3577
|
last_ran_at: now
|
|
2769
3578
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3579
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3580
|
+
const { error: error2 } = await query2;
|
|
2770
3581
|
if (error2)
|
|
2771
3582
|
throw error2;
|
|
2772
3583
|
return;
|
|
2773
3584
|
}
|
|
2774
|
-
|
|
3585
|
+
let getQuery = this.client.from(this.tableName).select("run_attempts").eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3586
|
+
getQuery = this.applyPrefixFilters(getQuery);
|
|
3587
|
+
const { data: current, error: getError } = await getQuery.single();
|
|
2775
3588
|
if (getError)
|
|
2776
3589
|
throw getError;
|
|
2777
3590
|
const nextAttempts = (current?.run_attempts ?? 0) + 1;
|
|
2778
3591
|
if (jobDetails.status === "PENDING" /* PENDING */) {
|
|
2779
|
-
|
|
3592
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2780
3593
|
error: jobDetails.error ?? null,
|
|
2781
3594
|
error_code: jobDetails.error_code ?? null,
|
|
2782
3595
|
status: jobDetails.status,
|
|
@@ -2787,12 +3600,14 @@ class SupabaseQueueStorage {
|
|
|
2787
3600
|
run_attempts: nextAttempts,
|
|
2788
3601
|
last_ran_at: now
|
|
2789
3602
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3603
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3604
|
+
const { error: error2 } = await query2;
|
|
2790
3605
|
if (error2)
|
|
2791
3606
|
throw error2;
|
|
2792
3607
|
return;
|
|
2793
3608
|
}
|
|
2794
3609
|
if (jobDetails.status === "COMPLETED" /* COMPLETED */ || jobDetails.status === "FAILED" /* FAILED */) {
|
|
2795
|
-
|
|
3610
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2796
3611
|
output: jobDetails.output ?? null,
|
|
2797
3612
|
error: jobDetails.error ?? null,
|
|
2798
3613
|
error_code: jobDetails.error_code ?? null,
|
|
@@ -2804,11 +3619,13 @@ class SupabaseQueueStorage {
|
|
|
2804
3619
|
completed_at: now,
|
|
2805
3620
|
last_ran_at: now
|
|
2806
3621
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3622
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3623
|
+
const { error: error2 } = await query2;
|
|
2807
3624
|
if (error2)
|
|
2808
3625
|
throw error2;
|
|
2809
3626
|
return;
|
|
2810
3627
|
}
|
|
2811
|
-
|
|
3628
|
+
let query = this.client.from(this.tableName).update({
|
|
2812
3629
|
status: jobDetails.status,
|
|
2813
3630
|
output: jobDetails.output ?? null,
|
|
2814
3631
|
error: jobDetails.error ?? null,
|
|
@@ -2817,17 +3634,23 @@ class SupabaseQueueStorage {
|
|
|
2817
3634
|
run_attempts: nextAttempts,
|
|
2818
3635
|
last_ran_at: now
|
|
2819
3636
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3637
|
+
query = this.applyPrefixFilters(query);
|
|
3638
|
+
const { error } = await query;
|
|
2820
3639
|
if (error)
|
|
2821
3640
|
throw error;
|
|
2822
3641
|
}
|
|
2823
3642
|
async deleteAll() {
|
|
2824
|
-
|
|
3643
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
|
|
3644
|
+
query = this.applyPrefixFilters(query);
|
|
3645
|
+
const { error } = await query;
|
|
2825
3646
|
if (error)
|
|
2826
3647
|
throw error;
|
|
2827
3648
|
}
|
|
2828
3649
|
async outputForInput(input) {
|
|
2829
3650
|
const fingerprint = await makeFingerprint7(input);
|
|
2830
|
-
|
|
3651
|
+
let query = this.client.from(this.tableName).select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", "COMPLETED" /* COMPLETED */);
|
|
3652
|
+
query = this.applyPrefixFilters(query);
|
|
3653
|
+
const { data, error } = await query.single();
|
|
2831
3654
|
if (error) {
|
|
2832
3655
|
if (error.code === "PGRST116")
|
|
2833
3656
|
return null;
|
|
@@ -2836,42 +3659,617 @@ class SupabaseQueueStorage {
|
|
|
2836
3659
|
return data?.output ?? null;
|
|
2837
3660
|
}
|
|
2838
3661
|
async abort(jobId) {
|
|
2839
|
-
|
|
3662
|
+
let query = this.client.from(this.tableName).update({ status: "ABORTING" /* ABORTING */ }).eq("id", jobId).eq("queue", this.queueName);
|
|
3663
|
+
query = this.applyPrefixFilters(query);
|
|
3664
|
+
const { error } = await query;
|
|
2840
3665
|
if (error)
|
|
2841
3666
|
throw error;
|
|
2842
3667
|
}
|
|
2843
3668
|
async getByRunId(job_run_id) {
|
|
2844
|
-
|
|
3669
|
+
let query = this.client.from(this.tableName).select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
|
|
3670
|
+
query = this.applyPrefixFilters(query);
|
|
3671
|
+
const { data, error } = await query;
|
|
2845
3672
|
if (error)
|
|
2846
3673
|
throw error;
|
|
2847
3674
|
return data ?? [];
|
|
2848
3675
|
}
|
|
2849
3676
|
async saveProgress(jobId, progress, message, details) {
|
|
2850
|
-
|
|
3677
|
+
let query = this.client.from(this.tableName).update({
|
|
2851
3678
|
progress,
|
|
2852
3679
|
progress_message: message,
|
|
2853
3680
|
progress_details: details
|
|
2854
3681
|
}).eq("id", jobId).eq("queue", this.queueName);
|
|
3682
|
+
query = this.applyPrefixFilters(query);
|
|
3683
|
+
const { error } = await query;
|
|
2855
3684
|
if (error)
|
|
2856
3685
|
throw error;
|
|
2857
3686
|
}
|
|
2858
3687
|
async delete(jobId) {
|
|
2859
|
-
|
|
3688
|
+
let query = this.client.from(this.tableName).delete().eq("id", jobId).eq("queue", this.queueName);
|
|
3689
|
+
query = this.applyPrefixFilters(query);
|
|
3690
|
+
const { error } = await query;
|
|
2860
3691
|
if (error)
|
|
2861
3692
|
throw error;
|
|
2862
3693
|
}
|
|
2863
3694
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2864
3695
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
2865
|
-
|
|
3696
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
|
|
3697
|
+
query = this.applyPrefixFilters(query);
|
|
3698
|
+
const { error } = await query;
|
|
3699
|
+
if (error)
|
|
3700
|
+
throw error;
|
|
3701
|
+
}
|
|
3702
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
3703
|
+
if (!job)
|
|
3704
|
+
return false;
|
|
3705
|
+
if (job.queue !== this.queueName) {
|
|
3706
|
+
return false;
|
|
3707
|
+
}
|
|
3708
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
3709
|
+
return true;
|
|
3710
|
+
}
|
|
3711
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
3712
|
+
if (Object.keys(filterValues).length === 0) {
|
|
3713
|
+
return true;
|
|
3714
|
+
}
|
|
3715
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
3716
|
+
if (job[key] !== value) {
|
|
3717
|
+
return false;
|
|
3718
|
+
}
|
|
3719
|
+
}
|
|
3720
|
+
return true;
|
|
3721
|
+
}
|
|
3722
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
3723
|
+
if (prefixFilter === undefined) {
|
|
3724
|
+
return false;
|
|
3725
|
+
}
|
|
3726
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
3727
|
+
return true;
|
|
3728
|
+
}
|
|
3729
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
3730
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
3731
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
3732
|
+
return true;
|
|
3733
|
+
}
|
|
3734
|
+
for (const key of instanceKeys) {
|
|
3735
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
3736
|
+
return true;
|
|
3737
|
+
}
|
|
3738
|
+
}
|
|
3739
|
+
return false;
|
|
3740
|
+
}
|
|
3741
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
3742
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
3743
|
+
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
3744
|
+
query = query.eq(key, value);
|
|
3745
|
+
}
|
|
3746
|
+
const { data, error } = await query;
|
|
2866
3747
|
if (error)
|
|
2867
3748
|
throw error;
|
|
3749
|
+
return data ?? [];
|
|
3750
|
+
}
|
|
3751
|
+
subscribeToChanges(callback, options) {
|
|
3752
|
+
return this.subscribeToChangesWithRealtime(callback, options?.prefixFilter);
|
|
3753
|
+
}
|
|
3754
|
+
subscribeToChangesWithRealtime(callback, prefixFilter) {
|
|
3755
|
+
const channelName = `queue-${this.tableName}-${this.queueName}-${Date.now()}`;
|
|
3756
|
+
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
3757
|
+
event: "*",
|
|
3758
|
+
schema: "public",
|
|
3759
|
+
table: this.tableName,
|
|
3760
|
+
filter: `queue=eq.${this.queueName}`
|
|
3761
|
+
}, (payload) => {
|
|
3762
|
+
const newJob = payload.new;
|
|
3763
|
+
const oldJob = payload.old;
|
|
3764
|
+
const newMatches = this.matchesPrefixFilter(newJob, prefixFilter);
|
|
3765
|
+
const oldMatches = this.matchesPrefixFilter(oldJob, prefixFilter);
|
|
3766
|
+
if (!newMatches && !oldMatches) {
|
|
3767
|
+
return;
|
|
3768
|
+
}
|
|
3769
|
+
callback({
|
|
3770
|
+
type: payload.eventType.toUpperCase(),
|
|
3771
|
+
old: oldJob && Object.keys(oldJob).length > 0 ? oldJob : undefined,
|
|
3772
|
+
new: newJob && Object.keys(newJob).length > 0 ? newJob : undefined
|
|
3773
|
+
});
|
|
3774
|
+
}).subscribe();
|
|
3775
|
+
return () => {
|
|
3776
|
+
if (this.realtimeChannel) {
|
|
3777
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
3778
|
+
this.realtimeChannel = null;
|
|
3779
|
+
}
|
|
3780
|
+
};
|
|
3781
|
+
}
|
|
3782
|
+
getPollingManager() {
|
|
3783
|
+
if (!this.pollingManager) {
|
|
3784
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
3785
|
+
const jobs = await this.getAllJobs();
|
|
3786
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
3787
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
3788
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
3789
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
3790
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
3791
|
+
});
|
|
3792
|
+
}
|
|
3793
|
+
return this.pollingManager;
|
|
3794
|
+
}
|
|
3795
|
+
subscribeWithCustomPrefixFilterPolling(callback, prefixFilter, intervalMs) {
|
|
3796
|
+
let lastKnownJobs = new Map;
|
|
3797
|
+
let cancelled = false;
|
|
3798
|
+
const poll = async () => {
|
|
3799
|
+
if (cancelled)
|
|
3800
|
+
return;
|
|
3801
|
+
try {
|
|
3802
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
3803
|
+
if (cancelled)
|
|
3804
|
+
return;
|
|
3805
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
3806
|
+
for (const [id, job] of currentMap) {
|
|
3807
|
+
const old = lastKnownJobs.get(id);
|
|
3808
|
+
if (!old) {
|
|
3809
|
+
callback({ type: "INSERT", new: job });
|
|
3810
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
3811
|
+
callback({ type: "UPDATE", old, new: job });
|
|
3812
|
+
}
|
|
3813
|
+
}
|
|
3814
|
+
for (const [id, job] of lastKnownJobs) {
|
|
3815
|
+
if (!currentMap.has(id)) {
|
|
3816
|
+
callback({ type: "DELETE", old: job });
|
|
3817
|
+
}
|
|
3818
|
+
}
|
|
3819
|
+
lastKnownJobs = currentMap;
|
|
3820
|
+
} catch {}
|
|
3821
|
+
};
|
|
3822
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
3823
|
+
poll();
|
|
3824
|
+
return () => {
|
|
3825
|
+
cancelled = true;
|
|
3826
|
+
clearInterval(intervalId);
|
|
3827
|
+
};
|
|
3828
|
+
}
|
|
3829
|
+
subscribeToChangesWithPolling(callback, options) {
|
|
3830
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
3831
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
3832
|
+
return this.subscribeWithCustomPrefixFilterPolling(callback, options.prefixFilter, intervalMs);
|
|
3833
|
+
}
|
|
3834
|
+
const manager = this.getPollingManager();
|
|
3835
|
+
return manager.subscribe(callback, { intervalMs });
|
|
3836
|
+
}
|
|
3837
|
+
}
|
|
3838
|
+
// src/limiter/PostgresRateLimiterStorage.ts
|
|
3839
|
+
import { createServiceToken as createServiceToken22 } from "@workglow/util";
|
|
3840
|
+
var POSTGRES_RATE_LIMITER_STORAGE = createServiceToken22("ratelimiter.storage.postgres");
|
|
3841
|
+
|
|
3842
|
+
class PostgresRateLimiterStorage {
|
|
3843
|
+
db;
|
|
3844
|
+
prefixes;
|
|
3845
|
+
prefixValues;
|
|
3846
|
+
executionTableName;
|
|
3847
|
+
nextAvailableTableName;
|
|
3848
|
+
constructor(db, options) {
|
|
3849
|
+
this.db = db;
|
|
3850
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3851
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3852
|
+
if (this.prefixes.length > 0) {
|
|
3853
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3854
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
3855
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
3856
|
+
} else {
|
|
3857
|
+
this.executionTableName = "rate_limit_executions";
|
|
3858
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
3859
|
+
}
|
|
3860
|
+
}
|
|
3861
|
+
getPrefixColumnType(type) {
|
|
3862
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
3863
|
+
}
|
|
3864
|
+
buildPrefixColumnsSql() {
|
|
3865
|
+
if (this.prefixes.length === 0)
|
|
3866
|
+
return "";
|
|
3867
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
3868
|
+
`) + `,
|
|
3869
|
+
`;
|
|
3870
|
+
}
|
|
3871
|
+
getPrefixColumnNames() {
|
|
3872
|
+
return this.prefixes.map((p) => p.name);
|
|
3873
|
+
}
|
|
3874
|
+
buildPrefixWhereClause(startParam) {
|
|
3875
|
+
if (this.prefixes.length === 0) {
|
|
3876
|
+
return { conditions: "", params: [] };
|
|
3877
|
+
}
|
|
3878
|
+
const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
|
|
3879
|
+
const params = this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3880
|
+
return { conditions: " AND " + conditions, params };
|
|
3881
|
+
}
|
|
3882
|
+
getPrefixParamValues() {
|
|
3883
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3884
|
+
}
|
|
3885
|
+
async setupDatabase() {
|
|
3886
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
3887
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3888
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3889
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
3890
|
+
await this.db.query(`
|
|
3891
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
3892
|
+
id SERIAL PRIMARY KEY,
|
|
3893
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3894
|
+
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
3895
|
+
)
|
|
3896
|
+
`);
|
|
3897
|
+
await this.db.query(`
|
|
3898
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
3899
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
3900
|
+
`);
|
|
3901
|
+
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
3902
|
+
await this.db.query(`
|
|
3903
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
3904
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3905
|
+
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
3906
|
+
PRIMARY KEY (${primaryKeyColumns})
|
|
3907
|
+
)
|
|
3908
|
+
`);
|
|
3909
|
+
}
|
|
3910
|
+
async recordExecution(queueName) {
|
|
3911
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3912
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3913
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
3914
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
|
|
3915
|
+
const queueParamNum = prefixColumnNames.length + 1;
|
|
3916
|
+
await this.db.query(`
|
|
3917
|
+
INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
|
|
3918
|
+
VALUES (${prefixParamPlaceholders}$${queueParamNum})
|
|
3919
|
+
`, [...prefixParamValues, queueName]);
|
|
3920
|
+
}
|
|
3921
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
3922
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
3923
|
+
const result = await this.db.query(`
|
|
3924
|
+
SELECT COUNT(*) AS count
|
|
3925
|
+
FROM ${this.executionTableName}
|
|
3926
|
+
WHERE queue_name = $1 AND executed_at > $2${prefixConditions}
|
|
3927
|
+
`, [queueName, windowStartTime, ...prefixParams]);
|
|
3928
|
+
return parseInt(result.rows[0]?.count ?? "0", 10);
|
|
3929
|
+
}
|
|
3930
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
3931
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
3932
|
+
const result = await this.db.query(`
|
|
3933
|
+
SELECT executed_at
|
|
3934
|
+
FROM ${this.executionTableName}
|
|
3935
|
+
WHERE queue_name = $1${prefixConditions}
|
|
3936
|
+
ORDER BY executed_at ASC
|
|
3937
|
+
LIMIT 1 OFFSET $2
|
|
3938
|
+
`, [queueName, offset, ...prefixParams]);
|
|
3939
|
+
const executedAt = result.rows[0]?.executed_at;
|
|
3940
|
+
if (!executedAt)
|
|
3941
|
+
return;
|
|
3942
|
+
return new Date(executedAt).toISOString();
|
|
3943
|
+
}
|
|
3944
|
+
async getNextAvailableTime(queueName) {
|
|
3945
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
3946
|
+
const result = await this.db.query(`
|
|
3947
|
+
SELECT next_available_at
|
|
3948
|
+
FROM ${this.nextAvailableTableName}
|
|
3949
|
+
WHERE queue_name = $1${prefixConditions}
|
|
3950
|
+
`, [queueName, ...prefixParams]);
|
|
3951
|
+
const nextAvailableAt = result.rows[0]?.next_available_at;
|
|
3952
|
+
if (!nextAvailableAt)
|
|
3953
|
+
return;
|
|
3954
|
+
return new Date(nextAvailableAt).toISOString();
|
|
3955
|
+
}
|
|
3956
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
3957
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3958
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3959
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
3960
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
|
|
3961
|
+
const baseParamStart = prefixColumnNames.length + 1;
|
|
3962
|
+
const conflictColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
3963
|
+
await this.db.query(`
|
|
3964
|
+
INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
|
|
3965
|
+
VALUES (${prefixParamPlaceholders}$${baseParamStart}, $${baseParamStart + 1})
|
|
3966
|
+
ON CONFLICT (${conflictColumns})
|
|
3967
|
+
DO UPDATE SET next_available_at = EXCLUDED.next_available_at
|
|
3968
|
+
`, [...prefixParamValues, queueName, nextAvailableAt]);
|
|
3969
|
+
}
|
|
3970
|
+
async clear(queueName) {
|
|
3971
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
3972
|
+
await this.db.query(`DELETE FROM ${this.executionTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
|
|
3973
|
+
await this.db.query(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
|
|
3974
|
+
}
|
|
3975
|
+
}
|
|
3976
|
+
// src/limiter/SqliteRateLimiterStorage.ts
|
|
3977
|
+
import { createServiceToken as createServiceToken23, sleep as sleep5, toSQLiteTimestamp } from "@workglow/util";
|
|
3978
|
+
var SQLITE_RATE_LIMITER_STORAGE = createServiceToken23("ratelimiter.storage.sqlite");
|
|
3979
|
+
|
|
3980
|
+
class SqliteRateLimiterStorage {
|
|
3981
|
+
db;
|
|
3982
|
+
prefixes;
|
|
3983
|
+
prefixValues;
|
|
3984
|
+
executionTableName;
|
|
3985
|
+
nextAvailableTableName;
|
|
3986
|
+
constructor(db, options) {
|
|
3987
|
+
this.db = db;
|
|
3988
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3989
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3990
|
+
if (this.prefixes.length > 0) {
|
|
3991
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3992
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
3993
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
3994
|
+
} else {
|
|
3995
|
+
this.executionTableName = "rate_limit_executions";
|
|
3996
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
3997
|
+
}
|
|
3998
|
+
}
|
|
3999
|
+
getPrefixColumnType(type) {
|
|
4000
|
+
return type === "uuid" ? "TEXT" : "INTEGER";
|
|
4001
|
+
}
|
|
4002
|
+
buildPrefixColumnsSql() {
|
|
4003
|
+
if (this.prefixes.length === 0)
|
|
4004
|
+
return "";
|
|
4005
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
4006
|
+
`) + `,
|
|
4007
|
+
`;
|
|
4008
|
+
}
|
|
4009
|
+
getPrefixColumnNames() {
|
|
4010
|
+
return this.prefixes.map((p) => p.name);
|
|
4011
|
+
}
|
|
4012
|
+
buildPrefixWhereClause() {
|
|
4013
|
+
if (this.prefixes.length === 0) {
|
|
4014
|
+
return "";
|
|
4015
|
+
}
|
|
4016
|
+
const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
|
|
4017
|
+
return " AND " + conditions;
|
|
4018
|
+
}
|
|
4019
|
+
getPrefixParamValues() {
|
|
4020
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
4021
|
+
}
|
|
4022
|
+
async setupDatabase() {
|
|
4023
|
+
await sleep5(0);
|
|
4024
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
4025
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4026
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
4027
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
4028
|
+
this.db.exec(`
|
|
4029
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
4030
|
+
id INTEGER PRIMARY KEY,
|
|
4031
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
4032
|
+
executed_at TEXT DEFAULT CURRENT_TIMESTAMP
|
|
4033
|
+
);
|
|
4034
|
+
|
|
4035
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
4036
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at);
|
|
4037
|
+
`);
|
|
4038
|
+
this.db.exec(`
|
|
4039
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
4040
|
+
${prefixColumnsSql}queue_name TEXT PRIMARY KEY,
|
|
4041
|
+
next_available_at TEXT
|
|
4042
|
+
);
|
|
4043
|
+
`);
|
|
4044
|
+
}
|
|
4045
|
+
async recordExecution(queueName) {
|
|
4046
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4047
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
4048
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
4049
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
4050
|
+
const stmt = this.db.prepare(`
|
|
4051
|
+
INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
|
|
4052
|
+
VALUES (${prefixPlaceholders}?)
|
|
4053
|
+
`);
|
|
4054
|
+
stmt.run(...prefixParamValues, queueName);
|
|
4055
|
+
}
|
|
4056
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
4057
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
4058
|
+
const prefixParams = this.getPrefixParamValues();
|
|
4059
|
+
const thresholdTime = toSQLiteTimestamp(new Date(windowStartTime));
|
|
4060
|
+
const stmt = this.db.prepare(`
|
|
4061
|
+
SELECT COUNT(*) AS count
|
|
4062
|
+
FROM ${this.executionTableName}
|
|
4063
|
+
WHERE queue_name = ? AND executed_at > ?${prefixConditions}
|
|
4064
|
+
`);
|
|
4065
|
+
const result = stmt.get(queueName, thresholdTime, ...prefixParams);
|
|
4066
|
+
return result?.count ?? 0;
|
|
4067
|
+
}
|
|
4068
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
4069
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
4070
|
+
const prefixParams = this.getPrefixParamValues();
|
|
4071
|
+
const stmt = this.db.prepare(`
|
|
4072
|
+
SELECT executed_at
|
|
4073
|
+
FROM ${this.executionTableName}
|
|
4074
|
+
WHERE queue_name = ?${prefixConditions}
|
|
4075
|
+
ORDER BY executed_at ASC
|
|
4076
|
+
LIMIT 1 OFFSET ?
|
|
4077
|
+
`);
|
|
4078
|
+
const result = stmt.get(queueName, ...prefixParams, offset);
|
|
4079
|
+
if (!result)
|
|
4080
|
+
return;
|
|
4081
|
+
return result.executed_at + "Z";
|
|
4082
|
+
}
|
|
4083
|
+
async getNextAvailableTime(queueName) {
|
|
4084
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
4085
|
+
const prefixParams = this.getPrefixParamValues();
|
|
4086
|
+
const stmt = this.db.prepare(`
|
|
4087
|
+
SELECT next_available_at
|
|
4088
|
+
FROM ${this.nextAvailableTableName}
|
|
4089
|
+
WHERE queue_name = ?${prefixConditions}
|
|
4090
|
+
`);
|
|
4091
|
+
const result = stmt.get(queueName, ...prefixParams);
|
|
4092
|
+
if (!result?.next_available_at)
|
|
4093
|
+
return;
|
|
4094
|
+
return result.next_available_at + "Z";
|
|
4095
|
+
}
|
|
4096
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
4097
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4098
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
4099
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
4100
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
4101
|
+
const stmt = this.db.prepare(`
|
|
4102
|
+
INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
|
|
4103
|
+
VALUES (${prefixPlaceholders}?, ?)
|
|
4104
|
+
ON CONFLICT(queue_name) DO UPDATE SET next_available_at = excluded.next_available_at
|
|
4105
|
+
`);
|
|
4106
|
+
stmt.run(...prefixParamValues, queueName, nextAvailableAt);
|
|
4107
|
+
}
|
|
4108
|
+
async clear(queueName) {
|
|
4109
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
4110
|
+
const prefixParams = this.getPrefixParamValues();
|
|
4111
|
+
this.db.prepare(`DELETE FROM ${this.executionTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
|
|
4112
|
+
this.db.prepare(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
|
|
4113
|
+
}
|
|
4114
|
+
}
|
|
4115
|
+
// src/limiter/SupabaseRateLimiterStorage.ts
|
|
4116
|
+
import { createServiceToken as createServiceToken24 } from "@workglow/util";
|
|
4117
|
+
var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken24("ratelimiter.storage.supabase");
|
|
4118
|
+
|
|
4119
|
+
class SupabaseRateLimiterStorage {
|
|
4120
|
+
client;
|
|
4121
|
+
prefixes;
|
|
4122
|
+
prefixValues;
|
|
4123
|
+
executionTableName;
|
|
4124
|
+
nextAvailableTableName;
|
|
4125
|
+
constructor(client, options) {
|
|
4126
|
+
this.client = client;
|
|
4127
|
+
this.prefixes = options?.prefixes ?? [];
|
|
4128
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
4129
|
+
if (this.prefixes.length > 0) {
|
|
4130
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
4131
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
4132
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
4133
|
+
} else {
|
|
4134
|
+
this.executionTableName = "rate_limit_executions";
|
|
4135
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
4136
|
+
}
|
|
4137
|
+
}
|
|
4138
|
+
getPrefixColumnType(type) {
|
|
4139
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
4140
|
+
}
|
|
4141
|
+
buildPrefixColumnsSql() {
|
|
4142
|
+
if (this.prefixes.length === 0)
|
|
4143
|
+
return "";
|
|
4144
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
4145
|
+
`) + `,
|
|
4146
|
+
`;
|
|
4147
|
+
}
|
|
4148
|
+
getPrefixColumnNames() {
|
|
4149
|
+
return this.prefixes.map((p) => p.name);
|
|
4150
|
+
}
|
|
4151
|
+
applyPrefixFilters(query) {
|
|
4152
|
+
let result = query;
|
|
4153
|
+
for (const prefix of this.prefixes) {
|
|
4154
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
4155
|
+
}
|
|
4156
|
+
return result;
|
|
4157
|
+
}
|
|
4158
|
+
getPrefixInsertValues() {
|
|
4159
|
+
const values = {};
|
|
4160
|
+
for (const prefix of this.prefixes) {
|
|
4161
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
4162
|
+
}
|
|
4163
|
+
return values;
|
|
4164
|
+
}
|
|
4165
|
+
async setupDatabase() {
|
|
4166
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
4167
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4168
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
4169
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
4170
|
+
const createExecTableSql = `
|
|
4171
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
4172
|
+
id SERIAL PRIMARY KEY,
|
|
4173
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
4174
|
+
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
4175
|
+
)
|
|
4176
|
+
`;
|
|
4177
|
+
const { error: execTableError } = await this.client.rpc("exec_sql", {
|
|
4178
|
+
query: createExecTableSql
|
|
4179
|
+
});
|
|
4180
|
+
if (execTableError && execTableError.code !== "42P07") {
|
|
4181
|
+
throw execTableError;
|
|
4182
|
+
}
|
|
4183
|
+
const createExecIndexSql = `
|
|
4184
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
4185
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
4186
|
+
`;
|
|
4187
|
+
await this.client.rpc("exec_sql", { query: createExecIndexSql });
|
|
4188
|
+
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
4189
|
+
const createNextTableSql = `
|
|
4190
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
4191
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
4192
|
+
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
4193
|
+
PRIMARY KEY (${primaryKeyColumns})
|
|
4194
|
+
)
|
|
4195
|
+
`;
|
|
4196
|
+
const { error: nextTableError } = await this.client.rpc("exec_sql", {
|
|
4197
|
+
query: createNextTableSql
|
|
4198
|
+
});
|
|
4199
|
+
if (nextTableError && nextTableError.code !== "42P07") {
|
|
4200
|
+
throw nextTableError;
|
|
4201
|
+
}
|
|
4202
|
+
}
|
|
4203
|
+
async recordExecution(queueName) {
|
|
4204
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
4205
|
+
const { error } = await this.client.from(this.executionTableName).insert({
|
|
4206
|
+
...prefixInsertValues,
|
|
4207
|
+
queue_name: queueName
|
|
4208
|
+
});
|
|
4209
|
+
if (error)
|
|
4210
|
+
throw error;
|
|
4211
|
+
}
|
|
4212
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
4213
|
+
let query = this.client.from(this.executionTableName).select("*", { count: "exact", head: true }).eq("queue_name", queueName).gt("executed_at", windowStartTime);
|
|
4214
|
+
query = this.applyPrefixFilters(query);
|
|
4215
|
+
const { count, error } = await query;
|
|
4216
|
+
if (error)
|
|
4217
|
+
throw error;
|
|
4218
|
+
return count ?? 0;
|
|
4219
|
+
}
|
|
4220
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
4221
|
+
let query = this.client.from(this.executionTableName).select("executed_at").eq("queue_name", queueName);
|
|
4222
|
+
query = this.applyPrefixFilters(query);
|
|
4223
|
+
const { data, error } = await query.order("executed_at", { ascending: true }).range(offset, offset);
|
|
4224
|
+
if (error)
|
|
4225
|
+
throw error;
|
|
4226
|
+
if (!data || data.length === 0)
|
|
4227
|
+
return;
|
|
4228
|
+
return new Date(data[0].executed_at).toISOString();
|
|
4229
|
+
}
|
|
4230
|
+
async getNextAvailableTime(queueName) {
|
|
4231
|
+
let query = this.client.from(this.nextAvailableTableName).select("next_available_at").eq("queue_name", queueName);
|
|
4232
|
+
query = this.applyPrefixFilters(query);
|
|
4233
|
+
const { data, error } = await query.single();
|
|
4234
|
+
if (error) {
|
|
4235
|
+
if (error.code === "PGRST116")
|
|
4236
|
+
return;
|
|
4237
|
+
throw error;
|
|
4238
|
+
}
|
|
4239
|
+
if (!data?.next_available_at)
|
|
4240
|
+
return;
|
|
4241
|
+
return new Date(data.next_available_at).toISOString();
|
|
4242
|
+
}
|
|
4243
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
4244
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
4245
|
+
const { error } = await this.client.from(this.nextAvailableTableName).upsert({
|
|
4246
|
+
...prefixInsertValues,
|
|
4247
|
+
queue_name: queueName,
|
|
4248
|
+
next_available_at: nextAvailableAt
|
|
4249
|
+
}, {
|
|
4250
|
+
onConflict: this.prefixes.length > 0 ? `${this.getPrefixColumnNames().join(",")},queue_name` : "queue_name"
|
|
4251
|
+
});
|
|
4252
|
+
if (error)
|
|
4253
|
+
throw error;
|
|
4254
|
+
}
|
|
4255
|
+
async clear(queueName) {
|
|
4256
|
+
let execQuery = this.client.from(this.executionTableName).delete().eq("queue_name", queueName);
|
|
4257
|
+
execQuery = this.applyPrefixFilters(execQuery);
|
|
4258
|
+
const { error: execError } = await execQuery;
|
|
4259
|
+
if (execError)
|
|
4260
|
+
throw execError;
|
|
4261
|
+
let nextQuery = this.client.from(this.nextAvailableTableName).delete().eq("queue_name", queueName);
|
|
4262
|
+
nextQuery = this.applyPrefixFilters(nextQuery);
|
|
4263
|
+
const { error: nextError } = await nextQuery;
|
|
4264
|
+
if (nextError)
|
|
4265
|
+
throw nextError;
|
|
2868
4266
|
}
|
|
2869
4267
|
}
|
|
2870
4268
|
// src/kv/IndexedDbKvRepository.ts
|
|
2871
|
-
import { createServiceToken as
|
|
4269
|
+
import { createServiceToken as createServiceToken26 } from "@workglow/util";
|
|
2872
4270
|
|
|
2873
4271
|
// src/tabular/IndexedDbTabularRepository.ts
|
|
2874
|
-
import { createServiceToken as
|
|
4272
|
+
import { createServiceToken as createServiceToken25 } from "@workglow/util";
|
|
2875
4273
|
|
|
2876
4274
|
// src/util/IndexedDbTable.ts
|
|
2877
4275
|
var METADATA_STORE_NAME = "__schema_metadata__";
|
|
@@ -3211,7 +4609,7 @@ async function dropIndexedDbTable(tableName) {
|
|
|
3211
4609
|
}
|
|
3212
4610
|
|
|
3213
4611
|
// src/tabular/IndexedDbTabularRepository.ts
|
|
3214
|
-
var IDB_TABULAR_REPOSITORY =
|
|
4612
|
+
var IDB_TABULAR_REPOSITORY = createServiceToken25("storage.tabularRepository.indexedDb");
|
|
3215
4613
|
|
|
3216
4614
|
class IndexedDbTabularRepository extends TabularRepository {
|
|
3217
4615
|
table;
|
|
@@ -3547,7 +4945,7 @@ class IndexedDbTabularRepository extends TabularRepository {
|
|
|
3547
4945
|
}
|
|
3548
4946
|
|
|
3549
4947
|
// src/kv/IndexedDbKvRepository.ts
|
|
3550
|
-
var IDB_KV_REPOSITORY =
|
|
4948
|
+
var IDB_KV_REPOSITORY = createServiceToken26("storage.kvRepository.indexedDb");
|
|
3551
4949
|
|
|
3552
4950
|
class IndexedDbKvRepository extends KvViaTabularRepository {
|
|
3553
4951
|
dbName;
|
|
@@ -3559,18 +4957,42 @@ class IndexedDbKvRepository extends KvViaTabularRepository {
|
|
|
3559
4957
|
}
|
|
3560
4958
|
}
|
|
3561
4959
|
// src/queue/IndexedDbQueueStorage.ts
|
|
3562
|
-
import { createServiceToken as
|
|
3563
|
-
var INDEXED_DB_QUEUE_STORAGE =
|
|
4960
|
+
import { createServiceToken as createServiceToken27, makeFingerprint as makeFingerprint8, uuid4 as uuid45 } from "@workglow/util";
|
|
4961
|
+
var INDEXED_DB_QUEUE_STORAGE = createServiceToken27("jobqueue.storage.indexedDb");
|
|
3564
4962
|
|
|
3565
4963
|
class IndexedDbQueueStorage {
|
|
3566
4964
|
queueName;
|
|
3567
4965
|
db;
|
|
3568
4966
|
tableName;
|
|
3569
4967
|
migrationOptions;
|
|
3570
|
-
|
|
4968
|
+
prefixes;
|
|
4969
|
+
prefixValues;
|
|
4970
|
+
pollingManager = null;
|
|
4971
|
+
constructor(queueName, options = {}) {
|
|
3571
4972
|
this.queueName = queueName;
|
|
3572
|
-
this.
|
|
3573
|
-
this.
|
|
4973
|
+
this.migrationOptions = options;
|
|
4974
|
+
this.prefixes = options.prefixes ?? [];
|
|
4975
|
+
this.prefixValues = options.prefixValues ?? {};
|
|
4976
|
+
if (this.prefixes.length > 0) {
|
|
4977
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
4978
|
+
this.tableName = `jobs_${prefixNames}`;
|
|
4979
|
+
} else {
|
|
4980
|
+
this.tableName = "jobs";
|
|
4981
|
+
}
|
|
4982
|
+
}
|
|
4983
|
+
getPrefixColumnNames() {
|
|
4984
|
+
return this.prefixes.map((p) => p.name);
|
|
4985
|
+
}
|
|
4986
|
+
matchesPrefixes(job) {
|
|
4987
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
4988
|
+
if (job[key] !== value) {
|
|
4989
|
+
return false;
|
|
4990
|
+
}
|
|
4991
|
+
}
|
|
4992
|
+
return true;
|
|
4993
|
+
}
|
|
4994
|
+
getPrefixKeyValues() {
|
|
4995
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3574
4996
|
}
|
|
3575
4997
|
async getDb() {
|
|
3576
4998
|
if (this.db)
|
|
@@ -3579,25 +5001,29 @@ class IndexedDbQueueStorage {
|
|
|
3579
5001
|
return this.db;
|
|
3580
5002
|
}
|
|
3581
5003
|
async setupDatabase() {
|
|
5004
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
5005
|
+
const buildKeyPath = (basePath) => {
|
|
5006
|
+
return [...prefixColumnNames, ...basePath];
|
|
5007
|
+
};
|
|
3582
5008
|
const expectedIndexes = [
|
|
3583
5009
|
{
|
|
3584
|
-
name: "
|
|
3585
|
-
keyPath:
|
|
5010
|
+
name: "queue_status",
|
|
5011
|
+
keyPath: buildKeyPath(["queue", "status"]),
|
|
3586
5012
|
options: { unique: false }
|
|
3587
5013
|
},
|
|
3588
5014
|
{
|
|
3589
|
-
name: "
|
|
3590
|
-
keyPath: ["status", "run_after"],
|
|
5015
|
+
name: "queue_status_run_after",
|
|
5016
|
+
keyPath: buildKeyPath(["queue", "status", "run_after"]),
|
|
3591
5017
|
options: { unique: false }
|
|
3592
5018
|
},
|
|
3593
5019
|
{
|
|
3594
|
-
name: "
|
|
3595
|
-
keyPath:
|
|
5020
|
+
name: "queue_job_run_id",
|
|
5021
|
+
keyPath: buildKeyPath(["queue", "job_run_id"]),
|
|
3596
5022
|
options: { unique: false }
|
|
3597
5023
|
},
|
|
3598
5024
|
{
|
|
3599
|
-
name: "
|
|
3600
|
-
keyPath: ["fingerprint", "status"],
|
|
5025
|
+
name: "queue_fingerprint_status",
|
|
5026
|
+
keyPath: buildKeyPath(["queue", "fingerprint", "status"]),
|
|
3601
5027
|
options: { unique: false }
|
|
3602
5028
|
}
|
|
3603
5029
|
];
|
|
@@ -3606,21 +5032,25 @@ class IndexedDbQueueStorage {
|
|
|
3606
5032
|
async add(job) {
|
|
3607
5033
|
const db = await this.getDb();
|
|
3608
5034
|
const now = new Date().toISOString();
|
|
3609
|
-
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
5035
|
+
const jobWithPrefixes = job;
|
|
5036
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid45();
|
|
5037
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid45();
|
|
5038
|
+
jobWithPrefixes.queue = this.queueName;
|
|
5039
|
+
jobWithPrefixes.fingerprint = await makeFingerprint8(jobWithPrefixes.input);
|
|
5040
|
+
jobWithPrefixes.status = "PENDING" /* PENDING */;
|
|
5041
|
+
jobWithPrefixes.progress = 0;
|
|
5042
|
+
jobWithPrefixes.progress_message = "";
|
|
5043
|
+
jobWithPrefixes.progress_details = null;
|
|
5044
|
+
jobWithPrefixes.created_at = now;
|
|
5045
|
+
jobWithPrefixes.run_after = now;
|
|
5046
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5047
|
+
jobWithPrefixes[key] = value;
|
|
5048
|
+
}
|
|
3619
5049
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3620
5050
|
const store = tx.objectStore(this.tableName);
|
|
3621
5051
|
return new Promise((resolve, reject) => {
|
|
3622
|
-
const request = store.add(
|
|
3623
|
-
tx.oncomplete = () => resolve(
|
|
5052
|
+
const request = store.add(jobWithPrefixes);
|
|
5053
|
+
tx.oncomplete = () => resolve(jobWithPrefixes.id);
|
|
3624
5054
|
tx.onerror = () => reject(tx.error);
|
|
3625
5055
|
request.onerror = () => reject(request.error);
|
|
3626
5056
|
});
|
|
@@ -3631,7 +5061,14 @@ class IndexedDbQueueStorage {
|
|
|
3631
5061
|
const store = tx.objectStore(this.tableName);
|
|
3632
5062
|
const request = store.get(id);
|
|
3633
5063
|
return new Promise((resolve, reject) => {
|
|
3634
|
-
request.onsuccess = () =>
|
|
5064
|
+
request.onsuccess = () => {
|
|
5065
|
+
const job = request.result;
|
|
5066
|
+
if (job && job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
5067
|
+
resolve(job);
|
|
5068
|
+
} else {
|
|
5069
|
+
resolve(undefined);
|
|
5070
|
+
}
|
|
5071
|
+
};
|
|
3635
5072
|
request.onerror = () => reject(request.error);
|
|
3636
5073
|
tx.onerror = () => reject(tx.error);
|
|
3637
5074
|
});
|
|
@@ -3640,10 +5077,11 @@ class IndexedDbQueueStorage {
|
|
|
3640
5077
|
const db = await this.getDb();
|
|
3641
5078
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3642
5079
|
const store = tx.objectStore(this.tableName);
|
|
3643
|
-
const index = store.index("
|
|
5080
|
+
const index = store.index("queue_status_run_after");
|
|
5081
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3644
5082
|
return new Promise((resolve, reject) => {
|
|
3645
5083
|
const ret = new Map;
|
|
3646
|
-
const keyRange = IDBKeyRange.bound([status, ""], [status, ""]);
|
|
5084
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, status, ""], [...prefixKeyValues, this.queueName, status, ""]);
|
|
3647
5085
|
const cursorRequest = index.openCursor(keyRange);
|
|
3648
5086
|
const handleCursor = (e) => {
|
|
3649
5087
|
const cursor = e.target.result;
|
|
@@ -3651,7 +5089,10 @@ class IndexedDbQueueStorage {
|
|
|
3651
5089
|
resolve(Array.from(ret.values()));
|
|
3652
5090
|
return;
|
|
3653
5091
|
}
|
|
3654
|
-
|
|
5092
|
+
const job = cursor.value;
|
|
5093
|
+
if (this.matchesPrefixes(job)) {
|
|
5094
|
+
ret.set(cursor.value.id, cursor.value);
|
|
5095
|
+
}
|
|
3655
5096
|
cursor.continue();
|
|
3656
5097
|
};
|
|
3657
5098
|
cursorRequest.onsuccess = handleCursor;
|
|
@@ -3659,14 +5100,15 @@ class IndexedDbQueueStorage {
|
|
|
3659
5100
|
tx.onerror = () => reject(tx.error);
|
|
3660
5101
|
});
|
|
3661
5102
|
}
|
|
3662
|
-
async next() {
|
|
5103
|
+
async next(workerId) {
|
|
3663
5104
|
const db = await this.getDb();
|
|
3664
5105
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3665
5106
|
const store = tx.objectStore(this.tableName);
|
|
3666
|
-
const index = store.index("
|
|
5107
|
+
const index = store.index("queue_status_run_after");
|
|
3667
5108
|
const now = new Date().toISOString();
|
|
5109
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3668
5110
|
return new Promise((resolve, reject) => {
|
|
3669
|
-
const cursorRequest = index.openCursor(IDBKeyRange.bound(["PENDING" /* PENDING */, ""], ["PENDING" /* PENDING */, now], false, true));
|
|
5111
|
+
const cursorRequest = index.openCursor(IDBKeyRange.bound([...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, ""], [...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, now], false, true));
|
|
3670
5112
|
let jobToReturn;
|
|
3671
5113
|
cursorRequest.onsuccess = (e) => {
|
|
3672
5114
|
const cursor = e.target.result;
|
|
@@ -3679,12 +5121,13 @@ class IndexedDbQueueStorage {
|
|
|
3679
5121
|
return;
|
|
3680
5122
|
}
|
|
3681
5123
|
const job = cursor.value;
|
|
3682
|
-
if (job.status !== "PENDING" /* PENDING */) {
|
|
5124
|
+
if (job.queue !== this.queueName || job.status !== "PENDING" /* PENDING */ || !this.matchesPrefixes(job)) {
|
|
3683
5125
|
cursor.continue();
|
|
3684
5126
|
return;
|
|
3685
5127
|
}
|
|
3686
5128
|
job.status = "PROCESSING" /* PROCESSING */;
|
|
3687
5129
|
job.last_ran_at = now;
|
|
5130
|
+
job.worker_id = workerId ?? null;
|
|
3688
5131
|
try {
|
|
3689
5132
|
const updateRequest = store.put(job);
|
|
3690
5133
|
updateRequest.onsuccess = () => {
|
|
@@ -3708,11 +5151,13 @@ class IndexedDbQueueStorage {
|
|
|
3708
5151
|
}
|
|
3709
5152
|
async size(status = "PENDING" /* PENDING */) {
|
|
3710
5153
|
const db = await this.getDb();
|
|
5154
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3711
5155
|
return new Promise((resolve, reject) => {
|
|
3712
5156
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3713
5157
|
const store = tx.objectStore(this.tableName);
|
|
3714
|
-
const index = store.index("
|
|
3715
|
-
const
|
|
5158
|
+
const index = store.index("queue_status");
|
|
5159
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
5160
|
+
const request = index.count(keyRange);
|
|
3716
5161
|
request.onsuccess = () => resolve(request.result);
|
|
3717
5162
|
request.onerror = () => reject(request.error);
|
|
3718
5163
|
tx.onerror = () => reject(tx.error);
|
|
@@ -3726,9 +5171,18 @@ class IndexedDbQueueStorage {
|
|
|
3726
5171
|
const getReq = store.get(job.id);
|
|
3727
5172
|
getReq.onsuccess = () => {
|
|
3728
5173
|
const existing = getReq.result;
|
|
3729
|
-
|
|
5174
|
+
if (!existing || existing.queue !== this.queueName || !this.matchesPrefixes(existing)) {
|
|
5175
|
+
reject(new Error(`Job ${job.id} not found or does not belong to queue ${this.queueName}`));
|
|
5176
|
+
return;
|
|
5177
|
+
}
|
|
5178
|
+
const currentAttempts = existing.run_attempts ?? 0;
|
|
3730
5179
|
job.run_attempts = currentAttempts + 1;
|
|
3731
|
-
|
|
5180
|
+
job.queue = this.queueName;
|
|
5181
|
+
const jobWithPrefixes = job;
|
|
5182
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5183
|
+
jobWithPrefixes[key] = value;
|
|
5184
|
+
}
|
|
5185
|
+
const putReq = store.put(jobWithPrefixes);
|
|
3732
5186
|
putReq.onsuccess = () => {};
|
|
3733
5187
|
putReq.onerror = () => reject(putReq.error);
|
|
3734
5188
|
};
|
|
@@ -3748,10 +5202,15 @@ class IndexedDbQueueStorage {
|
|
|
3748
5202
|
const db = await this.getDb();
|
|
3749
5203
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3750
5204
|
const store = tx.objectStore(this.tableName);
|
|
3751
|
-
const index = store.index("
|
|
3752
|
-
const
|
|
5205
|
+
const index = store.index("queue_job_run_id");
|
|
5206
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5207
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, job_run_id]);
|
|
5208
|
+
const request = index.getAll(keyRange);
|
|
3753
5209
|
return new Promise((resolve, reject) => {
|
|
3754
|
-
request.onsuccess = () =>
|
|
5210
|
+
request.onsuccess = () => {
|
|
5211
|
+
const results = (request.result || []).filter((job) => this.matchesPrefixes(job));
|
|
5212
|
+
resolve(results);
|
|
5213
|
+
};
|
|
3755
5214
|
request.onerror = () => reject(request.error);
|
|
3756
5215
|
tx.onerror = () => reject(tx.error);
|
|
3757
5216
|
});
|
|
@@ -3760,11 +5219,31 @@ class IndexedDbQueueStorage {
|
|
|
3760
5219
|
const db = await this.getDb();
|
|
3761
5220
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3762
5221
|
const store = tx.objectStore(this.tableName);
|
|
3763
|
-
const
|
|
5222
|
+
const index = store.index("queue_status");
|
|
5223
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3764
5224
|
return new Promise((resolve, reject) => {
|
|
3765
|
-
|
|
3766
|
-
request
|
|
5225
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, ""]);
|
|
5226
|
+
const request = index.openCursor(keyRange);
|
|
5227
|
+
request.onsuccess = (event) => {
|
|
5228
|
+
const cursor = event.target.result;
|
|
5229
|
+
if (cursor) {
|
|
5230
|
+
const job = cursor.value;
|
|
5231
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
5232
|
+
const deleteRequest = cursor.delete();
|
|
5233
|
+
deleteRequest.onsuccess = () => {
|
|
5234
|
+
cursor.continue();
|
|
5235
|
+
};
|
|
5236
|
+
deleteRequest.onerror = () => {
|
|
5237
|
+
cursor.continue();
|
|
5238
|
+
};
|
|
5239
|
+
} else {
|
|
5240
|
+
cursor.continue();
|
|
5241
|
+
}
|
|
5242
|
+
}
|
|
5243
|
+
};
|
|
5244
|
+
tx.oncomplete = () => resolve();
|
|
3767
5245
|
tx.onerror = () => reject(tx.error);
|
|
5246
|
+
request.onerror = () => reject(request.error);
|
|
3768
5247
|
});
|
|
3769
5248
|
}
|
|
3770
5249
|
async outputForInput(input) {
|
|
@@ -3772,10 +5251,23 @@ class IndexedDbQueueStorage {
|
|
|
3772
5251
|
const db = await this.getDb();
|
|
3773
5252
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3774
5253
|
const store = tx.objectStore(this.tableName);
|
|
3775
|
-
const index = store.index("
|
|
3776
|
-
const
|
|
5254
|
+
const index = store.index("queue_fingerprint_status");
|
|
5255
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5256
|
+
const request = index.get([
|
|
5257
|
+
...prefixKeyValues,
|
|
5258
|
+
this.queueName,
|
|
5259
|
+
fingerprint,
|
|
5260
|
+
"COMPLETED" /* COMPLETED */
|
|
5261
|
+
]);
|
|
3777
5262
|
return new Promise((resolve, reject) => {
|
|
3778
|
-
request.onsuccess = () =>
|
|
5263
|
+
request.onsuccess = () => {
|
|
5264
|
+
const job = request.result;
|
|
5265
|
+
if (job && this.matchesPrefixes(job)) {
|
|
5266
|
+
resolve(job.output ?? null);
|
|
5267
|
+
} else {
|
|
5268
|
+
resolve(null);
|
|
5269
|
+
}
|
|
5270
|
+
};
|
|
3779
5271
|
request.onerror = () => reject(request.error);
|
|
3780
5272
|
tx.onerror = () => reject(tx.error);
|
|
3781
5273
|
});
|
|
@@ -3790,6 +5282,9 @@ class IndexedDbQueueStorage {
|
|
|
3790
5282
|
await this.complete(job);
|
|
3791
5283
|
}
|
|
3792
5284
|
async delete(id) {
|
|
5285
|
+
const job = await this.get(id);
|
|
5286
|
+
if (!job)
|
|
5287
|
+
return;
|
|
3793
5288
|
const db = await this.getDb();
|
|
3794
5289
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3795
5290
|
const store = tx.objectStore(this.tableName);
|
|
@@ -3804,15 +5299,17 @@ class IndexedDbQueueStorage {
|
|
|
3804
5299
|
const db = await this.getDb();
|
|
3805
5300
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3806
5301
|
const store = tx.objectStore(this.tableName);
|
|
3807
|
-
const index = store.index("
|
|
5302
|
+
const index = store.index("queue_status");
|
|
3808
5303
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
5304
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5305
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
3809
5306
|
return new Promise((resolve, reject) => {
|
|
3810
|
-
const request = index.openCursor();
|
|
5307
|
+
const request = index.openCursor(keyRange);
|
|
3811
5308
|
request.onsuccess = (event) => {
|
|
3812
5309
|
const cursor = event.target.result;
|
|
3813
5310
|
if (cursor) {
|
|
3814
5311
|
const job = cursor.value;
|
|
3815
|
-
if (job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
|
|
5312
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
|
|
3816
5313
|
cursor.delete();
|
|
3817
5314
|
}
|
|
3818
5315
|
cursor.continue();
|
|
@@ -3823,6 +5320,363 @@ class IndexedDbQueueStorage {
|
|
|
3823
5320
|
request.onerror = () => reject(request.error);
|
|
3824
5321
|
});
|
|
3825
5322
|
}
|
|
5323
|
+
async getAllJobs() {
|
|
5324
|
+
const db = await this.getDb();
|
|
5325
|
+
const tx = db.transaction(this.tableName, "readonly");
|
|
5326
|
+
const store = tx.objectStore(this.tableName);
|
|
5327
|
+
const index = store.index("queue_status");
|
|
5328
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5329
|
+
return new Promise((resolve, reject) => {
|
|
5330
|
+
const jobs = [];
|
|
5331
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, ""]);
|
|
5332
|
+
const request = index.openCursor(keyRange);
|
|
5333
|
+
request.onsuccess = (event) => {
|
|
5334
|
+
const cursor = event.target.result;
|
|
5335
|
+
if (cursor) {
|
|
5336
|
+
const job = cursor.value;
|
|
5337
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
5338
|
+
jobs.push(job);
|
|
5339
|
+
}
|
|
5340
|
+
cursor.continue();
|
|
5341
|
+
}
|
|
5342
|
+
};
|
|
5343
|
+
tx.oncomplete = () => resolve(jobs);
|
|
5344
|
+
tx.onerror = () => reject(tx.error);
|
|
5345
|
+
request.onerror = () => reject(request.error);
|
|
5346
|
+
});
|
|
5347
|
+
}
|
|
5348
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
5349
|
+
const db = await this.getDb();
|
|
5350
|
+
const tx = db.transaction(this.tableName, "readonly");
|
|
5351
|
+
const store = tx.objectStore(this.tableName);
|
|
5352
|
+
return new Promise((resolve, reject) => {
|
|
5353
|
+
const jobs = [];
|
|
5354
|
+
const request = store.openCursor();
|
|
5355
|
+
request.onsuccess = (event) => {
|
|
5356
|
+
const cursor = event.target.result;
|
|
5357
|
+
if (cursor) {
|
|
5358
|
+
const job = cursor.value;
|
|
5359
|
+
if (job.queue !== this.queueName) {
|
|
5360
|
+
cursor.continue();
|
|
5361
|
+
return;
|
|
5362
|
+
}
|
|
5363
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
5364
|
+
jobs.push(job);
|
|
5365
|
+
} else {
|
|
5366
|
+
let matches = true;
|
|
5367
|
+
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
5368
|
+
if (job[key] !== value) {
|
|
5369
|
+
matches = false;
|
|
5370
|
+
break;
|
|
5371
|
+
}
|
|
5372
|
+
}
|
|
5373
|
+
if (matches) {
|
|
5374
|
+
jobs.push(job);
|
|
5375
|
+
}
|
|
5376
|
+
}
|
|
5377
|
+
cursor.continue();
|
|
5378
|
+
}
|
|
5379
|
+
};
|
|
5380
|
+
tx.oncomplete = () => resolve(jobs);
|
|
5381
|
+
tx.onerror = () => reject(tx.error);
|
|
5382
|
+
request.onerror = () => reject(request.error);
|
|
5383
|
+
});
|
|
5384
|
+
}
|
|
5385
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
5386
|
+
if (prefixFilter === undefined) {
|
|
5387
|
+
return false;
|
|
5388
|
+
}
|
|
5389
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
5390
|
+
return true;
|
|
5391
|
+
}
|
|
5392
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
5393
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
5394
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
5395
|
+
return true;
|
|
5396
|
+
}
|
|
5397
|
+
for (const key of instanceKeys) {
|
|
5398
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
5399
|
+
return true;
|
|
5400
|
+
}
|
|
5401
|
+
}
|
|
5402
|
+
return false;
|
|
5403
|
+
}
|
|
5404
|
+
getPollingManager() {
|
|
5405
|
+
if (!this.pollingManager) {
|
|
5406
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
5407
|
+
const jobs = await this.getAllJobs();
|
|
5408
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
5409
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
5410
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
5411
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
5412
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
5413
|
+
});
|
|
5414
|
+
}
|
|
5415
|
+
return this.pollingManager;
|
|
5416
|
+
}
|
|
5417
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
5418
|
+
let lastKnownJobs = new Map;
|
|
5419
|
+
let cancelled = false;
|
|
5420
|
+
const poll = async () => {
|
|
5421
|
+
if (cancelled)
|
|
5422
|
+
return;
|
|
5423
|
+
try {
|
|
5424
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
5425
|
+
if (cancelled)
|
|
5426
|
+
return;
|
|
5427
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
5428
|
+
for (const [id, job] of currentMap) {
|
|
5429
|
+
const old = lastKnownJobs.get(id);
|
|
5430
|
+
if (!old) {
|
|
5431
|
+
callback({ type: "INSERT", new: job });
|
|
5432
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
5433
|
+
callback({ type: "UPDATE", old, new: job });
|
|
5434
|
+
}
|
|
5435
|
+
}
|
|
5436
|
+
for (const [id, job] of lastKnownJobs) {
|
|
5437
|
+
if (!currentMap.has(id)) {
|
|
5438
|
+
callback({ type: "DELETE", old: job });
|
|
5439
|
+
}
|
|
5440
|
+
}
|
|
5441
|
+
lastKnownJobs = currentMap;
|
|
5442
|
+
} catch {}
|
|
5443
|
+
};
|
|
5444
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
5445
|
+
poll();
|
|
5446
|
+
return () => {
|
|
5447
|
+
cancelled = true;
|
|
5448
|
+
clearInterval(intervalId);
|
|
5449
|
+
};
|
|
5450
|
+
}
|
|
5451
|
+
subscribeToChanges(callback, options) {
|
|
5452
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
5453
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
5454
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
5455
|
+
}
|
|
5456
|
+
const manager = this.getPollingManager();
|
|
5457
|
+
return manager.subscribe(callback, { intervalMs });
|
|
5458
|
+
}
|
|
5459
|
+
}
|
|
5460
|
+
// src/limiter/IndexedDbRateLimiterStorage.ts
|
|
5461
|
+
import { createServiceToken as createServiceToken28 } from "@workglow/util";
|
|
5462
|
+
var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken28("ratelimiter.storage.indexedDb");
|
|
5463
|
+
|
|
5464
|
+
class IndexedDbRateLimiterStorage {
|
|
5465
|
+
executionDb;
|
|
5466
|
+
nextAvailableDb;
|
|
5467
|
+
executionTableName;
|
|
5468
|
+
nextAvailableTableName;
|
|
5469
|
+
migrationOptions;
|
|
5470
|
+
prefixes;
|
|
5471
|
+
prefixValues;
|
|
5472
|
+
constructor(options = {}) {
|
|
5473
|
+
this.migrationOptions = options;
|
|
5474
|
+
this.prefixes = options.prefixes ?? [];
|
|
5475
|
+
this.prefixValues = options.prefixValues ?? {};
|
|
5476
|
+
if (this.prefixes.length > 0) {
|
|
5477
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
5478
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
5479
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
5480
|
+
} else {
|
|
5481
|
+
this.executionTableName = "rate_limit_executions";
|
|
5482
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
5483
|
+
}
|
|
5484
|
+
}
|
|
5485
|
+
getPrefixColumnNames() {
|
|
5486
|
+
return this.prefixes.map((p) => p.name);
|
|
5487
|
+
}
|
|
5488
|
+
matchesPrefixes(record) {
|
|
5489
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5490
|
+
if (record[key] !== value) {
|
|
5491
|
+
return false;
|
|
5492
|
+
}
|
|
5493
|
+
}
|
|
5494
|
+
return true;
|
|
5495
|
+
}
|
|
5496
|
+
getPrefixKeyValues() {
|
|
5497
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
5498
|
+
}
|
|
5499
|
+
async getExecutionDb() {
|
|
5500
|
+
if (this.executionDb)
|
|
5501
|
+
return this.executionDb;
|
|
5502
|
+
await this.setupDatabase();
|
|
5503
|
+
return this.executionDb;
|
|
5504
|
+
}
|
|
5505
|
+
async getNextAvailableDb() {
|
|
5506
|
+
if (this.nextAvailableDb)
|
|
5507
|
+
return this.nextAvailableDb;
|
|
5508
|
+
await this.setupDatabase();
|
|
5509
|
+
return this.nextAvailableDb;
|
|
5510
|
+
}
|
|
5511
|
+
async setupDatabase() {
|
|
5512
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
5513
|
+
const buildKeyPath = (basePath) => {
|
|
5514
|
+
return [...prefixColumnNames, ...basePath];
|
|
5515
|
+
};
|
|
5516
|
+
const executionIndexes = [
|
|
5517
|
+
{
|
|
5518
|
+
name: "queue_executed_at",
|
|
5519
|
+
keyPath: buildKeyPath(["queue_name", "executed_at"]),
|
|
5520
|
+
options: { unique: false }
|
|
5521
|
+
}
|
|
5522
|
+
];
|
|
5523
|
+
this.executionDb = await ensureIndexedDbTable(this.executionTableName, "id", executionIndexes, this.migrationOptions);
|
|
5524
|
+
const nextAvailableIndexes = [
|
|
5525
|
+
{
|
|
5526
|
+
name: "queue_name",
|
|
5527
|
+
keyPath: buildKeyPath(["queue_name"]),
|
|
5528
|
+
options: { unique: true }
|
|
5529
|
+
}
|
|
5530
|
+
];
|
|
5531
|
+
this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
|
|
5532
|
+
}
|
|
5533
|
+
async recordExecution(queueName) {
|
|
5534
|
+
const db = await this.getExecutionDb();
|
|
5535
|
+
const tx = db.transaction(this.executionTableName, "readwrite");
|
|
5536
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5537
|
+
const record = {
|
|
5538
|
+
id: crypto.randomUUID(),
|
|
5539
|
+
queue_name: queueName,
|
|
5540
|
+
executed_at: new Date().toISOString()
|
|
5541
|
+
};
|
|
5542
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5543
|
+
record[key] = value;
|
|
5544
|
+
}
|
|
5545
|
+
return new Promise((resolve, reject) => {
|
|
5546
|
+
const request = store.add(record);
|
|
5547
|
+
tx.oncomplete = () => resolve();
|
|
5548
|
+
tx.onerror = () => reject(tx.error);
|
|
5549
|
+
request.onerror = () => reject(request.error);
|
|
5550
|
+
});
|
|
5551
|
+
}
|
|
5552
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
5553
|
+
const db = await this.getExecutionDb();
|
|
5554
|
+
const tx = db.transaction(this.executionTableName, "readonly");
|
|
5555
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5556
|
+
const index = store.index("queue_executed_at");
|
|
5557
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5558
|
+
return new Promise((resolve, reject) => {
|
|
5559
|
+
let count = 0;
|
|
5560
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartTime], [...prefixKeyValues, queueName, ""], true, false);
|
|
5561
|
+
const request = index.openCursor(keyRange);
|
|
5562
|
+
request.onsuccess = (event) => {
|
|
5563
|
+
const cursor = event.target.result;
|
|
5564
|
+
if (cursor) {
|
|
5565
|
+
const record = cursor.value;
|
|
5566
|
+
if (this.matchesPrefixes(record)) {
|
|
5567
|
+
count++;
|
|
5568
|
+
}
|
|
5569
|
+
cursor.continue();
|
|
5570
|
+
}
|
|
5571
|
+
};
|
|
5572
|
+
tx.oncomplete = () => resolve(count);
|
|
5573
|
+
tx.onerror = () => reject(tx.error);
|
|
5574
|
+
request.onerror = () => reject(request.error);
|
|
5575
|
+
});
|
|
5576
|
+
}
|
|
5577
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
5578
|
+
const db = await this.getExecutionDb();
|
|
5579
|
+
const tx = db.transaction(this.executionTableName, "readonly");
|
|
5580
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5581
|
+
const index = store.index("queue_executed_at");
|
|
5582
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5583
|
+
return new Promise((resolve, reject) => {
|
|
5584
|
+
const executions = [];
|
|
5585
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, ""]);
|
|
5586
|
+
const request = index.openCursor(keyRange);
|
|
5587
|
+
request.onsuccess = (event) => {
|
|
5588
|
+
const cursor = event.target.result;
|
|
5589
|
+
if (cursor) {
|
|
5590
|
+
const record = cursor.value;
|
|
5591
|
+
if (this.matchesPrefixes(record)) {
|
|
5592
|
+
executions.push(record.executed_at);
|
|
5593
|
+
}
|
|
5594
|
+
cursor.continue();
|
|
5595
|
+
}
|
|
5596
|
+
};
|
|
5597
|
+
tx.oncomplete = () => {
|
|
5598
|
+
executions.sort();
|
|
5599
|
+
resolve(executions[offset]);
|
|
5600
|
+
};
|
|
5601
|
+
tx.onerror = () => reject(tx.error);
|
|
5602
|
+
request.onerror = () => reject(request.error);
|
|
5603
|
+
});
|
|
5604
|
+
}
|
|
5605
|
+
async getNextAvailableTime(queueName) {
|
|
5606
|
+
const db = await this.getNextAvailableDb();
|
|
5607
|
+
const tx = db.transaction(this.nextAvailableTableName, "readonly");
|
|
5608
|
+
const store = tx.objectStore(this.nextAvailableTableName);
|
|
5609
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5610
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5611
|
+
return new Promise((resolve, reject) => {
|
|
5612
|
+
const request = store.get(key);
|
|
5613
|
+
request.onsuccess = () => {
|
|
5614
|
+
const record = request.result;
|
|
5615
|
+
if (record && this.matchesPrefixes(record)) {
|
|
5616
|
+
resolve(record.next_available_at);
|
|
5617
|
+
} else {
|
|
5618
|
+
resolve(undefined);
|
|
5619
|
+
}
|
|
5620
|
+
};
|
|
5621
|
+
request.onerror = () => reject(request.error);
|
|
5622
|
+
tx.onerror = () => reject(tx.error);
|
|
5623
|
+
});
|
|
5624
|
+
}
|
|
5625
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
5626
|
+
const db = await this.getNextAvailableDb();
|
|
5627
|
+
const tx = db.transaction(this.nextAvailableTableName, "readwrite");
|
|
5628
|
+
const store = tx.objectStore(this.nextAvailableTableName);
|
|
5629
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5630
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5631
|
+
const record = {
|
|
5632
|
+
queue_name: queueName,
|
|
5633
|
+
next_available_at: nextAvailableAt
|
|
5634
|
+
};
|
|
5635
|
+
for (const [k, value] of Object.entries(this.prefixValues)) {
|
|
5636
|
+
record[k] = value;
|
|
5637
|
+
}
|
|
5638
|
+
record[this.getPrefixColumnNames().concat(["queue_name"]).join("_")] = key;
|
|
5639
|
+
return new Promise((resolve, reject) => {
|
|
5640
|
+
const request = store.put(record);
|
|
5641
|
+
tx.oncomplete = () => resolve();
|
|
5642
|
+
tx.onerror = () => reject(tx.error);
|
|
5643
|
+
request.onerror = () => reject(request.error);
|
|
5644
|
+
});
|
|
5645
|
+
}
|
|
5646
|
+
async clear(queueName) {
|
|
5647
|
+
const execDb = await this.getExecutionDb();
|
|
5648
|
+
const execTx = execDb.transaction(this.executionTableName, "readwrite");
|
|
5649
|
+
const execStore = execTx.objectStore(this.executionTableName);
|
|
5650
|
+
const execIndex = execStore.index("queue_executed_at");
|
|
5651
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5652
|
+
await new Promise((resolve, reject) => {
|
|
5653
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, ""]);
|
|
5654
|
+
const request = execIndex.openCursor(keyRange);
|
|
5655
|
+
request.onsuccess = (event) => {
|
|
5656
|
+
const cursor = event.target.result;
|
|
5657
|
+
if (cursor) {
|
|
5658
|
+
const record = cursor.value;
|
|
5659
|
+
if (this.matchesPrefixes(record)) {
|
|
5660
|
+
cursor.delete();
|
|
5661
|
+
}
|
|
5662
|
+
cursor.continue();
|
|
5663
|
+
}
|
|
5664
|
+
};
|
|
5665
|
+
execTx.oncomplete = () => resolve();
|
|
5666
|
+
execTx.onerror = () => reject(execTx.error);
|
|
5667
|
+
request.onerror = () => reject(request.error);
|
|
5668
|
+
});
|
|
5669
|
+
const nextDb = await this.getNextAvailableDb();
|
|
5670
|
+
const nextTx = nextDb.transaction(this.nextAvailableTableName, "readwrite");
|
|
5671
|
+
const nextStore = nextTx.objectStore(this.nextAvailableTableName);
|
|
5672
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5673
|
+
await new Promise((resolve, reject) => {
|
|
5674
|
+
const request = nextStore.delete(key);
|
|
5675
|
+
nextTx.oncomplete = () => resolve();
|
|
5676
|
+
nextTx.onerror = () => reject(nextTx.error);
|
|
5677
|
+
request.onerror = () => reject(request.error);
|
|
5678
|
+
});
|
|
5679
|
+
}
|
|
3826
5680
|
}
|
|
3827
5681
|
export {
|
|
3828
5682
|
ensureIndexedDbTable,
|
|
@@ -3830,22 +5684,29 @@ export {
|
|
|
3830
5684
|
TabularRepository,
|
|
3831
5685
|
TABULAR_REPOSITORY,
|
|
3832
5686
|
SupabaseTabularRepository,
|
|
5687
|
+
SupabaseRateLimiterStorage,
|
|
3833
5688
|
SupabaseQueueStorage,
|
|
3834
5689
|
SupabaseKvRepository,
|
|
3835
5690
|
SqliteTabularRepository,
|
|
5691
|
+
SqliteRateLimiterStorage,
|
|
3836
5692
|
SqliteQueueStorage,
|
|
3837
5693
|
SqliteKvRepository,
|
|
3838
5694
|
SUPABASE_TABULAR_REPOSITORY,
|
|
5695
|
+
SUPABASE_RATE_LIMITER_STORAGE,
|
|
3839
5696
|
SUPABASE_QUEUE_STORAGE,
|
|
3840
5697
|
SUPABASE_KV_REPOSITORY,
|
|
3841
5698
|
SQLITE_TABULAR_REPOSITORY,
|
|
5699
|
+
SQLITE_RATE_LIMITER_STORAGE,
|
|
3842
5700
|
SQLITE_QUEUE_STORAGE,
|
|
3843
5701
|
SQLITE_KV_REPOSITORY,
|
|
5702
|
+
RATE_LIMITER_STORAGE,
|
|
3844
5703
|
QUEUE_STORAGE,
|
|
3845
5704
|
PostgresTabularRepository,
|
|
5705
|
+
PostgresRateLimiterStorage,
|
|
3846
5706
|
PostgresQueueStorage,
|
|
3847
5707
|
PostgresKvRepository,
|
|
3848
5708
|
POSTGRES_TABULAR_REPOSITORY,
|
|
5709
|
+
POSTGRES_RATE_LIMITER_STORAGE,
|
|
3849
5710
|
POSTGRES_QUEUE_STORAGE,
|
|
3850
5711
|
POSTGRES_KV_REPOSITORY,
|
|
3851
5712
|
MEMORY_TABULAR_REPOSITORY,
|
|
@@ -3855,12 +5716,16 @@ export {
|
|
|
3855
5716
|
KV_REPOSITORY,
|
|
3856
5717
|
JobStatus,
|
|
3857
5718
|
IndexedDbTabularRepository,
|
|
5719
|
+
IndexedDbRateLimiterStorage,
|
|
3858
5720
|
IndexedDbQueueStorage,
|
|
3859
5721
|
IndexedDbKvRepository,
|
|
3860
5722
|
InMemoryTabularRepository,
|
|
5723
|
+
InMemoryRateLimiterStorage,
|
|
3861
5724
|
InMemoryQueueStorage,
|
|
3862
5725
|
InMemoryKvRepository,
|
|
5726
|
+
IN_MEMORY_RATE_LIMITER_STORAGE,
|
|
3863
5727
|
IN_MEMORY_QUEUE_STORAGE,
|
|
5728
|
+
INDEXED_DB_RATE_LIMITER_STORAGE,
|
|
3864
5729
|
INDEXED_DB_QUEUE_STORAGE,
|
|
3865
5730
|
IDB_TABULAR_REPOSITORY,
|
|
3866
5731
|
IDB_KV_REPOSITORY,
|
|
@@ -3876,4 +5741,4 @@ export {
|
|
|
3876
5741
|
CACHED_TABULAR_REPOSITORY
|
|
3877
5742
|
};
|
|
3878
5743
|
|
|
3879
|
-
//# debugId=
|
|
5744
|
+
//# debugId=252A6E832532D65264756E2164756E21
|