@workglow/storage 0.0.57 → 0.0.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +1266 -121
- package/dist/browser.js.map +15 -10
- package/dist/bun.js +2132 -267
- package/dist/bun.js.map +20 -13
- package/dist/common-server.d.ts +4 -0
- package/dist/common-server.d.ts.map +1 -1
- package/dist/common.d.ts +2 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/limiter/IRateLimiterStorage.d.ts +81 -0
- package/dist/limiter/IRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/InMemoryRateLimiterStorage.d.ts +32 -0
- package/dist/limiter/InMemoryRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/IndexedDbRateLimiterStorage.d.ts +52 -0
- package/dist/limiter/IndexedDbRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/PostgresRateLimiterStorage.d.ts +54 -0
- package/dist/limiter/PostgresRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/SqliteRateLimiterStorage.d.ts +53 -0
- package/dist/limiter/SqliteRateLimiterStorage.d.ts.map +1 -0
- package/dist/limiter/SupabaseRateLimiterStorage.d.ts +53 -0
- package/dist/limiter/SupabaseRateLimiterStorage.d.ts.map +1 -0
- package/dist/node.js +2132 -267
- package/dist/node.js.map +20 -13
- package/dist/queue/IQueueStorage.d.ts +72 -1
- package/dist/queue/IQueueStorage.d.ts.map +1 -1
- package/dist/queue/InMemoryQueueStorage.d.ts +44 -11
- package/dist/queue/InMemoryQueueStorage.d.ts.map +1 -1
- package/dist/queue/IndexedDbQueueStorage.d.ts +70 -5
- package/dist/queue/IndexedDbQueueStorage.d.ts.map +1 -1
- package/dist/queue/PostgresQueueStorage.d.ts +80 -8
- package/dist/queue/PostgresQueueStorage.d.ts.map +1 -1
- package/dist/queue/SqliteQueueStorage.d.ts +90 -34
- package/dist/queue/SqliteQueueStorage.d.ts.map +1 -1
- package/dist/queue/SupabaseQueueStorage.d.ts +98 -4
- package/dist/queue/SupabaseQueueStorage.d.ts.map +1 -1
- package/dist/tabular/ITabularRepository.d.ts +18 -0
- package/dist/tabular/ITabularRepository.d.ts.map +1 -1
- package/dist/tabular/InMemoryTabularRepository.d.ts +9 -1
- package/dist/tabular/InMemoryTabularRepository.d.ts.map +1 -1
- package/dist/tabular/SqliteTabularRepository.d.ts.map +1 -1
- package/dist/tabular/SupabaseTabularRepository.d.ts +21 -1
- package/dist/tabular/SupabaseTabularRepository.d.ts.map +1 -1
- package/dist/tabular/TabularRepository.d.ts +10 -1
- package/dist/tabular/TabularRepository.d.ts.map +1 -1
- package/dist/util/PollingSubscriptionManager.d.ts +112 -0
- package/dist/util/PollingSubscriptionManager.d.ts.map +1 -0
- package/package.json +5 -5
package/dist/bun.js
CHANGED
|
@@ -107,6 +107,9 @@ class TabularRepository {
|
|
|
107
107
|
waitOn(name) {
|
|
108
108
|
return this.events.waitOn(name);
|
|
109
109
|
}
|
|
110
|
+
subscribeToChanges(_callback) {
|
|
111
|
+
throw new Error(`subscribeToChanges is not implemented for ${this.constructor.name}. ` + `Use InMemoryTabularRepository or SupabaseTabularRepository for subscription support.`);
|
|
112
|
+
}
|
|
110
113
|
primaryKeyColumns() {
|
|
111
114
|
const columns = [];
|
|
112
115
|
for (const key of Object.keys(this.primaryKeySchema.properties)) {
|
|
@@ -274,12 +277,30 @@ class InMemoryTabularRepository extends TabularRepository {
|
|
|
274
277
|
return false;
|
|
275
278
|
}
|
|
276
279
|
});
|
|
277
|
-
for (const [id,
|
|
280
|
+
for (const [id, entity] of entriesToDelete) {
|
|
278
281
|
this.values.delete(id);
|
|
282
|
+
const { key } = this.separateKeyValueFromCombined(entity);
|
|
283
|
+
this.events.emit("delete", key);
|
|
279
284
|
}
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
285
|
+
}
|
|
286
|
+
subscribeToChanges(callback) {
|
|
287
|
+
const handlePut = (entity) => {
|
|
288
|
+
callback({ type: "UPDATE", new: entity });
|
|
289
|
+
};
|
|
290
|
+
const handleDelete = (_key) => {
|
|
291
|
+
callback({ type: "DELETE" });
|
|
292
|
+
};
|
|
293
|
+
const handleClearAll = () => {
|
|
294
|
+
callback({ type: "DELETE" });
|
|
295
|
+
};
|
|
296
|
+
this.events.on("put", handlePut);
|
|
297
|
+
this.events.on("delete", handleDelete);
|
|
298
|
+
this.events.on("clearall", handleClearAll);
|
|
299
|
+
return () => {
|
|
300
|
+
this.events.off("put", handlePut);
|
|
301
|
+
this.events.off("delete", handleDelete);
|
|
302
|
+
this.events.off("clearall", handleClearAll);
|
|
303
|
+
};
|
|
283
304
|
}
|
|
284
305
|
destroy() {
|
|
285
306
|
this.values.clear();
|
|
@@ -541,7 +562,7 @@ class InMemoryKvRepository extends KvViaTabularRepository {
|
|
|
541
562
|
}
|
|
542
563
|
}
|
|
543
564
|
// src/queue/InMemoryQueueStorage.ts
|
|
544
|
-
import { createServiceToken as createServiceToken7, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
|
|
565
|
+
import { createServiceToken as createServiceToken7, EventEmitter as EventEmitter3, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
|
|
545
566
|
|
|
546
567
|
// src/queue/IQueueStorage.ts
|
|
547
568
|
import { createServiceToken as createServiceToken6 } from "@workglow/util";
|
|
@@ -561,63 +582,88 @@ var IN_MEMORY_QUEUE_STORAGE = createServiceToken7("jobqueue.storage.inMemory");
|
|
|
561
582
|
|
|
562
583
|
class InMemoryQueueStorage {
|
|
563
584
|
queueName;
|
|
564
|
-
|
|
585
|
+
prefixValues;
|
|
586
|
+
events = new EventEmitter3;
|
|
587
|
+
constructor(queueName, options) {
|
|
565
588
|
this.queueName = queueName;
|
|
566
589
|
this.jobQueue = [];
|
|
590
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
567
591
|
}
|
|
568
592
|
jobQueue;
|
|
593
|
+
matchesPrefixes(job) {
|
|
594
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
595
|
+
if (job[key] !== value) {
|
|
596
|
+
return false;
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
return true;
|
|
600
|
+
}
|
|
569
601
|
pendingQueue() {
|
|
570
602
|
const now = new Date().toISOString();
|
|
571
|
-
return this.jobQueue.filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
603
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
|
|
572
604
|
}
|
|
573
605
|
async add(job) {
|
|
574
606
|
await sleep(0);
|
|
575
607
|
const now = new Date().toISOString();
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
608
|
+
const jobWithPrefixes = job;
|
|
609
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid4();
|
|
610
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid4();
|
|
611
|
+
jobWithPrefixes.queue = this.queueName;
|
|
612
|
+
jobWithPrefixes.fingerprint = await makeFingerprint4(jobWithPrefixes.input);
|
|
613
|
+
jobWithPrefixes.status = "PENDING" /* PENDING */;
|
|
614
|
+
jobWithPrefixes.progress = 0;
|
|
615
|
+
jobWithPrefixes.progress_message = "";
|
|
616
|
+
jobWithPrefixes.progress_details = null;
|
|
617
|
+
jobWithPrefixes.created_at = now;
|
|
618
|
+
jobWithPrefixes.run_after = now;
|
|
619
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
620
|
+
jobWithPrefixes[key] = value;
|
|
621
|
+
}
|
|
622
|
+
this.jobQueue.push(jobWithPrefixes);
|
|
623
|
+
this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
|
|
624
|
+
return jobWithPrefixes.id;
|
|
588
625
|
}
|
|
589
626
|
async get(id) {
|
|
590
627
|
await sleep(0);
|
|
591
|
-
|
|
628
|
+
const job = this.jobQueue.find((j) => j.id === id);
|
|
629
|
+
if (job && this.matchesPrefixes(job)) {
|
|
630
|
+
return job;
|
|
631
|
+
}
|
|
632
|
+
return;
|
|
592
633
|
}
|
|
593
634
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
594
635
|
await sleep(0);
|
|
595
636
|
num = Number(num) || 100;
|
|
596
|
-
return this.jobQueue.sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
637
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
597
638
|
}
|
|
598
|
-
async next() {
|
|
639
|
+
async next(workerId) {
|
|
599
640
|
await sleep(0);
|
|
600
641
|
const top = this.pendingQueue();
|
|
601
642
|
const job = top[0];
|
|
602
643
|
if (job) {
|
|
644
|
+
const oldJob = { ...job };
|
|
603
645
|
job.status = "PROCESSING" /* PROCESSING */;
|
|
604
646
|
job.last_ran_at = new Date().toISOString();
|
|
647
|
+
job.worker_id = workerId ?? null;
|
|
648
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
605
649
|
return job;
|
|
606
650
|
}
|
|
607
651
|
}
|
|
608
652
|
async size(status = "PENDING" /* PENDING */) {
|
|
609
653
|
await sleep(0);
|
|
610
|
-
return this.jobQueue.filter((j) => j.status === status).length;
|
|
654
|
+
return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
|
|
611
655
|
}
|
|
612
656
|
async saveProgress(id, progress, message, details) {
|
|
613
657
|
await sleep(0);
|
|
614
|
-
const job = this.jobQueue.find((j) => j.id === id);
|
|
658
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
615
659
|
if (!job) {
|
|
616
660
|
throw new Error(`Job ${id} not found`);
|
|
617
661
|
}
|
|
662
|
+
const oldJob = { ...job };
|
|
618
663
|
job.progress = progress;
|
|
619
664
|
job.progress_message = message;
|
|
620
665
|
job.progress_details = details;
|
|
666
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
621
667
|
}
|
|
622
668
|
async complete(job) {
|
|
623
669
|
await sleep(0);
|
|
@@ -627,44 +673,149 @@ class InMemoryQueueStorage {
|
|
|
627
673
|
const currentAttempts = existing?.run_attempts ?? 0;
|
|
628
674
|
job.run_attempts = currentAttempts + 1;
|
|
629
675
|
this.jobQueue[index] = job;
|
|
676
|
+
this.events.emit("change", { type: "UPDATE", old: existing, new: job });
|
|
630
677
|
}
|
|
631
678
|
}
|
|
632
679
|
async abort(id) {
|
|
633
680
|
await sleep(0);
|
|
634
|
-
const job = this.jobQueue.find((j) => j.id === id);
|
|
681
|
+
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
635
682
|
if (job) {
|
|
683
|
+
const oldJob = { ...job };
|
|
636
684
|
job.status = "ABORTING" /* ABORTING */;
|
|
685
|
+
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
637
686
|
}
|
|
638
687
|
}
|
|
639
688
|
async getByRunId(runId) {
|
|
640
689
|
await sleep(0);
|
|
641
|
-
return this.jobQueue.filter((job) => job.job_run_id === runId);
|
|
690
|
+
return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
|
|
642
691
|
}
|
|
643
692
|
async deleteAll() {
|
|
644
693
|
await sleep(0);
|
|
645
|
-
this.jobQueue
|
|
694
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
|
|
695
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
|
|
696
|
+
for (const job of deletedJobs) {
|
|
697
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
698
|
+
}
|
|
646
699
|
}
|
|
647
700
|
async outputForInput(input) {
|
|
648
701
|
await sleep(0);
|
|
649
702
|
const fingerprint = await makeFingerprint4(input);
|
|
650
|
-
return this.jobQueue.find((j) => j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
|
|
703
|
+
return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
|
|
651
704
|
}
|
|
652
705
|
async delete(id) {
|
|
653
706
|
await sleep(0);
|
|
654
|
-
|
|
707
|
+
const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
|
|
708
|
+
this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
|
|
709
|
+
if (deletedJob) {
|
|
710
|
+
this.events.emit("change", { type: "DELETE", old: deletedJob });
|
|
711
|
+
}
|
|
655
712
|
}
|
|
656
713
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
657
714
|
await sleep(0);
|
|
658
715
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
659
|
-
|
|
716
|
+
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
|
|
717
|
+
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
|
|
718
|
+
for (const job of deletedJobs) {
|
|
719
|
+
this.events.emit("change", { type: "DELETE", old: job });
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
async setupDatabase() {}
|
|
723
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
724
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
725
|
+
return true;
|
|
726
|
+
}
|
|
727
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
728
|
+
if (Object.keys(filterValues).length === 0) {
|
|
729
|
+
return true;
|
|
730
|
+
}
|
|
731
|
+
const jobWithPrefixes = job;
|
|
732
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
733
|
+
if (jobWithPrefixes[key] !== value) {
|
|
734
|
+
return false;
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
return true;
|
|
738
|
+
}
|
|
739
|
+
subscribeToChanges(callback, options) {
|
|
740
|
+
const prefixFilter = options?.prefixFilter;
|
|
741
|
+
const filteredCallback = (change) => {
|
|
742
|
+
const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
|
|
743
|
+
const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
|
|
744
|
+
if (!newMatches && !oldMatches) {
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
747
|
+
callback(change);
|
|
748
|
+
};
|
|
749
|
+
return this.events.subscribe("change", filteredCallback);
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
// src/limiter/IRateLimiterStorage.ts
|
|
753
|
+
import { createServiceToken as createServiceToken8 } from "@workglow/util";
|
|
754
|
+
var RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage");
|
|
755
|
+
// src/limiter/InMemoryRateLimiterStorage.ts
|
|
756
|
+
import { createServiceToken as createServiceToken9, sleep as sleep2 } from "@workglow/util";
|
|
757
|
+
var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken9("ratelimiter.storage.inMemory");
|
|
758
|
+
|
|
759
|
+
class InMemoryRateLimiterStorage {
|
|
760
|
+
prefixValues;
|
|
761
|
+
executions = new Map;
|
|
762
|
+
nextAvailableTimes = new Map;
|
|
763
|
+
constructor(options) {
|
|
764
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
765
|
+
}
|
|
766
|
+
makeKey(queueName) {
|
|
767
|
+
const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
|
|
768
|
+
return prefixPart ? `${prefixPart}|${queueName}` : queueName;
|
|
660
769
|
}
|
|
661
770
|
async setupDatabase() {}
|
|
771
|
+
async recordExecution(queueName) {
|
|
772
|
+
await sleep2(0);
|
|
773
|
+
const key = this.makeKey(queueName);
|
|
774
|
+
const executions = this.executions.get(key) ?? [];
|
|
775
|
+
executions.push({
|
|
776
|
+
queueName,
|
|
777
|
+
executedAt: new Date
|
|
778
|
+
});
|
|
779
|
+
this.executions.set(key, executions);
|
|
780
|
+
}
|
|
781
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
782
|
+
await sleep2(0);
|
|
783
|
+
const key = this.makeKey(queueName);
|
|
784
|
+
const executions = this.executions.get(key) ?? [];
|
|
785
|
+
const windowStart = new Date(windowStartTime);
|
|
786
|
+
return executions.filter((e) => e.executedAt > windowStart).length;
|
|
787
|
+
}
|
|
788
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
789
|
+
await sleep2(0);
|
|
790
|
+
const key = this.makeKey(queueName);
|
|
791
|
+
const executions = this.executions.get(key) ?? [];
|
|
792
|
+
const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
|
|
793
|
+
const execution = sorted[offset];
|
|
794
|
+
return execution?.executedAt.toISOString();
|
|
795
|
+
}
|
|
796
|
+
async getNextAvailableTime(queueName) {
|
|
797
|
+
await sleep2(0);
|
|
798
|
+
const key = this.makeKey(queueName);
|
|
799
|
+
const time = this.nextAvailableTimes.get(key);
|
|
800
|
+
return time?.toISOString();
|
|
801
|
+
}
|
|
802
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
803
|
+
await sleep2(0);
|
|
804
|
+
const key = this.makeKey(queueName);
|
|
805
|
+
this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
|
|
806
|
+
}
|
|
807
|
+
async clear(queueName) {
|
|
808
|
+
await sleep2(0);
|
|
809
|
+
const key = this.makeKey(queueName);
|
|
810
|
+
this.executions.delete(key);
|
|
811
|
+
this.nextAvailableTimes.delete(key);
|
|
812
|
+
}
|
|
662
813
|
}
|
|
663
814
|
// src/tabular/FsFolderTabularRepository.ts
|
|
664
|
-
import { createServiceToken as
|
|
815
|
+
import { createServiceToken as createServiceToken10, sleep as sleep3 } from "@workglow/util";
|
|
665
816
|
import { mkdir, readdir, readFile, rm, writeFile } from "fs/promises";
|
|
666
817
|
import path from "path";
|
|
667
|
-
var FS_FOLDER_TABULAR_REPOSITORY =
|
|
818
|
+
var FS_FOLDER_TABULAR_REPOSITORY = createServiceToken10("storage.tabularRepository.fsFolder");
|
|
668
819
|
|
|
669
820
|
class FsFolderTabularRepository extends TabularRepository {
|
|
670
821
|
folderPath;
|
|
@@ -689,7 +840,7 @@ class FsFolderTabularRepository extends TabularRepository {
|
|
|
689
840
|
await writeFile(filePath, JSON.stringify(entity));
|
|
690
841
|
} catch (error) {
|
|
691
842
|
try {
|
|
692
|
-
await
|
|
843
|
+
await sleep3(1);
|
|
693
844
|
await writeFile(filePath, JSON.stringify(entity));
|
|
694
845
|
} catch (error2) {
|
|
695
846
|
console.error("Error writing file", filePath, error2);
|
|
@@ -773,7 +924,7 @@ class FsFolderTabularRepository extends TabularRepository {
|
|
|
773
924
|
}
|
|
774
925
|
}
|
|
775
926
|
// src/tabular/PostgresTabularRepository.ts
|
|
776
|
-
import { createServiceToken as
|
|
927
|
+
import { createServiceToken as createServiceToken11 } from "@workglow/util";
|
|
777
928
|
|
|
778
929
|
// src/tabular/BaseSqlTabularRepository.ts
|
|
779
930
|
class BaseSqlTabularRepository extends TabularRepository {
|
|
@@ -959,7 +1110,7 @@ class BaseSqlTabularRepository extends TabularRepository {
|
|
|
959
1110
|
}
|
|
960
1111
|
|
|
961
1112
|
// src/tabular/PostgresTabularRepository.ts
|
|
962
|
-
var POSTGRES_TABULAR_REPOSITORY =
|
|
1113
|
+
var POSTGRES_TABULAR_REPOSITORY = createServiceToken11("storage.tabularRepository.postgres");
|
|
963
1114
|
|
|
964
1115
|
class PostgresTabularRepository extends BaseSqlTabularRepository {
|
|
965
1116
|
db;
|
|
@@ -1317,8 +1468,8 @@ class PostgresTabularRepository extends BaseSqlTabularRepository {
|
|
|
1317
1468
|
}
|
|
1318
1469
|
// src/tabular/SqliteTabularRepository.ts
|
|
1319
1470
|
import { Sqlite } from "@workglow/sqlite";
|
|
1320
|
-
import { createServiceToken as
|
|
1321
|
-
var SQLITE_TABULAR_REPOSITORY =
|
|
1471
|
+
import { createServiceToken as createServiceToken12 } from "@workglow/util";
|
|
1472
|
+
var SQLITE_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.sqlite");
|
|
1322
1473
|
var Database = Sqlite.Database;
|
|
1323
1474
|
|
|
1324
1475
|
class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
@@ -1369,13 +1520,31 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1369
1520
|
return this.db;
|
|
1370
1521
|
}
|
|
1371
1522
|
jsToSqlValue(column, value) {
|
|
1523
|
+
if (value !== null && value !== undefined && typeof value === "object") {
|
|
1524
|
+
if (value instanceof Date) {
|
|
1525
|
+
return super.jsToSqlValue(column, value);
|
|
1526
|
+
}
|
|
1527
|
+
if (value instanceof Uint8Array) {
|
|
1528
|
+
return super.jsToSqlValue(column, value);
|
|
1529
|
+
}
|
|
1530
|
+
if (typeof Buffer !== "undefined" && value instanceof Buffer) {
|
|
1531
|
+
return super.jsToSqlValue(column, value);
|
|
1532
|
+
}
|
|
1533
|
+
return JSON.stringify(value);
|
|
1534
|
+
}
|
|
1535
|
+
if (value === null) {
|
|
1536
|
+
const typeDef2 = this.schema.properties[column];
|
|
1537
|
+
if (typeDef2 && this.isNullable(typeDef2)) {
|
|
1538
|
+
return null;
|
|
1539
|
+
}
|
|
1540
|
+
}
|
|
1372
1541
|
const typeDef = this.schema.properties[column];
|
|
1373
1542
|
if (typeDef) {
|
|
1374
1543
|
const actualType = this.getNonNullType(typeDef);
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1544
|
+
const isObject = typeDef === true || typeof actualType !== "boolean" && actualType.type === "object";
|
|
1545
|
+
const isArray = typeDef === true || typeof actualType !== "boolean" && actualType.type === "array";
|
|
1546
|
+
const isBoolean = typeDef === true || typeof actualType !== "boolean" && actualType.type === "boolean";
|
|
1547
|
+
if (isBoolean) {
|
|
1379
1548
|
const v = value;
|
|
1380
1549
|
if (typeof v === "boolean")
|
|
1381
1550
|
return v ? 1 : 0;
|
|
@@ -1384,8 +1553,22 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1384
1553
|
if (typeof v === "string")
|
|
1385
1554
|
return v === "1" || v.toLowerCase() === "true" ? 1 : 0;
|
|
1386
1555
|
}
|
|
1556
|
+
if ((isObject || isArray) && value !== null && typeof value === "object") {
|
|
1557
|
+
if (!(value instanceof Date) && !(value instanceof Uint8Array) && (typeof Buffer === "undefined" || !(value instanceof Buffer))) {
|
|
1558
|
+
if (Array.isArray(value) || Object.getPrototypeOf(value) === Object.prototype) {
|
|
1559
|
+
return JSON.stringify(value);
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
const result = super.jsToSqlValue(column, value);
|
|
1565
|
+
if (result !== null && typeof result === "object") {
|
|
1566
|
+
const resultObj = result;
|
|
1567
|
+
if (!(resultObj instanceof Uint8Array) && (typeof Buffer === "undefined" || !(resultObj instanceof Buffer))) {
|
|
1568
|
+
return JSON.stringify(resultObj);
|
|
1569
|
+
}
|
|
1387
1570
|
}
|
|
1388
|
-
return
|
|
1571
|
+
return result;
|
|
1389
1572
|
}
|
|
1390
1573
|
sqlToJsValue(column, value) {
|
|
1391
1574
|
const typeDef = this.schema.properties[column];
|
|
@@ -1394,14 +1577,27 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1394
1577
|
return null;
|
|
1395
1578
|
}
|
|
1396
1579
|
const actualType = this.getNonNullType(typeDef);
|
|
1397
|
-
|
|
1580
|
+
const isObject = typeDef === true || typeof actualType !== "boolean" && actualType.type === "object";
|
|
1581
|
+
const isArray = typeDef === true || typeof actualType !== "boolean" && actualType.type === "array";
|
|
1582
|
+
const isBoolean = typeDef === true || typeof actualType !== "boolean" && actualType.type === "boolean";
|
|
1583
|
+
if (isBoolean) {
|
|
1398
1584
|
const v = value;
|
|
1399
1585
|
if (typeof v === "boolean")
|
|
1400
1586
|
return v;
|
|
1401
1587
|
if (typeof v === "number")
|
|
1402
|
-
return v !== 0;
|
|
1588
|
+
return v !== 0 ? true : false;
|
|
1403
1589
|
if (typeof v === "string")
|
|
1404
|
-
return v === "1" || v.toLowerCase() === "true";
|
|
1590
|
+
return v === "1" || v.toLowerCase() === "true" ? true : false;
|
|
1591
|
+
}
|
|
1592
|
+
if (isArray || isObject) {
|
|
1593
|
+
if (typeof value === "string") {
|
|
1594
|
+
try {
|
|
1595
|
+
return JSON.parse(value);
|
|
1596
|
+
} catch (e) {
|
|
1597
|
+
return value;
|
|
1598
|
+
}
|
|
1599
|
+
}
|
|
1600
|
+
return value;
|
|
1405
1601
|
}
|
|
1406
1602
|
}
|
|
1407
1603
|
return super.sqlToJsValue(column, value);
|
|
@@ -1453,6 +1649,50 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1453
1649
|
const primaryKeyParams = this.getPrimaryKeyAsOrderedArray(key);
|
|
1454
1650
|
const valueParams = this.getValueAsOrderedArray(value);
|
|
1455
1651
|
const params = [...primaryKeyParams, ...valueParams];
|
|
1652
|
+
for (let i = 0;i < params.length; i++) {
|
|
1653
|
+
let param = params[i];
|
|
1654
|
+
if (param === undefined) {
|
|
1655
|
+
params[i] = null;
|
|
1656
|
+
continue;
|
|
1657
|
+
}
|
|
1658
|
+
if (param !== null && typeof param === "object") {
|
|
1659
|
+
const paramObj = param;
|
|
1660
|
+
if (paramObj instanceof Uint8Array) {
|
|
1661
|
+
continue;
|
|
1662
|
+
}
|
|
1663
|
+
if (typeof Buffer !== "undefined" && paramObj instanceof Buffer) {
|
|
1664
|
+
params[i] = new Uint8Array(paramObj);
|
|
1665
|
+
continue;
|
|
1666
|
+
}
|
|
1667
|
+
try {
|
|
1668
|
+
params[i] = JSON.stringify(paramObj);
|
|
1669
|
+
} catch (e) {
|
|
1670
|
+
throw new Error(`Failed to stringify param at index ${i} for column binding: ${String(e)}`);
|
|
1671
|
+
}
|
|
1672
|
+
continue;
|
|
1673
|
+
}
|
|
1674
|
+
}
|
|
1675
|
+
const invalidParams = [];
|
|
1676
|
+
for (let i = 0;i < params.length; i++) {
|
|
1677
|
+
const param = params[i];
|
|
1678
|
+
if (param === null || param === undefined || typeof param === "string" || typeof param === "number" || typeof param === "boolean" || typeof param === "bigint") {
|
|
1679
|
+
continue;
|
|
1680
|
+
}
|
|
1681
|
+
if (typeof param === "object") {
|
|
1682
|
+
const paramObj = param;
|
|
1683
|
+
if (paramObj instanceof Uint8Array || typeof Buffer !== "undefined" && paramObj instanceof Buffer) {
|
|
1684
|
+
continue;
|
|
1685
|
+
}
|
|
1686
|
+
invalidParams.push({ index: i, type: typeof param, value: param });
|
|
1687
|
+
} else {
|
|
1688
|
+
invalidParams.push({ index: i, type: typeof param, value: param });
|
|
1689
|
+
}
|
|
1690
|
+
}
|
|
1691
|
+
if (invalidParams.length > 0) {
|
|
1692
|
+
console.error("Invalid params detected:", invalidParams);
|
|
1693
|
+
console.error("All params:", params.map((p, i) => ({ i, type: typeof p, value: p, isArray: Array.isArray(p) })));
|
|
1694
|
+
throw new Error(`Invalid SQLite params detected at indices: ${invalidParams.map((p) => p.index).join(", ")}`);
|
|
1695
|
+
}
|
|
1456
1696
|
const updatedEntity = stmt.get(...params);
|
|
1457
1697
|
for (const k in this.schema.properties) {
|
|
1458
1698
|
updatedEntity[k] = this.sqlToJsValue(k, updatedEntity[k]);
|
|
@@ -1480,6 +1720,17 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1480
1720
|
const primaryKeyParams = this.getPrimaryKeyAsOrderedArray(key);
|
|
1481
1721
|
const valueParams = this.getValueAsOrderedArray(value);
|
|
1482
1722
|
const params = [...primaryKeyParams, ...valueParams];
|
|
1723
|
+
for (let i = 0;i < params.length; i++) {
|
|
1724
|
+
let param = params[i];
|
|
1725
|
+
if (param === undefined) {
|
|
1726
|
+
params[i] = null;
|
|
1727
|
+
} else if (param !== null && typeof param === "object") {
|
|
1728
|
+
const paramObj = param;
|
|
1729
|
+
if (!(paramObj instanceof Uint8Array) && (typeof Buffer === "undefined" || !(paramObj instanceof Buffer))) {
|
|
1730
|
+
params[i] = JSON.stringify(paramObj);
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1483
1734
|
const updatedEntity = stmt.get(...params);
|
|
1484
1735
|
for (const k in this.schema.properties) {
|
|
1485
1736
|
updatedEntity[k] = this.sqlToJsValue(k, updatedEntity[k]);
|
|
@@ -1595,11 +1846,12 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
|
|
|
1595
1846
|
}
|
|
1596
1847
|
}
|
|
1597
1848
|
// src/tabular/SupabaseTabularRepository.ts
|
|
1598
|
-
import { createServiceToken as
|
|
1599
|
-
var SUPABASE_TABULAR_REPOSITORY =
|
|
1849
|
+
import { createServiceToken as createServiceToken13 } from "@workglow/util";
|
|
1850
|
+
var SUPABASE_TABULAR_REPOSITORY = createServiceToken13("storage.tabularRepository.supabase");
|
|
1600
1851
|
|
|
1601
1852
|
class SupabaseTabularRepository extends BaseSqlTabularRepository {
|
|
1602
1853
|
client;
|
|
1854
|
+
realtimeChannel = null;
|
|
1603
1855
|
constructor(client, table = "tabular_store", schema, primaryKeyNames, indexes = []) {
|
|
1604
1856
|
super(table, schema, primaryKeyNames, indexes);
|
|
1605
1857
|
this.client = client;
|
|
@@ -1972,10 +2224,44 @@ class SupabaseTabularRepository extends BaseSqlTabularRepository {
|
|
|
1972
2224
|
throw error;
|
|
1973
2225
|
this.events.emit("delete", column);
|
|
1974
2226
|
}
|
|
2227
|
+
convertRealtimeRow(row) {
|
|
2228
|
+
const entity = { ...row };
|
|
2229
|
+
for (const key in this.schema.properties) {
|
|
2230
|
+
entity[key] = this.sqlToJsValue(key, row[key]);
|
|
2231
|
+
}
|
|
2232
|
+
return entity;
|
|
2233
|
+
}
|
|
2234
|
+
subscribeToChanges(callback) {
|
|
2235
|
+
const channelName = `tabular-${this.table}-${Date.now()}`;
|
|
2236
|
+
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
2237
|
+
event: "*",
|
|
2238
|
+
schema: "public",
|
|
2239
|
+
table: this.table
|
|
2240
|
+
}, (payload) => {
|
|
2241
|
+
const change = {
|
|
2242
|
+
type: payload.eventType.toUpperCase(),
|
|
2243
|
+
old: payload.old && Object.keys(payload.old).length > 0 ? this.convertRealtimeRow(payload.old) : undefined,
|
|
2244
|
+
new: payload.new && Object.keys(payload.new).length > 0 ? this.convertRealtimeRow(payload.new) : undefined
|
|
2245
|
+
};
|
|
2246
|
+
callback(change);
|
|
2247
|
+
}).subscribe();
|
|
2248
|
+
return () => {
|
|
2249
|
+
if (this.realtimeChannel) {
|
|
2250
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
2251
|
+
this.realtimeChannel = null;
|
|
2252
|
+
}
|
|
2253
|
+
};
|
|
2254
|
+
}
|
|
2255
|
+
destroy() {
|
|
2256
|
+
if (this.realtimeChannel) {
|
|
2257
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
2258
|
+
this.realtimeChannel = null;
|
|
2259
|
+
}
|
|
2260
|
+
}
|
|
1975
2261
|
}
|
|
1976
2262
|
// src/kv/FsFolderJsonKvRepository.ts
|
|
1977
|
-
import { createServiceToken as
|
|
1978
|
-
var FS_FOLDER_JSON_KV_REPOSITORY =
|
|
2263
|
+
import { createServiceToken as createServiceToken14 } from "@workglow/util";
|
|
2264
|
+
var FS_FOLDER_JSON_KV_REPOSITORY = createServiceToken14("storage.kvRepository.fsFolderJson");
|
|
1979
2265
|
|
|
1980
2266
|
class FsFolderJsonKvRepository extends KvViaTabularRepository {
|
|
1981
2267
|
folderPath;
|
|
@@ -1987,10 +2273,10 @@ class FsFolderJsonKvRepository extends KvViaTabularRepository {
|
|
|
1987
2273
|
}
|
|
1988
2274
|
}
|
|
1989
2275
|
// src/kv/FsFolderKvRepository.ts
|
|
1990
|
-
import { createServiceToken as
|
|
2276
|
+
import { createServiceToken as createServiceToken15 } from "@workglow/util";
|
|
1991
2277
|
import { mkdir as mkdir2, readFile as readFile2, rm as rm2, unlink, writeFile as writeFile2 } from "fs/promises";
|
|
1992
2278
|
import path2 from "path";
|
|
1993
|
-
var FS_FOLDER_KV_REPOSITORY =
|
|
2279
|
+
var FS_FOLDER_KV_REPOSITORY = createServiceToken15("storage.kvRepository.fsFolder");
|
|
1994
2280
|
|
|
1995
2281
|
class FsFolderKvRepository extends KvRepository {
|
|
1996
2282
|
folderPath;
|
|
@@ -2067,8 +2353,8 @@ class FsFolderKvRepository extends KvRepository {
|
|
|
2067
2353
|
}
|
|
2068
2354
|
}
|
|
2069
2355
|
// src/kv/PostgresKvRepository.ts
|
|
2070
|
-
import { createServiceToken as
|
|
2071
|
-
var POSTGRES_KV_REPOSITORY =
|
|
2356
|
+
import { createServiceToken as createServiceToken16 } from "@workglow/util";
|
|
2357
|
+
var POSTGRES_KV_REPOSITORY = createServiceToken16("storage.kvRepository.postgres");
|
|
2072
2358
|
|
|
2073
2359
|
class PostgresKvRepository extends KvViaTabularRepository {
|
|
2074
2360
|
db;
|
|
@@ -2082,8 +2368,8 @@ class PostgresKvRepository extends KvViaTabularRepository {
|
|
|
2082
2368
|
}
|
|
2083
2369
|
}
|
|
2084
2370
|
// src/kv/SqliteKvRepository.ts
|
|
2085
|
-
import { createServiceToken as
|
|
2086
|
-
var SQLITE_KV_REPOSITORY =
|
|
2371
|
+
import { createServiceToken as createServiceToken17 } from "@workglow/util";
|
|
2372
|
+
var SQLITE_KV_REPOSITORY = createServiceToken17("storage.kvRepository.sqlite");
|
|
2087
2373
|
|
|
2088
2374
|
class SqliteKvRepository extends KvViaTabularRepository {
|
|
2089
2375
|
db;
|
|
@@ -2097,8 +2383,8 @@ class SqliteKvRepository extends KvViaTabularRepository {
|
|
|
2097
2383
|
}
|
|
2098
2384
|
}
|
|
2099
2385
|
// src/kv/SupabaseKvRepository.ts
|
|
2100
|
-
import { createServiceToken as
|
|
2101
|
-
var SUPABASE_KV_REPOSITORY =
|
|
2386
|
+
import { createServiceToken as createServiceToken18 } from "@workglow/util";
|
|
2387
|
+
var SUPABASE_KV_REPOSITORY = createServiceToken18("storage.kvRepository.supabase");
|
|
2102
2388
|
|
|
2103
2389
|
class SupabaseKvRepository extends KvViaTabularRepository {
|
|
2104
2390
|
client;
|
|
@@ -2112,15 +2398,169 @@ class SupabaseKvRepository extends KvViaTabularRepository {
|
|
|
2112
2398
|
}
|
|
2113
2399
|
}
|
|
2114
2400
|
// src/queue/PostgresQueueStorage.ts
|
|
2115
|
-
import { createServiceToken as
|
|
2116
|
-
|
|
2401
|
+
import { createServiceToken as createServiceToken19, makeFingerprint as makeFingerprint5, uuid4 as uuid42 } from "@workglow/util";
|
|
2402
|
+
|
|
2403
|
+
// src/util/PollingSubscriptionManager.ts
|
|
2404
|
+
class PollingSubscriptionManager {
|
|
2405
|
+
intervals = new Map;
|
|
2406
|
+
lastKnownState = new Map;
|
|
2407
|
+
initialized = false;
|
|
2408
|
+
fetchState;
|
|
2409
|
+
compareItems;
|
|
2410
|
+
payloadFactory;
|
|
2411
|
+
defaultIntervalMs;
|
|
2412
|
+
constructor(fetchState, compareItems, payloadFactory, options) {
|
|
2413
|
+
this.fetchState = fetchState;
|
|
2414
|
+
this.compareItems = compareItems;
|
|
2415
|
+
this.payloadFactory = payloadFactory;
|
|
2416
|
+
this.defaultIntervalMs = options?.defaultIntervalMs ?? 1000;
|
|
2417
|
+
}
|
|
2418
|
+
subscribe(callback, options) {
|
|
2419
|
+
const interval = options?.intervalMs ?? this.defaultIntervalMs;
|
|
2420
|
+
const subscription = {
|
|
2421
|
+
callback,
|
|
2422
|
+
intervalMs: interval
|
|
2423
|
+
};
|
|
2424
|
+
let intervalGroup = this.intervals.get(interval);
|
|
2425
|
+
if (!intervalGroup) {
|
|
2426
|
+
const subscribers = new Set;
|
|
2427
|
+
const intervalId = setInterval(() => this.poll(subscribers), interval);
|
|
2428
|
+
intervalGroup = { intervalId, subscribers };
|
|
2429
|
+
this.intervals.set(interval, intervalGroup);
|
|
2430
|
+
if (!this.initialized) {
|
|
2431
|
+
this.initialized = true;
|
|
2432
|
+
this.initAndPoll(subscribers, subscription);
|
|
2433
|
+
} else {
|
|
2434
|
+
this.pollForNewSubscriber(subscription);
|
|
2435
|
+
}
|
|
2436
|
+
} else {
|
|
2437
|
+
this.pollForNewSubscriber(subscription);
|
|
2438
|
+
}
|
|
2439
|
+
intervalGroup.subscribers.add(subscription);
|
|
2440
|
+
return () => {
|
|
2441
|
+
const group = this.intervals.get(interval);
|
|
2442
|
+
if (group) {
|
|
2443
|
+
group.subscribers.delete(subscription);
|
|
2444
|
+
if (group.subscribers.size === 0) {
|
|
2445
|
+
clearInterval(group.intervalId);
|
|
2446
|
+
this.intervals.delete(interval);
|
|
2447
|
+
}
|
|
2448
|
+
}
|
|
2449
|
+
};
|
|
2450
|
+
}
|
|
2451
|
+
async initAndPoll(subscribers, newSubscription) {
|
|
2452
|
+
try {
|
|
2453
|
+
this.lastKnownState = await this.fetchState();
|
|
2454
|
+
for (const [, item] of this.lastKnownState) {
|
|
2455
|
+
const payload = this.payloadFactory.insert(item);
|
|
2456
|
+
try {
|
|
2457
|
+
newSubscription.callback(payload);
|
|
2458
|
+
} catch {}
|
|
2459
|
+
}
|
|
2460
|
+
} catch {}
|
|
2461
|
+
}
|
|
2462
|
+
pollForNewSubscriber(subscription) {
|
|
2463
|
+
for (const [, item] of this.lastKnownState) {
|
|
2464
|
+
const payload = this.payloadFactory.insert(item);
|
|
2465
|
+
try {
|
|
2466
|
+
subscription.callback(payload);
|
|
2467
|
+
} catch {}
|
|
2468
|
+
}
|
|
2469
|
+
}
|
|
2470
|
+
async poll(subscribers) {
|
|
2471
|
+
if (subscribers.size === 0)
|
|
2472
|
+
return;
|
|
2473
|
+
try {
|
|
2474
|
+
const currentState = await this.fetchState();
|
|
2475
|
+
const changes = [];
|
|
2476
|
+
for (const [key, item] of currentState) {
|
|
2477
|
+
const oldItem = this.lastKnownState.get(key);
|
|
2478
|
+
if (!oldItem) {
|
|
2479
|
+
changes.push(this.payloadFactory.insert(item));
|
|
2480
|
+
} else if (!this.compareItems(oldItem, item)) {
|
|
2481
|
+
changes.push(this.payloadFactory.update(oldItem, item));
|
|
2482
|
+
}
|
|
2483
|
+
}
|
|
2484
|
+
for (const [key, item] of this.lastKnownState) {
|
|
2485
|
+
if (!currentState.has(key)) {
|
|
2486
|
+
changes.push(this.payloadFactory.delete(item));
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
this.lastKnownState = currentState;
|
|
2490
|
+
for (const change of changes) {
|
|
2491
|
+
for (const sub of subscribers) {
|
|
2492
|
+
try {
|
|
2493
|
+
sub.callback(change);
|
|
2494
|
+
} catch {}
|
|
2495
|
+
}
|
|
2496
|
+
}
|
|
2497
|
+
} catch {}
|
|
2498
|
+
}
|
|
2499
|
+
get subscriptionCount() {
|
|
2500
|
+
let count = 0;
|
|
2501
|
+
for (const group of this.intervals.values()) {
|
|
2502
|
+
count += group.subscribers.size;
|
|
2503
|
+
}
|
|
2504
|
+
return count;
|
|
2505
|
+
}
|
|
2506
|
+
get hasSubscriptions() {
|
|
2507
|
+
return this.intervals.size > 0;
|
|
2508
|
+
}
|
|
2509
|
+
destroy() {
|
|
2510
|
+
for (const group of this.intervals.values()) {
|
|
2511
|
+
clearInterval(group.intervalId);
|
|
2512
|
+
}
|
|
2513
|
+
this.intervals.clear();
|
|
2514
|
+
this.lastKnownState.clear();
|
|
2515
|
+
this.initialized = false;
|
|
2516
|
+
}
|
|
2517
|
+
}
|
|
2518
|
+
|
|
2519
|
+
// src/queue/PostgresQueueStorage.ts
|
|
2520
|
+
var POSTGRES_QUEUE_STORAGE = createServiceToken19("jobqueue.storage.postgres");
|
|
2117
2521
|
|
|
2118
2522
|
class PostgresQueueStorage {
|
|
2119
2523
|
db;
|
|
2120
2524
|
queueName;
|
|
2121
|
-
|
|
2525
|
+
prefixes;
|
|
2526
|
+
prefixValues;
|
|
2527
|
+
tableName;
|
|
2528
|
+
pollingManager = null;
|
|
2529
|
+
constructor(db, queueName, options) {
|
|
2122
2530
|
this.db = db;
|
|
2123
2531
|
this.queueName = queueName;
|
|
2532
|
+
this.prefixes = options?.prefixes ?? [];
|
|
2533
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
2534
|
+
if (this.prefixes.length > 0) {
|
|
2535
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
2536
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
2537
|
+
} else {
|
|
2538
|
+
this.tableName = "job_queue";
|
|
2539
|
+
}
|
|
2540
|
+
}
|
|
2541
|
+
getPrefixColumnType(type) {
|
|
2542
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
2543
|
+
}
|
|
2544
|
+
buildPrefixColumnsSql() {
|
|
2545
|
+
if (this.prefixes.length === 0)
|
|
2546
|
+
return "";
|
|
2547
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
2548
|
+
`) + `,
|
|
2549
|
+
`;
|
|
2550
|
+
}
|
|
2551
|
+
getPrefixColumnNames() {
|
|
2552
|
+
return this.prefixes.map((p) => p.name);
|
|
2553
|
+
}
|
|
2554
|
+
buildPrefixWhereClause(startParam) {
|
|
2555
|
+
if (this.prefixes.length === 0) {
|
|
2556
|
+
return { conditions: "", params: [] };
|
|
2557
|
+
}
|
|
2558
|
+
const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
|
|
2559
|
+
const params = this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2560
|
+
return { conditions: " AND " + conditions, params };
|
|
2561
|
+
}
|
|
2562
|
+
getPrefixParamValues() {
|
|
2563
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2124
2564
|
}
|
|
2125
2565
|
async setupDatabase() {
|
|
2126
2566
|
let sql;
|
|
@@ -2131,10 +2571,13 @@ class PostgresQueueStorage {
|
|
|
2131
2571
|
if (e.code !== "42710")
|
|
2132
2572
|
throw e;
|
|
2133
2573
|
}
|
|
2574
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
2575
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2576
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2134
2577
|
sql = `
|
|
2135
|
-
CREATE TABLE IF NOT EXISTS
|
|
2578
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
2136
2579
|
id SERIAL NOT NULL,
|
|
2137
|
-
fingerprint text NOT NULL,
|
|
2580
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
2138
2581
|
queue text NOT NULL,
|
|
2139
2582
|
job_run_id text NOT NULL,
|
|
2140
2583
|
status job_status NOT NULL default 'PENDING',
|
|
@@ -2151,20 +2594,22 @@ class PostgresQueueStorage {
|
|
|
2151
2594
|
error_code text,
|
|
2152
2595
|
progress real DEFAULT 0,
|
|
2153
2596
|
progress_message text DEFAULT '',
|
|
2154
|
-
progress_details jsonb
|
|
2597
|
+
progress_details jsonb,
|
|
2598
|
+
worker_id text
|
|
2155
2599
|
)`;
|
|
2156
2600
|
await this.db.query(sql);
|
|
2601
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
2157
2602
|
sql = `
|
|
2158
|
-
CREATE INDEX IF NOT EXISTS
|
|
2159
|
-
ON
|
|
2603
|
+
CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx
|
|
2604
|
+
ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`;
|
|
2160
2605
|
await this.db.query(sql);
|
|
2161
2606
|
sql = `
|
|
2162
|
-
CREATE INDEX IF NOT EXISTS
|
|
2163
|
-
ON
|
|
2607
|
+
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx
|
|
2608
|
+
ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`;
|
|
2164
2609
|
await this.db.query(sql);
|
|
2165
2610
|
sql = `
|
|
2166
|
-
CREATE INDEX IF NOT EXISTS
|
|
2167
|
-
ON
|
|
2611
|
+
CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx
|
|
2612
|
+
ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`;
|
|
2168
2613
|
await this.db.query(sql);
|
|
2169
2614
|
}
|
|
2170
2615
|
async add(job) {
|
|
@@ -2178,9 +2623,14 @@ class PostgresQueueStorage {
|
|
|
2178
2623
|
job.progress_details = null;
|
|
2179
2624
|
job.created_at = now;
|
|
2180
2625
|
job.run_after = now;
|
|
2626
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2627
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2628
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
2629
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(",") + "," : "";
|
|
2630
|
+
const baseParamStart = prefixColumnNames.length + 1;
|
|
2181
2631
|
const sql = `
|
|
2182
|
-
INSERT INTO
|
|
2183
|
-
queue,
|
|
2632
|
+
INSERT INTO ${this.tableName}(
|
|
2633
|
+
${prefixColumnsInsert}queue,
|
|
2184
2634
|
fingerprint,
|
|
2185
2635
|
input,
|
|
2186
2636
|
run_after,
|
|
@@ -2193,9 +2643,10 @@ class PostgresQueueStorage {
|
|
|
2193
2643
|
progress_details
|
|
2194
2644
|
)
|
|
2195
2645
|
VALUES
|
|
2196
|
-
($1
|
|
2646
|
+
(${prefixParamPlaceholders}$${baseParamStart},$${baseParamStart + 1},$${baseParamStart + 2},$${baseParamStart + 3},$${baseParamStart + 4},$${baseParamStart + 5},$${baseParamStart + 6},$${baseParamStart + 7},$${baseParamStart + 8},$${baseParamStart + 9},$${baseParamStart + 10})
|
|
2197
2647
|
RETURNING id`;
|
|
2198
2648
|
const params = [
|
|
2649
|
+
...prefixParamValues,
|
|
2199
2650
|
job.queue,
|
|
2200
2651
|
job.fingerprint,
|
|
2201
2652
|
JSON.stringify(job.input),
|
|
@@ -2215,68 +2666,75 @@ class PostgresQueueStorage {
|
|
|
2215
2666
|
return job.id;
|
|
2216
2667
|
}
|
|
2217
2668
|
async get(id) {
|
|
2669
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2218
2670
|
const result = await this.db.query(`SELECT *
|
|
2219
|
-
FROM
|
|
2220
|
-
WHERE id = $1 AND queue = $2
|
|
2671
|
+
FROM ${this.tableName}
|
|
2672
|
+
WHERE id = $1 AND queue = $2${prefixConditions}
|
|
2221
2673
|
FOR UPDATE SKIP LOCKED
|
|
2222
|
-
LIMIT 1`, [id, this.queueName]);
|
|
2674
|
+
LIMIT 1`, [id, this.queueName, ...prefixParams]);
|
|
2223
2675
|
if (!result || result.rows.length === 0)
|
|
2224
2676
|
return;
|
|
2225
2677
|
return result.rows[0];
|
|
2226
2678
|
}
|
|
2227
2679
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2228
2680
|
num = Number(num) || 100;
|
|
2681
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
2229
2682
|
const result = await this.db.query(`
|
|
2230
2683
|
SELECT *
|
|
2231
|
-
FROM
|
|
2684
|
+
FROM ${this.tableName}
|
|
2232
2685
|
WHERE queue = $1
|
|
2233
|
-
AND status = $2
|
|
2686
|
+
AND status = $2${prefixConditions}
|
|
2234
2687
|
ORDER BY run_after ASC
|
|
2235
2688
|
LIMIT $3
|
|
2236
|
-
FOR UPDATE SKIP LOCKED`, [this.queueName, status, num]);
|
|
2689
|
+
FOR UPDATE SKIP LOCKED`, [this.queueName, status, num, ...prefixParams]);
|
|
2237
2690
|
if (!result)
|
|
2238
2691
|
return [];
|
|
2239
2692
|
return result.rows;
|
|
2240
2693
|
}
|
|
2241
|
-
async next() {
|
|
2694
|
+
async next(workerId) {
|
|
2695
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
|
|
2242
2696
|
const result = await this.db.query(`
|
|
2243
|
-
UPDATE
|
|
2244
|
-
SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2697
|
+
UPDATE ${this.tableName}
|
|
2698
|
+
SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = $4
|
|
2245
2699
|
WHERE id = (
|
|
2246
2700
|
SELECT id
|
|
2247
|
-
FROM
|
|
2701
|
+
FROM ${this.tableName}
|
|
2248
2702
|
WHERE queue = $2
|
|
2249
|
-
AND status = $3
|
|
2703
|
+
AND status = $3${prefixConditions}
|
|
2250
2704
|
AND run_after <= NOW() AT TIME ZONE 'UTC'
|
|
2251
2705
|
ORDER BY run_after ASC
|
|
2252
2706
|
FOR UPDATE SKIP LOCKED
|
|
2253
2707
|
LIMIT 1
|
|
2254
2708
|
)
|
|
2255
|
-
RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING
|
|
2709
|
+
RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING */, workerId ?? null, ...prefixParams]);
|
|
2256
2710
|
return result?.rows?.[0] ?? undefined;
|
|
2257
2711
|
}
|
|
2258
2712
|
async size(status = "PENDING" /* PENDING */) {
|
|
2713
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2259
2714
|
const result = await this.db.query(`
|
|
2260
2715
|
SELECT COUNT(*) as count
|
|
2261
|
-
FROM
|
|
2716
|
+
FROM ${this.tableName}
|
|
2262
2717
|
WHERE queue = $1
|
|
2263
|
-
AND status = $2`, [this.queueName, status]);
|
|
2718
|
+
AND status = $2${prefixConditions}`, [this.queueName, status, ...prefixParams]);
|
|
2264
2719
|
if (!result)
|
|
2265
2720
|
return 0;
|
|
2266
2721
|
return parseInt(result.rows[0].count, 10);
|
|
2267
2722
|
}
|
|
2268
2723
|
async complete(jobDetails) {
|
|
2724
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2269
2725
|
if (jobDetails.status === "DISABLED" /* DISABLED */) {
|
|
2270
|
-
|
|
2726
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(4);
|
|
2727
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
2271
2728
|
SET
|
|
2272
2729
|
status = $1,
|
|
2273
2730
|
progress = 100,
|
|
2274
2731
|
progress_message = '',
|
|
2275
2732
|
progress_details = NULL,
|
|
2276
2733
|
completed_at = NOW() AT TIME ZONE 'UTC'
|
|
2277
|
-
WHERE id = $2 AND queue = $3`, [jobDetails.status, jobDetails.id, this.queueName]);
|
|
2734
|
+
WHERE id = $2 AND queue = $3${prefixConditions}`, [jobDetails.status, jobDetails.id, this.queueName, ...prefixParams]);
|
|
2278
2735
|
} else if (jobDetails.status === "PENDING" /* PENDING */) {
|
|
2279
|
-
|
|
2736
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
|
|
2737
|
+
await this.db.query(`UPDATE ${this.tableName}
|
|
2280
2738
|
SET
|
|
2281
2739
|
error = $1,
|
|
2282
2740
|
error_code = $2,
|
|
@@ -2287,17 +2745,19 @@ class PostgresQueueStorage {
|
|
|
2287
2745
|
progress_details = NULL,
|
|
2288
2746
|
run_attempts = run_attempts + 1,
|
|
2289
2747
|
last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2290
|
-
WHERE id = $5 AND queue = $6`, [
|
|
2748
|
+
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
2291
2749
|
jobDetails.error,
|
|
2292
2750
|
jobDetails.error_code,
|
|
2293
2751
|
jobDetails.status,
|
|
2294
2752
|
jobDetails.run_after,
|
|
2295
2753
|
jobDetails.id,
|
|
2296
|
-
this.queueName
|
|
2754
|
+
this.queueName,
|
|
2755
|
+
...prefixParams
|
|
2297
2756
|
]);
|
|
2298
2757
|
} else {
|
|
2758
|
+
const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
|
|
2299
2759
|
await this.db.query(`
|
|
2300
|
-
UPDATE
|
|
2760
|
+
UPDATE ${this.tableName}
|
|
2301
2761
|
SET
|
|
2302
2762
|
output = $1,
|
|
2303
2763
|
error = $2,
|
|
@@ -2309,86 +2769,231 @@ class PostgresQueueStorage {
|
|
|
2309
2769
|
run_attempts = run_attempts + 1,
|
|
2310
2770
|
completed_at = NOW() AT TIME ZONE 'UTC',
|
|
2311
2771
|
last_ran_at = NOW() AT TIME ZONE 'UTC'
|
|
2312
|
-
WHERE id = $5 AND queue = $6`, [
|
|
2772
|
+
WHERE id = $5 AND queue = $6${prefixConditions}`, [
|
|
2313
2773
|
jobDetails.output ? JSON.stringify(jobDetails.output) : null,
|
|
2314
2774
|
jobDetails.error ?? null,
|
|
2315
2775
|
jobDetails.error_code ?? null,
|
|
2316
2776
|
jobDetails.status,
|
|
2317
2777
|
jobDetails.id,
|
|
2318
|
-
this.queueName
|
|
2778
|
+
this.queueName,
|
|
2779
|
+
...prefixParams
|
|
2319
2780
|
]);
|
|
2320
2781
|
}
|
|
2321
2782
|
}
|
|
2322
2783
|
async deleteAll() {
|
|
2784
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
2323
2785
|
await this.db.query(`
|
|
2324
|
-
DELETE FROM
|
|
2325
|
-
WHERE queue = $1`, [this.queueName]);
|
|
2786
|
+
DELETE FROM ${this.tableName}
|
|
2787
|
+
WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
|
|
2326
2788
|
}
|
|
2327
2789
|
async outputForInput(input) {
|
|
2328
2790
|
const fingerprint = await makeFingerprint5(input);
|
|
2791
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2329
2792
|
const result = await this.db.query(`
|
|
2330
2793
|
SELECT output
|
|
2331
|
-
FROM
|
|
2332
|
-
WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'`, [fingerprint, this.queueName]);
|
|
2333
|
-
if (!result)
|
|
2794
|
+
FROM ${this.tableName}
|
|
2795
|
+
WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'${prefixConditions}`, [fingerprint, this.queueName, ...prefixParams]);
|
|
2796
|
+
if (!result || result.rows.length === 0)
|
|
2334
2797
|
return null;
|
|
2335
2798
|
return result.rows[0].output;
|
|
2336
2799
|
}
|
|
2337
2800
|
async abort(jobId) {
|
|
2338
|
-
const
|
|
2339
|
-
|
|
2801
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2802
|
+
await this.db.query(`
|
|
2803
|
+
UPDATE ${this.tableName}
|
|
2340
2804
|
SET status = 'ABORTING'
|
|
2341
|
-
WHERE id = $1 AND queue = $2`, [jobId, this.queueName]);
|
|
2805
|
+
WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
|
|
2342
2806
|
}
|
|
2343
2807
|
async getByRunId(job_run_id) {
|
|
2808
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2344
2809
|
const result = await this.db.query(`
|
|
2345
|
-
SELECT * FROM
|
|
2810
|
+
SELECT * FROM ${this.tableName} WHERE job_run_id = $1 AND queue = $2${prefixConditions}`, [job_run_id, this.queueName, ...prefixParams]);
|
|
2346
2811
|
if (!result)
|
|
2347
2812
|
return [];
|
|
2348
2813
|
return result.rows;
|
|
2349
2814
|
}
|
|
2350
2815
|
async saveProgress(jobId, progress, message, details) {
|
|
2816
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(6);
|
|
2351
2817
|
await this.db.query(`
|
|
2352
|
-
UPDATE
|
|
2818
|
+
UPDATE ${this.tableName}
|
|
2353
2819
|
SET progress = $1,
|
|
2354
2820
|
progress_message = $2,
|
|
2355
2821
|
progress_details = $3
|
|
2356
|
-
WHERE id = $4 AND queue = $5`, [
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2822
|
+
WHERE id = $4 AND queue = $5${prefixConditions}`, [
|
|
2823
|
+
progress,
|
|
2824
|
+
message,
|
|
2825
|
+
details ? JSON.stringify(details) : null,
|
|
2360
2826
|
jobId,
|
|
2361
|
-
this.queueName
|
|
2827
|
+
this.queueName,
|
|
2828
|
+
...prefixParams
|
|
2362
2829
|
]);
|
|
2363
2830
|
}
|
|
2831
|
+
async delete(jobId) {
|
|
2832
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
2833
|
+
await this.db.query(`DELETE FROM ${this.tableName} WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
|
|
2834
|
+
}
|
|
2364
2835
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2365
2836
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
2366
|
-
|
|
2837
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
|
|
2838
|
+
await this.db.query(`DELETE FROM ${this.tableName}
|
|
2367
2839
|
WHERE queue = $1
|
|
2368
2840
|
AND status = $2
|
|
2369
2841
|
AND completed_at IS NOT NULL
|
|
2370
|
-
AND completed_at <= $3`, [this.queueName, status, cutoffDate]);
|
|
2842
|
+
AND completed_at <= $3${prefixConditions}`, [this.queueName, status, cutoffDate, ...prefixParams]);
|
|
2843
|
+
}
|
|
2844
|
+
async getAllJobs() {
|
|
2845
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
2846
|
+
const result = await this.db.query(`SELECT * FROM ${this.tableName} WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
|
|
2847
|
+
if (!result)
|
|
2848
|
+
return [];
|
|
2849
|
+
return result.rows;
|
|
2850
|
+
}
|
|
2851
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
2852
|
+
const filterEntries = Object.entries(prefixFilter);
|
|
2853
|
+
let query = `SELECT * FROM ${this.tableName} WHERE queue = $1`;
|
|
2854
|
+
const params = [this.queueName];
|
|
2855
|
+
filterEntries.forEach(([key, value], index) => {
|
|
2856
|
+
query += ` AND ${key} = $${index + 2}`;
|
|
2857
|
+
params.push(value);
|
|
2858
|
+
});
|
|
2859
|
+
const result = await this.db.query(query, params);
|
|
2860
|
+
if (!result)
|
|
2861
|
+
return [];
|
|
2862
|
+
return result.rows;
|
|
2863
|
+
}
|
|
2864
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
2865
|
+
if (prefixFilter === undefined) {
|
|
2866
|
+
return false;
|
|
2867
|
+
}
|
|
2868
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
2869
|
+
return true;
|
|
2870
|
+
}
|
|
2871
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
2872
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
2873
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
2874
|
+
return true;
|
|
2875
|
+
}
|
|
2876
|
+
for (const key of instanceKeys) {
|
|
2877
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
2878
|
+
return true;
|
|
2879
|
+
}
|
|
2880
|
+
}
|
|
2881
|
+
return false;
|
|
2882
|
+
}
|
|
2883
|
+
getPollingManager() {
|
|
2884
|
+
if (!this.pollingManager) {
|
|
2885
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
2886
|
+
const jobs = await this.getAllJobs();
|
|
2887
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
2888
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
2889
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
2890
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
2891
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
2892
|
+
});
|
|
2893
|
+
}
|
|
2894
|
+
return this.pollingManager;
|
|
2895
|
+
}
|
|
2896
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
2897
|
+
let lastKnownJobs = new Map;
|
|
2898
|
+
let cancelled = false;
|
|
2899
|
+
const poll = async () => {
|
|
2900
|
+
if (cancelled)
|
|
2901
|
+
return;
|
|
2902
|
+
try {
|
|
2903
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
2904
|
+
if (cancelled)
|
|
2905
|
+
return;
|
|
2906
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
2907
|
+
for (const [id, job] of currentMap) {
|
|
2908
|
+
const old = lastKnownJobs.get(id);
|
|
2909
|
+
if (!old) {
|
|
2910
|
+
callback({ type: "INSERT", new: job });
|
|
2911
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
2912
|
+
callback({ type: "UPDATE", old, new: job });
|
|
2913
|
+
}
|
|
2914
|
+
}
|
|
2915
|
+
for (const [id, job] of lastKnownJobs) {
|
|
2916
|
+
if (!currentMap.has(id)) {
|
|
2917
|
+
callback({ type: "DELETE", old: job });
|
|
2918
|
+
}
|
|
2919
|
+
}
|
|
2920
|
+
lastKnownJobs = currentMap;
|
|
2921
|
+
} catch {}
|
|
2922
|
+
};
|
|
2923
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
2924
|
+
poll();
|
|
2925
|
+
return () => {
|
|
2926
|
+
cancelled = true;
|
|
2927
|
+
clearInterval(intervalId);
|
|
2928
|
+
};
|
|
2929
|
+
}
|
|
2930
|
+
subscribeToChanges(callback, options) {
|
|
2931
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
2932
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
2933
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
2934
|
+
}
|
|
2935
|
+
const manager = this.getPollingManager();
|
|
2936
|
+
return manager.subscribe(callback, { intervalMs });
|
|
2371
2937
|
}
|
|
2372
2938
|
}
|
|
2373
2939
|
// src/queue/SqliteQueueStorage.ts
|
|
2374
|
-
import { createServiceToken as
|
|
2375
|
-
var SQLITE_QUEUE_STORAGE =
|
|
2940
|
+
import { createServiceToken as createServiceToken20, makeFingerprint as makeFingerprint6, sleep as sleep4, uuid4 as uuid43 } from "@workglow/util";
|
|
2941
|
+
var SQLITE_QUEUE_STORAGE = createServiceToken20("jobqueue.storage.sqlite");
|
|
2376
2942
|
|
|
2377
2943
|
class SqliteQueueStorage {
|
|
2378
2944
|
db;
|
|
2379
2945
|
queueName;
|
|
2380
2946
|
options;
|
|
2947
|
+
prefixes;
|
|
2948
|
+
prefixValues;
|
|
2949
|
+
tableName;
|
|
2950
|
+
pollingManager = null;
|
|
2381
2951
|
constructor(db, queueName, options) {
|
|
2382
2952
|
this.db = db;
|
|
2383
2953
|
this.queueName = queueName;
|
|
2384
2954
|
this.options = options;
|
|
2955
|
+
this.prefixes = options?.prefixes ?? [];
|
|
2956
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
2957
|
+
if (this.prefixes.length > 0) {
|
|
2958
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
2959
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
2960
|
+
} else {
|
|
2961
|
+
this.tableName = "job_queue";
|
|
2962
|
+
}
|
|
2963
|
+
}
|
|
2964
|
+
getPrefixColumnType(type) {
|
|
2965
|
+
return type === "uuid" ? "TEXT" : "INTEGER";
|
|
2966
|
+
}
|
|
2967
|
+
buildPrefixColumnsSql() {
|
|
2968
|
+
if (this.prefixes.length === 0)
|
|
2969
|
+
return "";
|
|
2970
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
2971
|
+
`) + `,
|
|
2972
|
+
`;
|
|
2973
|
+
}
|
|
2974
|
+
getPrefixColumnNames() {
|
|
2975
|
+
return this.prefixes.map((p) => p.name);
|
|
2976
|
+
}
|
|
2977
|
+
buildPrefixWhereClause() {
|
|
2978
|
+
if (this.prefixes.length === 0) {
|
|
2979
|
+
return "";
|
|
2980
|
+
}
|
|
2981
|
+
const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
|
|
2982
|
+
return " AND " + conditions;
|
|
2983
|
+
}
|
|
2984
|
+
getPrefixParamValues() {
|
|
2985
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
2385
2986
|
}
|
|
2386
2987
|
async setupDatabase() {
|
|
2387
|
-
await
|
|
2988
|
+
await sleep4(0);
|
|
2989
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
2990
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
2991
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
2992
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
2388
2993
|
this.db.exec(`
|
|
2389
|
-
CREATE TABLE IF NOT EXISTS
|
|
2994
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
2390
2995
|
id INTEGER PRIMARY KEY,
|
|
2391
|
-
fingerprint text NOT NULL,
|
|
2996
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
2392
2997
|
queue text NOT NULL,
|
|
2393
2998
|
job_run_id text NOT NULL,
|
|
2394
2999
|
status TEXT NOT NULL default 'PENDING',
|
|
@@ -2405,12 +3010,13 @@ class SqliteQueueStorage {
|
|
|
2405
3010
|
error_code TEXT,
|
|
2406
3011
|
progress REAL DEFAULT 0,
|
|
2407
3012
|
progress_message TEXT DEFAULT '',
|
|
2408
|
-
progress_details TEXT NULL
|
|
3013
|
+
progress_details TEXT NULL,
|
|
3014
|
+
worker_id TEXT
|
|
2409
3015
|
);
|
|
2410
3016
|
|
|
2411
|
-
CREATE INDEX IF NOT EXISTS
|
|
2412
|
-
CREATE INDEX IF NOT EXISTS
|
|
2413
|
-
CREATE INDEX IF NOT EXISTS
|
|
3017
|
+
CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after);
|
|
3018
|
+
CREATE INDEX IF NOT EXISTS job_queue_fingerprint${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status);
|
|
3019
|
+
CREATE INDEX IF NOT EXISTS job_queue_job_run_id${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, job_run_id);
|
|
2414
3020
|
`);
|
|
2415
3021
|
}
|
|
2416
3022
|
async add(job) {
|
|
@@ -2424,9 +3030,13 @@ class SqliteQueueStorage {
|
|
|
2424
3030
|
job.progress_details = null;
|
|
2425
3031
|
job.created_at = now;
|
|
2426
3032
|
job.run_after = now;
|
|
3033
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3034
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3035
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
3036
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
2427
3037
|
const AddQuery = `
|
|
2428
|
-
INSERT INTO
|
|
2429
|
-
queue,
|
|
3038
|
+
INSERT INTO ${this.tableName}(
|
|
3039
|
+
${prefixColumnsInsert}queue,
|
|
2430
3040
|
fingerprint,
|
|
2431
3041
|
input,
|
|
2432
3042
|
run_after,
|
|
@@ -2438,21 +3048,23 @@ class SqliteQueueStorage {
|
|
|
2438
3048
|
progress_details,
|
|
2439
3049
|
created_at
|
|
2440
3050
|
)
|
|
2441
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
3051
|
+
VALUES (${prefixPlaceholders}?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2442
3052
|
RETURNING id`;
|
|
2443
3053
|
const stmt = this.db.prepare(AddQuery);
|
|
2444
|
-
const result = stmt.get(job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
|
|
3054
|
+
const result = stmt.get(...prefixParamValues, job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
|
|
2445
3055
|
job.id = result?.id;
|
|
2446
3056
|
return result?.id;
|
|
2447
3057
|
}
|
|
2448
3058
|
async get(id) {
|
|
3059
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3060
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2449
3061
|
const JobQuery = `
|
|
2450
3062
|
SELECT *
|
|
2451
|
-
FROM
|
|
2452
|
-
WHERE id = ? AND queue =
|
|
3063
|
+
FROM ${this.tableName}
|
|
3064
|
+
WHERE id = ? AND queue = ?${prefixConditions}
|
|
2453
3065
|
LIMIT 1`;
|
|
2454
3066
|
const stmt = this.db.prepare(JobQuery);
|
|
2455
|
-
const result = stmt.get(id, this.queueName);
|
|
3067
|
+
const result = stmt.get(String(id), this.queueName, ...prefixParams);
|
|
2456
3068
|
if (!result)
|
|
2457
3069
|
return;
|
|
2458
3070
|
if (result.input)
|
|
@@ -2465,15 +3077,17 @@ class SqliteQueueStorage {
|
|
|
2465
3077
|
}
|
|
2466
3078
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2467
3079
|
num = Number(num) || 100;
|
|
3080
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3081
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2468
3082
|
const FutureJobQuery = `
|
|
2469
3083
|
SELECT *
|
|
2470
|
-
FROM
|
|
3084
|
+
FROM ${this.tableName}
|
|
2471
3085
|
WHERE queue = ?
|
|
2472
|
-
AND status =
|
|
3086
|
+
AND status = ?${prefixConditions}
|
|
2473
3087
|
ORDER BY run_after ASC
|
|
2474
3088
|
LIMIT ${num}`;
|
|
2475
3089
|
const stmt = this.db.prepare(FutureJobQuery);
|
|
2476
|
-
const result = stmt.all(this.queueName, status);
|
|
3090
|
+
const result = stmt.all(this.queueName, status, ...prefixParams);
|
|
2477
3091
|
return (result || []).map((details) => {
|
|
2478
3092
|
if (details.input)
|
|
2479
3093
|
details.input = JSON.parse(details.input);
|
|
@@ -2485,20 +3099,24 @@ class SqliteQueueStorage {
|
|
|
2485
3099
|
});
|
|
2486
3100
|
}
|
|
2487
3101
|
async abort(jobId) {
|
|
3102
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3103
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2488
3104
|
const AbortQuery = `
|
|
2489
|
-
UPDATE
|
|
3105
|
+
UPDATE ${this.tableName}
|
|
2490
3106
|
SET status = ?
|
|
2491
|
-
WHERE id = ? AND queue =
|
|
3107
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2492
3108
|
const stmt = this.db.prepare(AbortQuery);
|
|
2493
|
-
stmt.run("ABORTING" /* ABORTING */, jobId, this.queueName);
|
|
3109
|
+
stmt.run("ABORTING" /* ABORTING */, String(jobId), this.queueName, ...prefixParams);
|
|
2494
3110
|
}
|
|
2495
3111
|
async getByRunId(job_run_id) {
|
|
3112
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3113
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2496
3114
|
const JobsByRunIdQuery = `
|
|
2497
3115
|
SELECT *
|
|
2498
|
-
FROM
|
|
2499
|
-
WHERE job_run_id = ? AND queue =
|
|
3116
|
+
FROM ${this.tableName}
|
|
3117
|
+
WHERE job_run_id = ? AND queue = ?${prefixConditions}`;
|
|
2500
3118
|
const stmt = this.db.prepare(JobsByRunIdQuery);
|
|
2501
|
-
const result = stmt.all(job_run_id, this.queueName);
|
|
3119
|
+
const result = stmt.all(job_run_id, this.queueName, ...prefixParams);
|
|
2502
3120
|
return (result || []).map((details) => {
|
|
2503
3121
|
if (details.input)
|
|
2504
3122
|
details.input = JSON.parse(details.input);
|
|
@@ -2509,22 +3127,24 @@ class SqliteQueueStorage {
|
|
|
2509
3127
|
return details;
|
|
2510
3128
|
});
|
|
2511
3129
|
}
|
|
2512
|
-
async next() {
|
|
3130
|
+
async next(workerId) {
|
|
2513
3131
|
const now = new Date().toISOString();
|
|
3132
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3133
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2514
3134
|
const stmt = this.db.prepare(`
|
|
2515
|
-
UPDATE
|
|
2516
|
-
SET status = ?, last_ran_at = ?
|
|
3135
|
+
UPDATE ${this.tableName}
|
|
3136
|
+
SET status = ?, last_ran_at = ?, worker_id = ?
|
|
2517
3137
|
WHERE id = (
|
|
2518
3138
|
SELECT id
|
|
2519
|
-
FROM
|
|
3139
|
+
FROM ${this.tableName}
|
|
2520
3140
|
WHERE queue = ?
|
|
2521
|
-
AND status =
|
|
3141
|
+
AND status = ?${prefixConditions}
|
|
2522
3142
|
AND run_after <= ?
|
|
2523
3143
|
ORDER BY run_after ASC
|
|
2524
3144
|
LIMIT 1
|
|
2525
3145
|
)
|
|
2526
3146
|
RETURNING *`);
|
|
2527
|
-
const result = stmt.get("PROCESSING" /* PROCESSING */, now, this.queueName, "PENDING" /* PENDING */, now);
|
|
3147
|
+
const result = stmt.get("PROCESSING" /* PROCESSING */, now, workerId ?? null, this.queueName, "PENDING" /* PENDING */, ...prefixParams, now);
|
|
2528
3148
|
if (!result)
|
|
2529
3149
|
return;
|
|
2530
3150
|
if (result.input)
|
|
@@ -2536,33 +3156,37 @@ class SqliteQueueStorage {
|
|
|
2536
3156
|
return result;
|
|
2537
3157
|
}
|
|
2538
3158
|
async size(status = "PENDING" /* PENDING */) {
|
|
3159
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3160
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2539
3161
|
const sizeQuery = `
|
|
2540
3162
|
SELECT COUNT(*) as count
|
|
2541
|
-
FROM
|
|
3163
|
+
FROM ${this.tableName}
|
|
2542
3164
|
WHERE queue = ?
|
|
2543
|
-
AND status =
|
|
3165
|
+
AND status = ?${prefixConditions}`;
|
|
2544
3166
|
const stmt = this.db.prepare(sizeQuery);
|
|
2545
|
-
const result = stmt.get(this.queueName, status);
|
|
3167
|
+
const result = stmt.get(this.queueName, status, ...prefixParams);
|
|
2546
3168
|
return result.count;
|
|
2547
3169
|
}
|
|
2548
3170
|
async complete(job) {
|
|
2549
3171
|
const now = new Date().toISOString();
|
|
3172
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3173
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2550
3174
|
let updateQuery;
|
|
2551
3175
|
let params;
|
|
2552
3176
|
if (job.status === "DISABLED" /* DISABLED */) {
|
|
2553
3177
|
updateQuery = `
|
|
2554
|
-
UPDATE
|
|
3178
|
+
UPDATE ${this.tableName}
|
|
2555
3179
|
SET
|
|
2556
3180
|
status = ?,
|
|
2557
3181
|
progress = 100,
|
|
2558
3182
|
progress_message = '',
|
|
2559
3183
|
progress_details = NULL,
|
|
2560
3184
|
completed_at = ?
|
|
2561
|
-
WHERE id = ? AND queue =
|
|
2562
|
-
params = [job.status, now, job.id, this.queueName];
|
|
3185
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
3186
|
+
params = [job.status, now, job.id, this.queueName, ...prefixParams];
|
|
2563
3187
|
} else {
|
|
2564
3188
|
updateQuery = `
|
|
2565
|
-
UPDATE
|
|
3189
|
+
UPDATE ${this.tableName}
|
|
2566
3190
|
SET
|
|
2567
3191
|
output = ?,
|
|
2568
3192
|
error = ?,
|
|
@@ -2574,7 +3198,7 @@ class SqliteQueueStorage {
|
|
|
2574
3198
|
last_ran_at = ?,
|
|
2575
3199
|
completed_at = ?,
|
|
2576
3200
|
run_attempts = run_attempts + 1
|
|
2577
|
-
WHERE id = ? AND queue =
|
|
3201
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2578
3202
|
params = [
|
|
2579
3203
|
job.output ? JSON.stringify(job.output) : null,
|
|
2580
3204
|
job.error ?? null,
|
|
@@ -2583,80 +3207,243 @@ class SqliteQueueStorage {
|
|
|
2583
3207
|
now,
|
|
2584
3208
|
now,
|
|
2585
3209
|
job.id,
|
|
2586
|
-
this.queueName
|
|
3210
|
+
this.queueName,
|
|
3211
|
+
...prefixParams
|
|
2587
3212
|
];
|
|
2588
3213
|
}
|
|
2589
3214
|
const stmt = this.db.prepare(updateQuery);
|
|
2590
3215
|
stmt.run(...params);
|
|
2591
3216
|
}
|
|
2592
3217
|
async deleteAll() {
|
|
3218
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3219
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2593
3220
|
const ClearQuery = `
|
|
2594
|
-
DELETE FROM
|
|
2595
|
-
WHERE queue =
|
|
3221
|
+
DELETE FROM ${this.tableName}
|
|
3222
|
+
WHERE queue = ?${prefixConditions}`;
|
|
2596
3223
|
const stmt = this.db.prepare(ClearQuery);
|
|
2597
|
-
stmt.run(this.queueName);
|
|
3224
|
+
stmt.run(this.queueName, ...prefixParams);
|
|
2598
3225
|
}
|
|
2599
3226
|
async outputForInput(input) {
|
|
2600
3227
|
const fingerprint = await makeFingerprint6(input);
|
|
3228
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3229
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2601
3230
|
const OutputQuery = `
|
|
2602
3231
|
SELECT output
|
|
2603
|
-
FROM
|
|
2604
|
-
WHERE queue = ? AND fingerprint = ? AND status =
|
|
3232
|
+
FROM ${this.tableName}
|
|
3233
|
+
WHERE queue = ? AND fingerprint = ? AND status = ?${prefixConditions}`;
|
|
2605
3234
|
const stmt = this.db.prepare(OutputQuery);
|
|
2606
|
-
const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED
|
|
3235
|
+
const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED */, ...prefixParams);
|
|
2607
3236
|
return result?.output ? JSON.parse(result.output) : null;
|
|
2608
3237
|
}
|
|
2609
3238
|
async saveProgress(jobId, progress, message, details) {
|
|
3239
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3240
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2610
3241
|
const UpdateProgressQuery = `
|
|
2611
|
-
UPDATE
|
|
3242
|
+
UPDATE ${this.tableName}
|
|
2612
3243
|
SET progress = ?,
|
|
2613
3244
|
progress_message = ?,
|
|
2614
3245
|
progress_details = ?
|
|
2615
|
-
WHERE id = ? AND queue =
|
|
3246
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2616
3247
|
const stmt = this.db.prepare(UpdateProgressQuery);
|
|
2617
|
-
stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName);
|
|
3248
|
+
stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName, ...prefixParams);
|
|
2618
3249
|
}
|
|
2619
3250
|
async delete(jobId) {
|
|
3251
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3252
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2620
3253
|
const DeleteQuery = `
|
|
2621
|
-
DELETE FROM
|
|
2622
|
-
WHERE id = ? AND queue =
|
|
3254
|
+
DELETE FROM ${this.tableName}
|
|
3255
|
+
WHERE id = ? AND queue = ?${prefixConditions}`;
|
|
2623
3256
|
const stmt = this.db.prepare(DeleteQuery);
|
|
2624
|
-
stmt.run(String(jobId), this.queueName);
|
|
3257
|
+
stmt.run(String(jobId), this.queueName, ...prefixParams);
|
|
2625
3258
|
}
|
|
2626
3259
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2627
3260
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
3261
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3262
|
+
const prefixParams = this.getPrefixParamValues();
|
|
2628
3263
|
const DeleteQuery = `
|
|
2629
|
-
DELETE FROM
|
|
3264
|
+
DELETE FROM ${this.tableName}
|
|
2630
3265
|
WHERE queue = ?
|
|
2631
3266
|
AND status = ?
|
|
2632
3267
|
AND completed_at IS NOT NULL
|
|
2633
|
-
AND completed_at <=
|
|
3268
|
+
AND completed_at <= ?${prefixConditions}`;
|
|
2634
3269
|
const stmt = this.db.prepare(DeleteQuery);
|
|
2635
|
-
stmt.run(this.queueName, status, cutoffDate);
|
|
3270
|
+
stmt.run(this.queueName, status, cutoffDate, ...prefixParams);
|
|
2636
3271
|
}
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
3272
|
+
getAllJobs() {
|
|
3273
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
3274
|
+
const prefixParams = this.getPrefixParamValues();
|
|
3275
|
+
const AllJobsQuery = `
|
|
3276
|
+
SELECT *
|
|
3277
|
+
FROM ${this.tableName}
|
|
3278
|
+
WHERE queue = ?${prefixConditions}`;
|
|
3279
|
+
const stmt = this.db.prepare(AllJobsQuery);
|
|
3280
|
+
const result = stmt.all(this.queueName, ...prefixParams);
|
|
3281
|
+
return (result || []).map((details) => {
|
|
3282
|
+
if (details.input)
|
|
3283
|
+
details.input = JSON.parse(details.input);
|
|
3284
|
+
if (details.output)
|
|
3285
|
+
details.output = JSON.parse(details.output);
|
|
3286
|
+
if (details.progress_details)
|
|
3287
|
+
details.progress_details = JSON.parse(details.progress_details);
|
|
3288
|
+
return details;
|
|
3289
|
+
});
|
|
2648
3290
|
}
|
|
2649
|
-
|
|
2650
|
-
const
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
3291
|
+
getAllJobsWithFilter(prefixFilter) {
|
|
3292
|
+
const filterEntries = Object.entries(prefixFilter);
|
|
3293
|
+
let whereClause = "WHERE queue = ?";
|
|
3294
|
+
const params = [this.queueName];
|
|
3295
|
+
for (const [key, value] of filterEntries) {
|
|
3296
|
+
whereClause += ` AND ${key} = ?`;
|
|
3297
|
+
params.push(value);
|
|
3298
|
+
}
|
|
3299
|
+
const AllJobsQuery = `SELECT * FROM ${this.tableName} ${whereClause}`;
|
|
3300
|
+
const stmt = this.db.prepare(AllJobsQuery);
|
|
3301
|
+
const result = stmt.all(...params);
|
|
3302
|
+
return (result || []).map((details) => {
|
|
3303
|
+
if (details.input)
|
|
3304
|
+
details.input = JSON.parse(details.input);
|
|
3305
|
+
if (details.output)
|
|
3306
|
+
details.output = JSON.parse(details.output);
|
|
3307
|
+
if (details.progress_details)
|
|
3308
|
+
details.progress_details = JSON.parse(details.progress_details);
|
|
3309
|
+
return details;
|
|
3310
|
+
});
|
|
3311
|
+
}
|
|
3312
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
3313
|
+
if (prefixFilter === undefined) {
|
|
3314
|
+
return false;
|
|
2654
3315
|
}
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
3316
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
3317
|
+
return true;
|
|
3318
|
+
}
|
|
3319
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
3320
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
3321
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
3322
|
+
return true;
|
|
3323
|
+
}
|
|
3324
|
+
for (const key of instanceKeys) {
|
|
3325
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
3326
|
+
return true;
|
|
3327
|
+
}
|
|
3328
|
+
}
|
|
3329
|
+
return false;
|
|
3330
|
+
}
|
|
3331
|
+
getPollingManager() {
|
|
3332
|
+
if (!this.pollingManager) {
|
|
3333
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
3334
|
+
const jobs = this.getAllJobs();
|
|
3335
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
3336
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
3337
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
3338
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
3339
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
3340
|
+
});
|
|
3341
|
+
}
|
|
3342
|
+
return this.pollingManager;
|
|
3343
|
+
}
|
|
3344
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
3345
|
+
let lastKnownJobs = new Map;
|
|
3346
|
+
const poll = () => {
|
|
3347
|
+
try {
|
|
3348
|
+
const currentJobs = this.getAllJobsWithFilter(prefixFilter);
|
|
3349
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
3350
|
+
for (const [id, job] of currentMap) {
|
|
3351
|
+
const old = lastKnownJobs.get(id);
|
|
3352
|
+
if (!old) {
|
|
3353
|
+
callback({ type: "INSERT", new: job });
|
|
3354
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
3355
|
+
callback({ type: "UPDATE", old, new: job });
|
|
3356
|
+
}
|
|
3357
|
+
}
|
|
3358
|
+
for (const [id, job] of lastKnownJobs) {
|
|
3359
|
+
if (!currentMap.has(id)) {
|
|
3360
|
+
callback({ type: "DELETE", old: job });
|
|
3361
|
+
}
|
|
3362
|
+
}
|
|
3363
|
+
lastKnownJobs = currentMap;
|
|
3364
|
+
} catch {}
|
|
3365
|
+
};
|
|
3366
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
3367
|
+
poll();
|
|
3368
|
+
return () => {
|
|
3369
|
+
clearInterval(intervalId);
|
|
3370
|
+
};
|
|
3371
|
+
}
|
|
3372
|
+
subscribeToChanges(callback, options) {
|
|
3373
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
3374
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
3375
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
3376
|
+
}
|
|
3377
|
+
const manager = this.getPollingManager();
|
|
3378
|
+
return manager.subscribe(callback, { intervalMs });
|
|
3379
|
+
}
|
|
3380
|
+
}
|
|
3381
|
+
// src/queue/SupabaseQueueStorage.ts
|
|
3382
|
+
import { createServiceToken as createServiceToken21, makeFingerprint as makeFingerprint7, uuid4 as uuid44 } from "@workglow/util";
|
|
3383
|
+
var SUPABASE_QUEUE_STORAGE = createServiceToken21("jobqueue.storage.supabase");
|
|
3384
|
+
|
|
3385
|
+
class SupabaseQueueStorage {
|
|
3386
|
+
client;
|
|
3387
|
+
queueName;
|
|
3388
|
+
prefixes;
|
|
3389
|
+
prefixValues;
|
|
3390
|
+
tableName;
|
|
3391
|
+
realtimeChannel = null;
|
|
3392
|
+
pollingManager = null;
|
|
3393
|
+
constructor(client, queueName, options) {
|
|
3394
|
+
this.client = client;
|
|
3395
|
+
this.queueName = queueName;
|
|
3396
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3397
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3398
|
+
if (this.prefixes.length > 0) {
|
|
3399
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3400
|
+
this.tableName = `job_queue_${prefixNames}`;
|
|
3401
|
+
} else {
|
|
3402
|
+
this.tableName = "job_queue";
|
|
3403
|
+
}
|
|
3404
|
+
}
|
|
3405
|
+
getPrefixColumnType(type) {
|
|
3406
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
3407
|
+
}
|
|
3408
|
+
buildPrefixColumnsSql() {
|
|
3409
|
+
if (this.prefixes.length === 0)
|
|
3410
|
+
return "";
|
|
3411
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
3412
|
+
`) + `,
|
|
3413
|
+
`;
|
|
3414
|
+
}
|
|
3415
|
+
getPrefixColumnNames() {
|
|
3416
|
+
return this.prefixes.map((p) => p.name);
|
|
3417
|
+
}
|
|
3418
|
+
applyPrefixFilters(query) {
|
|
3419
|
+
let result = query;
|
|
3420
|
+
for (const prefix of this.prefixes) {
|
|
3421
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
3422
|
+
}
|
|
3423
|
+
return result;
|
|
3424
|
+
}
|
|
3425
|
+
getPrefixInsertValues() {
|
|
3426
|
+
const values = {};
|
|
3427
|
+
for (const prefix of this.prefixes) {
|
|
3428
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
3429
|
+
}
|
|
3430
|
+
return values;
|
|
3431
|
+
}
|
|
3432
|
+
async setupDatabase() {
|
|
3433
|
+
const createTypeSql = `CREATE TYPE job_status AS ENUM (${Object.values(JobStatus).map((v) => `'${v}'`).join(",")})`;
|
|
3434
|
+
const { error: typeError } = await this.client.rpc("exec_sql", { query: createTypeSql });
|
|
3435
|
+
if (typeError && typeError.code !== "42710") {
|
|
3436
|
+
throw typeError;
|
|
3437
|
+
}
|
|
3438
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
3439
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3440
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3441
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
3442
|
+
const createTableSql = `
|
|
3443
|
+
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
3444
|
+
id SERIAL NOT NULL,
|
|
3445
|
+
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
3446
|
+
queue text NOT NULL,
|
|
2660
3447
|
job_run_id text NOT NULL,
|
|
2661
3448
|
status job_status NOT NULL default 'PENDING',
|
|
2662
3449
|
input jsonb NOT NULL,
|
|
@@ -2672,7 +3459,8 @@ class SupabaseQueueStorage {
|
|
|
2672
3459
|
error_code text,
|
|
2673
3460
|
progress real DEFAULT 0,
|
|
2674
3461
|
progress_message text DEFAULT '',
|
|
2675
|
-
progress_details jsonb
|
|
3462
|
+
progress_details jsonb,
|
|
3463
|
+
worker_id text
|
|
2676
3464
|
)`;
|
|
2677
3465
|
const { error: tableError } = await this.client.rpc("exec_sql", { query: createTableSql });
|
|
2678
3466
|
if (tableError) {
|
|
@@ -2681,12 +3469,12 @@ class SupabaseQueueStorage {
|
|
|
2681
3469
|
}
|
|
2682
3470
|
}
|
|
2683
3471
|
const indexes = [
|
|
2684
|
-
`CREATE INDEX IF NOT EXISTS
|
|
2685
|
-
`CREATE INDEX IF NOT EXISTS
|
|
2686
|
-
`CREATE INDEX IF NOT EXISTS
|
|
3472
|
+
`CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`,
|
|
3473
|
+
`CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`,
|
|
3474
|
+
`CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`
|
|
2687
3475
|
];
|
|
2688
3476
|
for (const indexSql of indexes) {
|
|
2689
|
-
|
|
3477
|
+
await this.client.rpc("exec_sql", { query: indexSql });
|
|
2690
3478
|
}
|
|
2691
3479
|
}
|
|
2692
3480
|
async add(job) {
|
|
@@ -2700,7 +3488,9 @@ class SupabaseQueueStorage {
|
|
|
2700
3488
|
job.progress_details = null;
|
|
2701
3489
|
job.created_at = now;
|
|
2702
3490
|
job.run_after = now;
|
|
2703
|
-
const
|
|
3491
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
3492
|
+
const { data, error } = await this.client.from(this.tableName).insert({
|
|
3493
|
+
...prefixInsertValues,
|
|
2704
3494
|
queue: job.queue,
|
|
2705
3495
|
fingerprint: job.fingerprint,
|
|
2706
3496
|
input: job.input,
|
|
@@ -2721,7 +3511,9 @@ class SupabaseQueueStorage {
|
|
|
2721
3511
|
return job.id;
|
|
2722
3512
|
}
|
|
2723
3513
|
async get(id) {
|
|
2724
|
-
|
|
3514
|
+
let query = this.client.from(this.tableName).select("*").eq("id", id).eq("queue", this.queueName);
|
|
3515
|
+
query = this.applyPrefixFilters(query);
|
|
3516
|
+
const { data, error } = await query.single();
|
|
2725
3517
|
if (error) {
|
|
2726
3518
|
if (error.code === "PGRST116")
|
|
2727
3519
|
return;
|
|
@@ -2731,36 +3523,53 @@ class SupabaseQueueStorage {
|
|
|
2731
3523
|
}
|
|
2732
3524
|
async peek(status = "PENDING" /* PENDING */, num = 100) {
|
|
2733
3525
|
num = Number(num) || 100;
|
|
2734
|
-
|
|
3526
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", status);
|
|
3527
|
+
query = this.applyPrefixFilters(query);
|
|
3528
|
+
const { data, error } = await query.order("run_after", { ascending: true }).limit(num);
|
|
2735
3529
|
if (error)
|
|
2736
3530
|
throw error;
|
|
2737
3531
|
return data ?? [];
|
|
2738
3532
|
}
|
|
2739
|
-
async next() {
|
|
2740
|
-
|
|
3533
|
+
async next(workerId) {
|
|
3534
|
+
let selectQuery = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", "PENDING" /* PENDING */).lte("run_after", new Date().toISOString());
|
|
3535
|
+
selectQuery = this.applyPrefixFilters(selectQuery);
|
|
3536
|
+
const { data: jobs, error: selectError } = await selectQuery.order("run_after", { ascending: true }).limit(1);
|
|
2741
3537
|
if (selectError)
|
|
2742
3538
|
throw selectError;
|
|
2743
3539
|
if (!jobs || jobs.length === 0)
|
|
2744
3540
|
return;
|
|
2745
3541
|
const job = jobs[0];
|
|
2746
|
-
|
|
3542
|
+
let updateQuery = this.client.from(this.tableName).update({
|
|
2747
3543
|
status: "PROCESSING" /* PROCESSING */,
|
|
2748
|
-
last_ran_at: new Date().toISOString()
|
|
2749
|
-
|
|
3544
|
+
last_ran_at: new Date().toISOString(),
|
|
3545
|
+
worker_id: workerId ?? null
|
|
3546
|
+
}).eq("id", job.id).eq("queue", this.queueName);
|
|
3547
|
+
updateQuery = this.applyPrefixFilters(updateQuery);
|
|
3548
|
+
const { data: updatedJob, error: updateError } = await updateQuery.select().single();
|
|
2750
3549
|
if (updateError)
|
|
2751
3550
|
throw updateError;
|
|
2752
3551
|
return updatedJob;
|
|
2753
3552
|
}
|
|
2754
3553
|
async size(status = "PENDING" /* PENDING */) {
|
|
2755
|
-
|
|
3554
|
+
let query = this.client.from(this.tableName).select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
|
|
3555
|
+
query = this.applyPrefixFilters(query);
|
|
3556
|
+
const { count, error } = await query;
|
|
2756
3557
|
if (error)
|
|
2757
3558
|
throw error;
|
|
2758
3559
|
return count ?? 0;
|
|
2759
3560
|
}
|
|
3561
|
+
async getAllJobs() {
|
|
3562
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
3563
|
+
query = this.applyPrefixFilters(query);
|
|
3564
|
+
const { data, error } = await query;
|
|
3565
|
+
if (error)
|
|
3566
|
+
throw error;
|
|
3567
|
+
return data ?? [];
|
|
3568
|
+
}
|
|
2760
3569
|
async complete(jobDetails) {
|
|
2761
3570
|
const now = new Date().toISOString();
|
|
2762
3571
|
if (jobDetails.status === "DISABLED" /* DISABLED */) {
|
|
2763
|
-
|
|
3572
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2764
3573
|
status: jobDetails.status,
|
|
2765
3574
|
progress: 100,
|
|
2766
3575
|
progress_message: "",
|
|
@@ -2768,16 +3577,20 @@ class SupabaseQueueStorage {
|
|
|
2768
3577
|
completed_at: now,
|
|
2769
3578
|
last_ran_at: now
|
|
2770
3579
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3580
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3581
|
+
const { error: error2 } = await query2;
|
|
2771
3582
|
if (error2)
|
|
2772
3583
|
throw error2;
|
|
2773
3584
|
return;
|
|
2774
3585
|
}
|
|
2775
|
-
|
|
3586
|
+
let getQuery = this.client.from(this.tableName).select("run_attempts").eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3587
|
+
getQuery = this.applyPrefixFilters(getQuery);
|
|
3588
|
+
const { data: current, error: getError } = await getQuery.single();
|
|
2776
3589
|
if (getError)
|
|
2777
3590
|
throw getError;
|
|
2778
3591
|
const nextAttempts = (current?.run_attempts ?? 0) + 1;
|
|
2779
3592
|
if (jobDetails.status === "PENDING" /* PENDING */) {
|
|
2780
|
-
|
|
3593
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2781
3594
|
error: jobDetails.error ?? null,
|
|
2782
3595
|
error_code: jobDetails.error_code ?? null,
|
|
2783
3596
|
status: jobDetails.status,
|
|
@@ -2788,12 +3601,14 @@ class SupabaseQueueStorage {
|
|
|
2788
3601
|
run_attempts: nextAttempts,
|
|
2789
3602
|
last_ran_at: now
|
|
2790
3603
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3604
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3605
|
+
const { error: error2 } = await query2;
|
|
2791
3606
|
if (error2)
|
|
2792
3607
|
throw error2;
|
|
2793
3608
|
return;
|
|
2794
3609
|
}
|
|
2795
3610
|
if (jobDetails.status === "COMPLETED" /* COMPLETED */ || jobDetails.status === "FAILED" /* FAILED */) {
|
|
2796
|
-
|
|
3611
|
+
let query2 = this.client.from(this.tableName).update({
|
|
2797
3612
|
output: jobDetails.output ?? null,
|
|
2798
3613
|
error: jobDetails.error ?? null,
|
|
2799
3614
|
error_code: jobDetails.error_code ?? null,
|
|
@@ -2805,11 +3620,13 @@ class SupabaseQueueStorage {
|
|
|
2805
3620
|
completed_at: now,
|
|
2806
3621
|
last_ran_at: now
|
|
2807
3622
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3623
|
+
query2 = this.applyPrefixFilters(query2);
|
|
3624
|
+
const { error: error2 } = await query2;
|
|
2808
3625
|
if (error2)
|
|
2809
3626
|
throw error2;
|
|
2810
3627
|
return;
|
|
2811
3628
|
}
|
|
2812
|
-
|
|
3629
|
+
let query = this.client.from(this.tableName).update({
|
|
2813
3630
|
status: jobDetails.status,
|
|
2814
3631
|
output: jobDetails.output ?? null,
|
|
2815
3632
|
error: jobDetails.error ?? null,
|
|
@@ -2818,17 +3635,23 @@ class SupabaseQueueStorage {
|
|
|
2818
3635
|
run_attempts: nextAttempts,
|
|
2819
3636
|
last_ran_at: now
|
|
2820
3637
|
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
3638
|
+
query = this.applyPrefixFilters(query);
|
|
3639
|
+
const { error } = await query;
|
|
2821
3640
|
if (error)
|
|
2822
3641
|
throw error;
|
|
2823
3642
|
}
|
|
2824
3643
|
async deleteAll() {
|
|
2825
|
-
|
|
3644
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
|
|
3645
|
+
query = this.applyPrefixFilters(query);
|
|
3646
|
+
const { error } = await query;
|
|
2826
3647
|
if (error)
|
|
2827
3648
|
throw error;
|
|
2828
3649
|
}
|
|
2829
3650
|
async outputForInput(input) {
|
|
2830
3651
|
const fingerprint = await makeFingerprint7(input);
|
|
2831
|
-
|
|
3652
|
+
let query = this.client.from(this.tableName).select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", "COMPLETED" /* COMPLETED */);
|
|
3653
|
+
query = this.applyPrefixFilters(query);
|
|
3654
|
+
const { data, error } = await query.single();
|
|
2832
3655
|
if (error) {
|
|
2833
3656
|
if (error.code === "PGRST116")
|
|
2834
3657
|
return null;
|
|
@@ -2837,42 +3660,617 @@ class SupabaseQueueStorage {
|
|
|
2837
3660
|
return data?.output ?? null;
|
|
2838
3661
|
}
|
|
2839
3662
|
async abort(jobId) {
|
|
2840
|
-
|
|
3663
|
+
let query = this.client.from(this.tableName).update({ status: "ABORTING" /* ABORTING */ }).eq("id", jobId).eq("queue", this.queueName);
|
|
3664
|
+
query = this.applyPrefixFilters(query);
|
|
3665
|
+
const { error } = await query;
|
|
2841
3666
|
if (error)
|
|
2842
3667
|
throw error;
|
|
2843
3668
|
}
|
|
2844
3669
|
async getByRunId(job_run_id) {
|
|
2845
|
-
|
|
3670
|
+
let query = this.client.from(this.tableName).select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
|
|
3671
|
+
query = this.applyPrefixFilters(query);
|
|
3672
|
+
const { data, error } = await query;
|
|
2846
3673
|
if (error)
|
|
2847
3674
|
throw error;
|
|
2848
3675
|
return data ?? [];
|
|
2849
3676
|
}
|
|
2850
3677
|
async saveProgress(jobId, progress, message, details) {
|
|
2851
|
-
|
|
3678
|
+
let query = this.client.from(this.tableName).update({
|
|
2852
3679
|
progress,
|
|
2853
3680
|
progress_message: message,
|
|
2854
3681
|
progress_details: details
|
|
2855
3682
|
}).eq("id", jobId).eq("queue", this.queueName);
|
|
3683
|
+
query = this.applyPrefixFilters(query);
|
|
3684
|
+
const { error } = await query;
|
|
2856
3685
|
if (error)
|
|
2857
3686
|
throw error;
|
|
2858
3687
|
}
|
|
2859
3688
|
async delete(jobId) {
|
|
2860
|
-
|
|
3689
|
+
let query = this.client.from(this.tableName).delete().eq("id", jobId).eq("queue", this.queueName);
|
|
3690
|
+
query = this.applyPrefixFilters(query);
|
|
3691
|
+
const { error } = await query;
|
|
2861
3692
|
if (error)
|
|
2862
3693
|
throw error;
|
|
2863
3694
|
}
|
|
2864
3695
|
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
2865
3696
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
2866
|
-
|
|
3697
|
+
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
|
|
3698
|
+
query = this.applyPrefixFilters(query);
|
|
3699
|
+
const { error } = await query;
|
|
3700
|
+
if (error)
|
|
3701
|
+
throw error;
|
|
3702
|
+
}
|
|
3703
|
+
matchesPrefixFilter(job, prefixFilter) {
|
|
3704
|
+
if (!job)
|
|
3705
|
+
return false;
|
|
3706
|
+
if (job.queue !== this.queueName) {
|
|
3707
|
+
return false;
|
|
3708
|
+
}
|
|
3709
|
+
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
3710
|
+
return true;
|
|
3711
|
+
}
|
|
3712
|
+
const filterValues = prefixFilter ?? this.prefixValues;
|
|
3713
|
+
if (Object.keys(filterValues).length === 0) {
|
|
3714
|
+
return true;
|
|
3715
|
+
}
|
|
3716
|
+
for (const [key, value] of Object.entries(filterValues)) {
|
|
3717
|
+
if (job[key] !== value) {
|
|
3718
|
+
return false;
|
|
3719
|
+
}
|
|
3720
|
+
}
|
|
3721
|
+
return true;
|
|
3722
|
+
}
|
|
3723
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
3724
|
+
if (prefixFilter === undefined) {
|
|
3725
|
+
return false;
|
|
3726
|
+
}
|
|
3727
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
3728
|
+
return true;
|
|
3729
|
+
}
|
|
3730
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
3731
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
3732
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
3733
|
+
return true;
|
|
3734
|
+
}
|
|
3735
|
+
for (const key of instanceKeys) {
|
|
3736
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
3737
|
+
return true;
|
|
3738
|
+
}
|
|
3739
|
+
}
|
|
3740
|
+
return false;
|
|
3741
|
+
}
|
|
3742
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
3743
|
+
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
3744
|
+
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
3745
|
+
query = query.eq(key, value);
|
|
3746
|
+
}
|
|
3747
|
+
const { data, error } = await query;
|
|
2867
3748
|
if (error)
|
|
2868
3749
|
throw error;
|
|
3750
|
+
return data ?? [];
|
|
3751
|
+
}
|
|
3752
|
+
subscribeToChanges(callback, options) {
|
|
3753
|
+
return this.subscribeToChangesWithRealtime(callback, options?.prefixFilter);
|
|
3754
|
+
}
|
|
3755
|
+
subscribeToChangesWithRealtime(callback, prefixFilter) {
|
|
3756
|
+
const channelName = `queue-${this.tableName}-${this.queueName}-${Date.now()}`;
|
|
3757
|
+
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
3758
|
+
event: "*",
|
|
3759
|
+
schema: "public",
|
|
3760
|
+
table: this.tableName,
|
|
3761
|
+
filter: `queue=eq.${this.queueName}`
|
|
3762
|
+
}, (payload) => {
|
|
3763
|
+
const newJob = payload.new;
|
|
3764
|
+
const oldJob = payload.old;
|
|
3765
|
+
const newMatches = this.matchesPrefixFilter(newJob, prefixFilter);
|
|
3766
|
+
const oldMatches = this.matchesPrefixFilter(oldJob, prefixFilter);
|
|
3767
|
+
if (!newMatches && !oldMatches) {
|
|
3768
|
+
return;
|
|
3769
|
+
}
|
|
3770
|
+
callback({
|
|
3771
|
+
type: payload.eventType.toUpperCase(),
|
|
3772
|
+
old: oldJob && Object.keys(oldJob).length > 0 ? oldJob : undefined,
|
|
3773
|
+
new: newJob && Object.keys(newJob).length > 0 ? newJob : undefined
|
|
3774
|
+
});
|
|
3775
|
+
}).subscribe();
|
|
3776
|
+
return () => {
|
|
3777
|
+
if (this.realtimeChannel) {
|
|
3778
|
+
this.client.removeChannel(this.realtimeChannel);
|
|
3779
|
+
this.realtimeChannel = null;
|
|
3780
|
+
}
|
|
3781
|
+
};
|
|
3782
|
+
}
|
|
3783
|
+
getPollingManager() {
|
|
3784
|
+
if (!this.pollingManager) {
|
|
3785
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
3786
|
+
const jobs = await this.getAllJobs();
|
|
3787
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
3788
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
3789
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
3790
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
3791
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
3792
|
+
});
|
|
3793
|
+
}
|
|
3794
|
+
return this.pollingManager;
|
|
3795
|
+
}
|
|
3796
|
+
subscribeWithCustomPrefixFilterPolling(callback, prefixFilter, intervalMs) {
|
|
3797
|
+
let lastKnownJobs = new Map;
|
|
3798
|
+
let cancelled = false;
|
|
3799
|
+
const poll = async () => {
|
|
3800
|
+
if (cancelled)
|
|
3801
|
+
return;
|
|
3802
|
+
try {
|
|
3803
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
3804
|
+
if (cancelled)
|
|
3805
|
+
return;
|
|
3806
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
3807
|
+
for (const [id, job] of currentMap) {
|
|
3808
|
+
const old = lastKnownJobs.get(id);
|
|
3809
|
+
if (!old) {
|
|
3810
|
+
callback({ type: "INSERT", new: job });
|
|
3811
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
3812
|
+
callback({ type: "UPDATE", old, new: job });
|
|
3813
|
+
}
|
|
3814
|
+
}
|
|
3815
|
+
for (const [id, job] of lastKnownJobs) {
|
|
3816
|
+
if (!currentMap.has(id)) {
|
|
3817
|
+
callback({ type: "DELETE", old: job });
|
|
3818
|
+
}
|
|
3819
|
+
}
|
|
3820
|
+
lastKnownJobs = currentMap;
|
|
3821
|
+
} catch {}
|
|
3822
|
+
};
|
|
3823
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
3824
|
+
poll();
|
|
3825
|
+
return () => {
|
|
3826
|
+
cancelled = true;
|
|
3827
|
+
clearInterval(intervalId);
|
|
3828
|
+
};
|
|
3829
|
+
}
|
|
3830
|
+
subscribeToChangesWithPolling(callback, options) {
|
|
3831
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
3832
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
3833
|
+
return this.subscribeWithCustomPrefixFilterPolling(callback, options.prefixFilter, intervalMs);
|
|
3834
|
+
}
|
|
3835
|
+
const manager = this.getPollingManager();
|
|
3836
|
+
return manager.subscribe(callback, { intervalMs });
|
|
3837
|
+
}
|
|
3838
|
+
}
|
|
3839
|
+
// src/limiter/PostgresRateLimiterStorage.ts
|
|
3840
|
+
import { createServiceToken as createServiceToken22 } from "@workglow/util";
|
|
3841
|
+
var POSTGRES_RATE_LIMITER_STORAGE = createServiceToken22("ratelimiter.storage.postgres");
|
|
3842
|
+
|
|
3843
|
+
class PostgresRateLimiterStorage {
|
|
3844
|
+
db;
|
|
3845
|
+
prefixes;
|
|
3846
|
+
prefixValues;
|
|
3847
|
+
executionTableName;
|
|
3848
|
+
nextAvailableTableName;
|
|
3849
|
+
constructor(db, options) {
|
|
3850
|
+
this.db = db;
|
|
3851
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3852
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3853
|
+
if (this.prefixes.length > 0) {
|
|
3854
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3855
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
3856
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
3857
|
+
} else {
|
|
3858
|
+
this.executionTableName = "rate_limit_executions";
|
|
3859
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
3860
|
+
}
|
|
3861
|
+
}
|
|
3862
|
+
getPrefixColumnType(type) {
|
|
3863
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
3864
|
+
}
|
|
3865
|
+
buildPrefixColumnsSql() {
|
|
3866
|
+
if (this.prefixes.length === 0)
|
|
3867
|
+
return "";
|
|
3868
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
3869
|
+
`) + `,
|
|
3870
|
+
`;
|
|
3871
|
+
}
|
|
3872
|
+
getPrefixColumnNames() {
|
|
3873
|
+
return this.prefixes.map((p) => p.name);
|
|
3874
|
+
}
|
|
3875
|
+
buildPrefixWhereClause(startParam) {
|
|
3876
|
+
if (this.prefixes.length === 0) {
|
|
3877
|
+
return { conditions: "", params: [] };
|
|
3878
|
+
}
|
|
3879
|
+
const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
|
|
3880
|
+
const params = this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3881
|
+
return { conditions: " AND " + conditions, params };
|
|
3882
|
+
}
|
|
3883
|
+
getPrefixParamValues() {
|
|
3884
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3885
|
+
}
|
|
3886
|
+
async setupDatabase() {
|
|
3887
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
3888
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3889
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3890
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
3891
|
+
await this.db.query(`
|
|
3892
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
3893
|
+
id SERIAL PRIMARY KEY,
|
|
3894
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3895
|
+
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
3896
|
+
)
|
|
3897
|
+
`);
|
|
3898
|
+
await this.db.query(`
|
|
3899
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
3900
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
3901
|
+
`);
|
|
3902
|
+
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
3903
|
+
await this.db.query(`
|
|
3904
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
3905
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
3906
|
+
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
3907
|
+
PRIMARY KEY (${primaryKeyColumns})
|
|
3908
|
+
)
|
|
3909
|
+
`);
|
|
3910
|
+
}
|
|
3911
|
+
async recordExecution(queueName) {
|
|
3912
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3913
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3914
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
3915
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
|
|
3916
|
+
const queueParamNum = prefixColumnNames.length + 1;
|
|
3917
|
+
await this.db.query(`
|
|
3918
|
+
INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
|
|
3919
|
+
VALUES (${prefixParamPlaceholders}$${queueParamNum})
|
|
3920
|
+
`, [...prefixParamValues, queueName]);
|
|
3921
|
+
}
|
|
3922
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
3923
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
3924
|
+
const result = await this.db.query(`
|
|
3925
|
+
SELECT COUNT(*) AS count
|
|
3926
|
+
FROM ${this.executionTableName}
|
|
3927
|
+
WHERE queue_name = $1 AND executed_at > $2${prefixConditions}
|
|
3928
|
+
`, [queueName, windowStartTime, ...prefixParams]);
|
|
3929
|
+
return parseInt(result.rows[0]?.count ?? "0", 10);
|
|
3930
|
+
}
|
|
3931
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
3932
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
3933
|
+
const result = await this.db.query(`
|
|
3934
|
+
SELECT executed_at
|
|
3935
|
+
FROM ${this.executionTableName}
|
|
3936
|
+
WHERE queue_name = $1${prefixConditions}
|
|
3937
|
+
ORDER BY executed_at ASC
|
|
3938
|
+
LIMIT 1 OFFSET $2
|
|
3939
|
+
`, [queueName, offset, ...prefixParams]);
|
|
3940
|
+
const executedAt = result.rows[0]?.executed_at;
|
|
3941
|
+
if (!executedAt)
|
|
3942
|
+
return;
|
|
3943
|
+
return new Date(executedAt).toISOString();
|
|
3944
|
+
}
|
|
3945
|
+
async getNextAvailableTime(queueName) {
|
|
3946
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
3947
|
+
const result = await this.db.query(`
|
|
3948
|
+
SELECT next_available_at
|
|
3949
|
+
FROM ${this.nextAvailableTableName}
|
|
3950
|
+
WHERE queue_name = $1${prefixConditions}
|
|
3951
|
+
`, [queueName, ...prefixParams]);
|
|
3952
|
+
const nextAvailableAt = result.rows[0]?.next_available_at;
|
|
3953
|
+
if (!nextAvailableAt)
|
|
3954
|
+
return;
|
|
3955
|
+
return new Date(nextAvailableAt).toISOString();
|
|
3956
|
+
}
|
|
3957
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
3958
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
3959
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
3960
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
3961
|
+
const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
|
|
3962
|
+
const baseParamStart = prefixColumnNames.length + 1;
|
|
3963
|
+
const conflictColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
3964
|
+
await this.db.query(`
|
|
3965
|
+
INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
|
|
3966
|
+
VALUES (${prefixParamPlaceholders}$${baseParamStart}, $${baseParamStart + 1})
|
|
3967
|
+
ON CONFLICT (${conflictColumns})
|
|
3968
|
+
DO UPDATE SET next_available_at = EXCLUDED.next_available_at
|
|
3969
|
+
`, [...prefixParamValues, queueName, nextAvailableAt]);
|
|
3970
|
+
}
|
|
3971
|
+
async clear(queueName) {
|
|
3972
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
|
|
3973
|
+
await this.db.query(`DELETE FROM ${this.executionTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
|
|
3974
|
+
await this.db.query(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
|
|
3975
|
+
}
|
|
3976
|
+
}
|
|
3977
|
+
// src/limiter/SqliteRateLimiterStorage.ts
|
|
3978
|
+
import { createServiceToken as createServiceToken23, sleep as sleep5, toSQLiteTimestamp } from "@workglow/util";
|
|
3979
|
+
var SQLITE_RATE_LIMITER_STORAGE = createServiceToken23("ratelimiter.storage.sqlite");
|
|
3980
|
+
|
|
3981
|
+
class SqliteRateLimiterStorage {
|
|
3982
|
+
db;
|
|
3983
|
+
prefixes;
|
|
3984
|
+
prefixValues;
|
|
3985
|
+
executionTableName;
|
|
3986
|
+
nextAvailableTableName;
|
|
3987
|
+
constructor(db, options) {
|
|
3988
|
+
this.db = db;
|
|
3989
|
+
this.prefixes = options?.prefixes ?? [];
|
|
3990
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
3991
|
+
if (this.prefixes.length > 0) {
|
|
3992
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
3993
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
3994
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
3995
|
+
} else {
|
|
3996
|
+
this.executionTableName = "rate_limit_executions";
|
|
3997
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
3998
|
+
}
|
|
3999
|
+
}
|
|
4000
|
+
getPrefixColumnType(type) {
|
|
4001
|
+
return type === "uuid" ? "TEXT" : "INTEGER";
|
|
4002
|
+
}
|
|
4003
|
+
buildPrefixColumnsSql() {
|
|
4004
|
+
if (this.prefixes.length === 0)
|
|
4005
|
+
return "";
|
|
4006
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
4007
|
+
`) + `,
|
|
4008
|
+
`;
|
|
4009
|
+
}
|
|
4010
|
+
getPrefixColumnNames() {
|
|
4011
|
+
return this.prefixes.map((p) => p.name);
|
|
4012
|
+
}
|
|
4013
|
+
buildPrefixWhereClause() {
|
|
4014
|
+
if (this.prefixes.length === 0) {
|
|
4015
|
+
return "";
|
|
4016
|
+
}
|
|
4017
|
+
const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
|
|
4018
|
+
return " AND " + conditions;
|
|
4019
|
+
}
|
|
4020
|
+
getPrefixParamValues() {
|
|
4021
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
4022
|
+
}
|
|
4023
|
+
async setupDatabase() {
|
|
4024
|
+
await sleep5(0);
|
|
4025
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
4026
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4027
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
4028
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
4029
|
+
this.db.exec(`
|
|
4030
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
4031
|
+
id INTEGER PRIMARY KEY,
|
|
4032
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
4033
|
+
executed_at TEXT DEFAULT CURRENT_TIMESTAMP
|
|
4034
|
+
);
|
|
4035
|
+
|
|
4036
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
4037
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at);
|
|
4038
|
+
`);
|
|
4039
|
+
this.db.exec(`
|
|
4040
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
4041
|
+
${prefixColumnsSql}queue_name TEXT PRIMARY KEY,
|
|
4042
|
+
next_available_at TEXT
|
|
4043
|
+
);
|
|
4044
|
+
`);
|
|
4045
|
+
}
|
|
4046
|
+
async recordExecution(queueName) {
|
|
4047
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4048
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
4049
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
4050
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
4051
|
+
const stmt = this.db.prepare(`
|
|
4052
|
+
INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
|
|
4053
|
+
VALUES (${prefixPlaceholders}?)
|
|
4054
|
+
`);
|
|
4055
|
+
stmt.run(...prefixParamValues, queueName);
|
|
4056
|
+
}
|
|
4057
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
4058
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
4059
|
+
const prefixParams = this.getPrefixParamValues();
|
|
4060
|
+
const thresholdTime = toSQLiteTimestamp(new Date(windowStartTime));
|
|
4061
|
+
const stmt = this.db.prepare(`
|
|
4062
|
+
SELECT COUNT(*) AS count
|
|
4063
|
+
FROM ${this.executionTableName}
|
|
4064
|
+
WHERE queue_name = ? AND executed_at > ?${prefixConditions}
|
|
4065
|
+
`);
|
|
4066
|
+
const result = stmt.get(queueName, thresholdTime, ...prefixParams);
|
|
4067
|
+
return result?.count ?? 0;
|
|
4068
|
+
}
|
|
4069
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
4070
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
4071
|
+
const prefixParams = this.getPrefixParamValues();
|
|
4072
|
+
const stmt = this.db.prepare(`
|
|
4073
|
+
SELECT executed_at
|
|
4074
|
+
FROM ${this.executionTableName}
|
|
4075
|
+
WHERE queue_name = ?${prefixConditions}
|
|
4076
|
+
ORDER BY executed_at ASC
|
|
4077
|
+
LIMIT 1 OFFSET ?
|
|
4078
|
+
`);
|
|
4079
|
+
const result = stmt.get(queueName, ...prefixParams, offset);
|
|
4080
|
+
if (!result)
|
|
4081
|
+
return;
|
|
4082
|
+
return result.executed_at + "Z";
|
|
4083
|
+
}
|
|
4084
|
+
async getNextAvailableTime(queueName) {
|
|
4085
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
4086
|
+
const prefixParams = this.getPrefixParamValues();
|
|
4087
|
+
const stmt = this.db.prepare(`
|
|
4088
|
+
SELECT next_available_at
|
|
4089
|
+
FROM ${this.nextAvailableTableName}
|
|
4090
|
+
WHERE queue_name = ?${prefixConditions}
|
|
4091
|
+
`);
|
|
4092
|
+
const result = stmt.get(queueName, ...prefixParams);
|
|
4093
|
+
if (!result?.next_available_at)
|
|
4094
|
+
return;
|
|
4095
|
+
return result.next_available_at + "Z";
|
|
4096
|
+
}
|
|
4097
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
4098
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4099
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
4100
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
4101
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
4102
|
+
const stmt = this.db.prepare(`
|
|
4103
|
+
INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
|
|
4104
|
+
VALUES (${prefixPlaceholders}?, ?)
|
|
4105
|
+
ON CONFLICT(queue_name) DO UPDATE SET next_available_at = excluded.next_available_at
|
|
4106
|
+
`);
|
|
4107
|
+
stmt.run(...prefixParamValues, queueName, nextAvailableAt);
|
|
4108
|
+
}
|
|
4109
|
+
async clear(queueName) {
|
|
4110
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
4111
|
+
const prefixParams = this.getPrefixParamValues();
|
|
4112
|
+
this.db.prepare(`DELETE FROM ${this.executionTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
|
|
4113
|
+
this.db.prepare(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
|
|
4114
|
+
}
|
|
4115
|
+
}
|
|
4116
|
+
// src/limiter/SupabaseRateLimiterStorage.ts
|
|
4117
|
+
import { createServiceToken as createServiceToken24 } from "@workglow/util";
|
|
4118
|
+
var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken24("ratelimiter.storage.supabase");
|
|
4119
|
+
|
|
4120
|
+
class SupabaseRateLimiterStorage {
|
|
4121
|
+
client;
|
|
4122
|
+
prefixes;
|
|
4123
|
+
prefixValues;
|
|
4124
|
+
executionTableName;
|
|
4125
|
+
nextAvailableTableName;
|
|
4126
|
+
constructor(client, options) {
|
|
4127
|
+
this.client = client;
|
|
4128
|
+
this.prefixes = options?.prefixes ?? [];
|
|
4129
|
+
this.prefixValues = options?.prefixValues ?? {};
|
|
4130
|
+
if (this.prefixes.length > 0) {
|
|
4131
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
4132
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
4133
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
4134
|
+
} else {
|
|
4135
|
+
this.executionTableName = "rate_limit_executions";
|
|
4136
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
4137
|
+
}
|
|
4138
|
+
}
|
|
4139
|
+
getPrefixColumnType(type) {
|
|
4140
|
+
return type === "uuid" ? "UUID" : "INTEGER";
|
|
4141
|
+
}
|
|
4142
|
+
buildPrefixColumnsSql() {
|
|
4143
|
+
if (this.prefixes.length === 0)
|
|
4144
|
+
return "";
|
|
4145
|
+
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
4146
|
+
`) + `,
|
|
4147
|
+
`;
|
|
4148
|
+
}
|
|
4149
|
+
getPrefixColumnNames() {
|
|
4150
|
+
return this.prefixes.map((p) => p.name);
|
|
4151
|
+
}
|
|
4152
|
+
applyPrefixFilters(query) {
|
|
4153
|
+
let result = query;
|
|
4154
|
+
for (const prefix of this.prefixes) {
|
|
4155
|
+
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
4156
|
+
}
|
|
4157
|
+
return result;
|
|
4158
|
+
}
|
|
4159
|
+
getPrefixInsertValues() {
|
|
4160
|
+
const values = {};
|
|
4161
|
+
for (const prefix of this.prefixes) {
|
|
4162
|
+
values[prefix.name] = this.prefixValues[prefix.name];
|
|
4163
|
+
}
|
|
4164
|
+
return values;
|
|
4165
|
+
}
|
|
4166
|
+
async setupDatabase() {
|
|
4167
|
+
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
4168
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4169
|
+
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
4170
|
+
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
4171
|
+
const createExecTableSql = `
|
|
4172
|
+
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
4173
|
+
id SERIAL PRIMARY KEY,
|
|
4174
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
4175
|
+
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
4176
|
+
)
|
|
4177
|
+
`;
|
|
4178
|
+
const { error: execTableError } = await this.client.rpc("exec_sql", {
|
|
4179
|
+
query: createExecTableSql
|
|
4180
|
+
});
|
|
4181
|
+
if (execTableError && execTableError.code !== "42P07") {
|
|
4182
|
+
throw execTableError;
|
|
4183
|
+
}
|
|
4184
|
+
const createExecIndexSql = `
|
|
4185
|
+
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
4186
|
+
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
4187
|
+
`;
|
|
4188
|
+
await this.client.rpc("exec_sql", { query: createExecIndexSql });
|
|
4189
|
+
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
4190
|
+
const createNextTableSql = `
|
|
4191
|
+
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
4192
|
+
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
4193
|
+
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
4194
|
+
PRIMARY KEY (${primaryKeyColumns})
|
|
4195
|
+
)
|
|
4196
|
+
`;
|
|
4197
|
+
const { error: nextTableError } = await this.client.rpc("exec_sql", {
|
|
4198
|
+
query: createNextTableSql
|
|
4199
|
+
});
|
|
4200
|
+
if (nextTableError && nextTableError.code !== "42P07") {
|
|
4201
|
+
throw nextTableError;
|
|
4202
|
+
}
|
|
4203
|
+
}
|
|
4204
|
+
async recordExecution(queueName) {
|
|
4205
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
4206
|
+
const { error } = await this.client.from(this.executionTableName).insert({
|
|
4207
|
+
...prefixInsertValues,
|
|
4208
|
+
queue_name: queueName
|
|
4209
|
+
});
|
|
4210
|
+
if (error)
|
|
4211
|
+
throw error;
|
|
4212
|
+
}
|
|
4213
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
4214
|
+
let query = this.client.from(this.executionTableName).select("*", { count: "exact", head: true }).eq("queue_name", queueName).gt("executed_at", windowStartTime);
|
|
4215
|
+
query = this.applyPrefixFilters(query);
|
|
4216
|
+
const { count, error } = await query;
|
|
4217
|
+
if (error)
|
|
4218
|
+
throw error;
|
|
4219
|
+
return count ?? 0;
|
|
4220
|
+
}
|
|
4221
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
4222
|
+
let query = this.client.from(this.executionTableName).select("executed_at").eq("queue_name", queueName);
|
|
4223
|
+
query = this.applyPrefixFilters(query);
|
|
4224
|
+
const { data, error } = await query.order("executed_at", { ascending: true }).range(offset, offset);
|
|
4225
|
+
if (error)
|
|
4226
|
+
throw error;
|
|
4227
|
+
if (!data || data.length === 0)
|
|
4228
|
+
return;
|
|
4229
|
+
return new Date(data[0].executed_at).toISOString();
|
|
4230
|
+
}
|
|
4231
|
+
async getNextAvailableTime(queueName) {
|
|
4232
|
+
let query = this.client.from(this.nextAvailableTableName).select("next_available_at").eq("queue_name", queueName);
|
|
4233
|
+
query = this.applyPrefixFilters(query);
|
|
4234
|
+
const { data, error } = await query.single();
|
|
4235
|
+
if (error) {
|
|
4236
|
+
if (error.code === "PGRST116")
|
|
4237
|
+
return;
|
|
4238
|
+
throw error;
|
|
4239
|
+
}
|
|
4240
|
+
if (!data?.next_available_at)
|
|
4241
|
+
return;
|
|
4242
|
+
return new Date(data.next_available_at).toISOString();
|
|
4243
|
+
}
|
|
4244
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
4245
|
+
const prefixInsertValues = this.getPrefixInsertValues();
|
|
4246
|
+
const { error } = await this.client.from(this.nextAvailableTableName).upsert({
|
|
4247
|
+
...prefixInsertValues,
|
|
4248
|
+
queue_name: queueName,
|
|
4249
|
+
next_available_at: nextAvailableAt
|
|
4250
|
+
}, {
|
|
4251
|
+
onConflict: this.prefixes.length > 0 ? `${this.getPrefixColumnNames().join(",")},queue_name` : "queue_name"
|
|
4252
|
+
});
|
|
4253
|
+
if (error)
|
|
4254
|
+
throw error;
|
|
4255
|
+
}
|
|
4256
|
+
async clear(queueName) {
|
|
4257
|
+
let execQuery = this.client.from(this.executionTableName).delete().eq("queue_name", queueName);
|
|
4258
|
+
execQuery = this.applyPrefixFilters(execQuery);
|
|
4259
|
+
const { error: execError } = await execQuery;
|
|
4260
|
+
if (execError)
|
|
4261
|
+
throw execError;
|
|
4262
|
+
let nextQuery = this.client.from(this.nextAvailableTableName).delete().eq("queue_name", queueName);
|
|
4263
|
+
nextQuery = this.applyPrefixFilters(nextQuery);
|
|
4264
|
+
const { error: nextError } = await nextQuery;
|
|
4265
|
+
if (nextError)
|
|
4266
|
+
throw nextError;
|
|
2869
4267
|
}
|
|
2870
4268
|
}
|
|
2871
4269
|
// src/kv/IndexedDbKvRepository.ts
|
|
2872
|
-
import { createServiceToken as
|
|
4270
|
+
import { createServiceToken as createServiceToken26 } from "@workglow/util";
|
|
2873
4271
|
|
|
2874
4272
|
// src/tabular/IndexedDbTabularRepository.ts
|
|
2875
|
-
import { createServiceToken as
|
|
4273
|
+
import { createServiceToken as createServiceToken25 } from "@workglow/util";
|
|
2876
4274
|
|
|
2877
4275
|
// src/util/IndexedDbTable.ts
|
|
2878
4276
|
var METADATA_STORE_NAME = "__schema_metadata__";
|
|
@@ -3212,7 +4610,7 @@ async function dropIndexedDbTable(tableName) {
|
|
|
3212
4610
|
}
|
|
3213
4611
|
|
|
3214
4612
|
// src/tabular/IndexedDbTabularRepository.ts
|
|
3215
|
-
var IDB_TABULAR_REPOSITORY =
|
|
4613
|
+
var IDB_TABULAR_REPOSITORY = createServiceToken25("storage.tabularRepository.indexedDb");
|
|
3216
4614
|
|
|
3217
4615
|
class IndexedDbTabularRepository extends TabularRepository {
|
|
3218
4616
|
table;
|
|
@@ -3548,7 +4946,7 @@ class IndexedDbTabularRepository extends TabularRepository {
|
|
|
3548
4946
|
}
|
|
3549
4947
|
|
|
3550
4948
|
// src/kv/IndexedDbKvRepository.ts
|
|
3551
|
-
var IDB_KV_REPOSITORY =
|
|
4949
|
+
var IDB_KV_REPOSITORY = createServiceToken26("storage.kvRepository.indexedDb");
|
|
3552
4950
|
|
|
3553
4951
|
class IndexedDbKvRepository extends KvViaTabularRepository {
|
|
3554
4952
|
dbName;
|
|
@@ -3560,18 +4958,42 @@ class IndexedDbKvRepository extends KvViaTabularRepository {
|
|
|
3560
4958
|
}
|
|
3561
4959
|
}
|
|
3562
4960
|
// src/queue/IndexedDbQueueStorage.ts
|
|
3563
|
-
import { createServiceToken as
|
|
3564
|
-
var INDEXED_DB_QUEUE_STORAGE =
|
|
4961
|
+
import { createServiceToken as createServiceToken27, makeFingerprint as makeFingerprint8, uuid4 as uuid45 } from "@workglow/util";
|
|
4962
|
+
var INDEXED_DB_QUEUE_STORAGE = createServiceToken27("jobqueue.storage.indexedDb");
|
|
3565
4963
|
|
|
3566
4964
|
class IndexedDbQueueStorage {
|
|
3567
4965
|
queueName;
|
|
3568
4966
|
db;
|
|
3569
4967
|
tableName;
|
|
3570
4968
|
migrationOptions;
|
|
3571
|
-
|
|
4969
|
+
prefixes;
|
|
4970
|
+
prefixValues;
|
|
4971
|
+
pollingManager = null;
|
|
4972
|
+
constructor(queueName, options = {}) {
|
|
3572
4973
|
this.queueName = queueName;
|
|
3573
|
-
this.
|
|
3574
|
-
this.
|
|
4974
|
+
this.migrationOptions = options;
|
|
4975
|
+
this.prefixes = options.prefixes ?? [];
|
|
4976
|
+
this.prefixValues = options.prefixValues ?? {};
|
|
4977
|
+
if (this.prefixes.length > 0) {
|
|
4978
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
4979
|
+
this.tableName = `jobs_${prefixNames}`;
|
|
4980
|
+
} else {
|
|
4981
|
+
this.tableName = "jobs";
|
|
4982
|
+
}
|
|
4983
|
+
}
|
|
4984
|
+
getPrefixColumnNames() {
|
|
4985
|
+
return this.prefixes.map((p) => p.name);
|
|
4986
|
+
}
|
|
4987
|
+
matchesPrefixes(job) {
|
|
4988
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
4989
|
+
if (job[key] !== value) {
|
|
4990
|
+
return false;
|
|
4991
|
+
}
|
|
4992
|
+
}
|
|
4993
|
+
return true;
|
|
4994
|
+
}
|
|
4995
|
+
getPrefixKeyValues() {
|
|
4996
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
3575
4997
|
}
|
|
3576
4998
|
async getDb() {
|
|
3577
4999
|
if (this.db)
|
|
@@ -3580,25 +5002,29 @@ class IndexedDbQueueStorage {
|
|
|
3580
5002
|
return this.db;
|
|
3581
5003
|
}
|
|
3582
5004
|
async setupDatabase() {
|
|
5005
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
5006
|
+
const buildKeyPath = (basePath) => {
|
|
5007
|
+
return [...prefixColumnNames, ...basePath];
|
|
5008
|
+
};
|
|
3583
5009
|
const expectedIndexes = [
|
|
3584
5010
|
{
|
|
3585
|
-
name: "
|
|
3586
|
-
keyPath:
|
|
5011
|
+
name: "queue_status",
|
|
5012
|
+
keyPath: buildKeyPath(["queue", "status"]),
|
|
3587
5013
|
options: { unique: false }
|
|
3588
5014
|
},
|
|
3589
5015
|
{
|
|
3590
|
-
name: "
|
|
3591
|
-
keyPath: ["status", "run_after"],
|
|
5016
|
+
name: "queue_status_run_after",
|
|
5017
|
+
keyPath: buildKeyPath(["queue", "status", "run_after"]),
|
|
3592
5018
|
options: { unique: false }
|
|
3593
5019
|
},
|
|
3594
5020
|
{
|
|
3595
|
-
name: "
|
|
3596
|
-
keyPath:
|
|
5021
|
+
name: "queue_job_run_id",
|
|
5022
|
+
keyPath: buildKeyPath(["queue", "job_run_id"]),
|
|
3597
5023
|
options: { unique: false }
|
|
3598
5024
|
},
|
|
3599
5025
|
{
|
|
3600
|
-
name: "
|
|
3601
|
-
keyPath: ["fingerprint", "status"],
|
|
5026
|
+
name: "queue_fingerprint_status",
|
|
5027
|
+
keyPath: buildKeyPath(["queue", "fingerprint", "status"]),
|
|
3602
5028
|
options: { unique: false }
|
|
3603
5029
|
}
|
|
3604
5030
|
];
|
|
@@ -3607,21 +5033,25 @@ class IndexedDbQueueStorage {
|
|
|
3607
5033
|
async add(job) {
|
|
3608
5034
|
const db = await this.getDb();
|
|
3609
5035
|
const now = new Date().toISOString();
|
|
3610
|
-
|
|
3611
|
-
|
|
3612
|
-
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
|
|
3619
|
-
|
|
5036
|
+
const jobWithPrefixes = job;
|
|
5037
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid45();
|
|
5038
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid45();
|
|
5039
|
+
jobWithPrefixes.queue = this.queueName;
|
|
5040
|
+
jobWithPrefixes.fingerprint = await makeFingerprint8(jobWithPrefixes.input);
|
|
5041
|
+
jobWithPrefixes.status = "PENDING" /* PENDING */;
|
|
5042
|
+
jobWithPrefixes.progress = 0;
|
|
5043
|
+
jobWithPrefixes.progress_message = "";
|
|
5044
|
+
jobWithPrefixes.progress_details = null;
|
|
5045
|
+
jobWithPrefixes.created_at = now;
|
|
5046
|
+
jobWithPrefixes.run_after = now;
|
|
5047
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5048
|
+
jobWithPrefixes[key] = value;
|
|
5049
|
+
}
|
|
3620
5050
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3621
5051
|
const store = tx.objectStore(this.tableName);
|
|
3622
5052
|
return new Promise((resolve, reject) => {
|
|
3623
|
-
const request = store.add(
|
|
3624
|
-
tx.oncomplete = () => resolve(
|
|
5053
|
+
const request = store.add(jobWithPrefixes);
|
|
5054
|
+
tx.oncomplete = () => resolve(jobWithPrefixes.id);
|
|
3625
5055
|
tx.onerror = () => reject(tx.error);
|
|
3626
5056
|
request.onerror = () => reject(request.error);
|
|
3627
5057
|
});
|
|
@@ -3632,7 +5062,14 @@ class IndexedDbQueueStorage {
|
|
|
3632
5062
|
const store = tx.objectStore(this.tableName);
|
|
3633
5063
|
const request = store.get(id);
|
|
3634
5064
|
return new Promise((resolve, reject) => {
|
|
3635
|
-
request.onsuccess = () =>
|
|
5065
|
+
request.onsuccess = () => {
|
|
5066
|
+
const job = request.result;
|
|
5067
|
+
if (job && job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
5068
|
+
resolve(job);
|
|
5069
|
+
} else {
|
|
5070
|
+
resolve(undefined);
|
|
5071
|
+
}
|
|
5072
|
+
};
|
|
3636
5073
|
request.onerror = () => reject(request.error);
|
|
3637
5074
|
tx.onerror = () => reject(tx.error);
|
|
3638
5075
|
});
|
|
@@ -3641,10 +5078,11 @@ class IndexedDbQueueStorage {
|
|
|
3641
5078
|
const db = await this.getDb();
|
|
3642
5079
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3643
5080
|
const store = tx.objectStore(this.tableName);
|
|
3644
|
-
const index = store.index("
|
|
5081
|
+
const index = store.index("queue_status_run_after");
|
|
5082
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3645
5083
|
return new Promise((resolve, reject) => {
|
|
3646
5084
|
const ret = new Map;
|
|
3647
|
-
const keyRange = IDBKeyRange.bound([status, ""], [status, "\uFFFF"]);
|
|
5085
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, status, ""], [...prefixKeyValues, this.queueName, status, "\uFFFF"]);
|
|
3648
5086
|
const cursorRequest = index.openCursor(keyRange);
|
|
3649
5087
|
const handleCursor = (e) => {
|
|
3650
5088
|
const cursor = e.target.result;
|
|
@@ -3652,7 +5090,10 @@ class IndexedDbQueueStorage {
|
|
|
3652
5090
|
resolve(Array.from(ret.values()));
|
|
3653
5091
|
return;
|
|
3654
5092
|
}
|
|
3655
|
-
|
|
5093
|
+
const job = cursor.value;
|
|
5094
|
+
if (this.matchesPrefixes(job)) {
|
|
5095
|
+
ret.set(cursor.value.id, cursor.value);
|
|
5096
|
+
}
|
|
3656
5097
|
cursor.continue();
|
|
3657
5098
|
};
|
|
3658
5099
|
cursorRequest.onsuccess = handleCursor;
|
|
@@ -3660,14 +5101,15 @@ class IndexedDbQueueStorage {
|
|
|
3660
5101
|
tx.onerror = () => reject(tx.error);
|
|
3661
5102
|
});
|
|
3662
5103
|
}
|
|
3663
|
-
async next() {
|
|
5104
|
+
async next(workerId) {
|
|
3664
5105
|
const db = await this.getDb();
|
|
3665
5106
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3666
5107
|
const store = tx.objectStore(this.tableName);
|
|
3667
|
-
const index = store.index("
|
|
5108
|
+
const index = store.index("queue_status_run_after");
|
|
3668
5109
|
const now = new Date().toISOString();
|
|
5110
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3669
5111
|
return new Promise((resolve, reject) => {
|
|
3670
|
-
const cursorRequest = index.openCursor(IDBKeyRange.bound(["PENDING" /* PENDING */, ""], ["PENDING" /* PENDING */, now], false, true));
|
|
5112
|
+
const cursorRequest = index.openCursor(IDBKeyRange.bound([...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, ""], [...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, now], false, true));
|
|
3671
5113
|
let jobToReturn;
|
|
3672
5114
|
cursorRequest.onsuccess = (e) => {
|
|
3673
5115
|
const cursor = e.target.result;
|
|
@@ -3680,12 +5122,13 @@ class IndexedDbQueueStorage {
|
|
|
3680
5122
|
return;
|
|
3681
5123
|
}
|
|
3682
5124
|
const job = cursor.value;
|
|
3683
|
-
if (job.status !== "PENDING" /* PENDING */) {
|
|
5125
|
+
if (job.queue !== this.queueName || job.status !== "PENDING" /* PENDING */ || !this.matchesPrefixes(job)) {
|
|
3684
5126
|
cursor.continue();
|
|
3685
5127
|
return;
|
|
3686
5128
|
}
|
|
3687
5129
|
job.status = "PROCESSING" /* PROCESSING */;
|
|
3688
5130
|
job.last_ran_at = now;
|
|
5131
|
+
job.worker_id = workerId ?? null;
|
|
3689
5132
|
try {
|
|
3690
5133
|
const updateRequest = store.put(job);
|
|
3691
5134
|
updateRequest.onsuccess = () => {
|
|
@@ -3709,11 +5152,13 @@ class IndexedDbQueueStorage {
|
|
|
3709
5152
|
}
|
|
3710
5153
|
async size(status = "PENDING" /* PENDING */) {
|
|
3711
5154
|
const db = await this.getDb();
|
|
5155
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3712
5156
|
return new Promise((resolve, reject) => {
|
|
3713
5157
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3714
5158
|
const store = tx.objectStore(this.tableName);
|
|
3715
|
-
const index = store.index("
|
|
3716
|
-
const
|
|
5159
|
+
const index = store.index("queue_status");
|
|
5160
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
5161
|
+
const request = index.count(keyRange);
|
|
3717
5162
|
request.onsuccess = () => resolve(request.result);
|
|
3718
5163
|
request.onerror = () => reject(request.error);
|
|
3719
5164
|
tx.onerror = () => reject(tx.error);
|
|
@@ -3727,9 +5172,18 @@ class IndexedDbQueueStorage {
|
|
|
3727
5172
|
const getReq = store.get(job.id);
|
|
3728
5173
|
getReq.onsuccess = () => {
|
|
3729
5174
|
const existing = getReq.result;
|
|
3730
|
-
|
|
5175
|
+
if (!existing || existing.queue !== this.queueName || !this.matchesPrefixes(existing)) {
|
|
5176
|
+
reject(new Error(`Job ${job.id} not found or does not belong to queue ${this.queueName}`));
|
|
5177
|
+
return;
|
|
5178
|
+
}
|
|
5179
|
+
const currentAttempts = existing.run_attempts ?? 0;
|
|
3731
5180
|
job.run_attempts = currentAttempts + 1;
|
|
3732
|
-
|
|
5181
|
+
job.queue = this.queueName;
|
|
5182
|
+
const jobWithPrefixes = job;
|
|
5183
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5184
|
+
jobWithPrefixes[key] = value;
|
|
5185
|
+
}
|
|
5186
|
+
const putReq = store.put(jobWithPrefixes);
|
|
3733
5187
|
putReq.onsuccess = () => {};
|
|
3734
5188
|
putReq.onerror = () => reject(putReq.error);
|
|
3735
5189
|
};
|
|
@@ -3749,10 +5203,15 @@ class IndexedDbQueueStorage {
|
|
|
3749
5203
|
const db = await this.getDb();
|
|
3750
5204
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3751
5205
|
const store = tx.objectStore(this.tableName);
|
|
3752
|
-
const index = store.index("
|
|
3753
|
-
const
|
|
5206
|
+
const index = store.index("queue_job_run_id");
|
|
5207
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5208
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, job_run_id]);
|
|
5209
|
+
const request = index.getAll(keyRange);
|
|
3754
5210
|
return new Promise((resolve, reject) => {
|
|
3755
|
-
request.onsuccess = () =>
|
|
5211
|
+
request.onsuccess = () => {
|
|
5212
|
+
const results = (request.result || []).filter((job) => this.matchesPrefixes(job));
|
|
5213
|
+
resolve(results);
|
|
5214
|
+
};
|
|
3756
5215
|
request.onerror = () => reject(request.error);
|
|
3757
5216
|
tx.onerror = () => reject(tx.error);
|
|
3758
5217
|
});
|
|
@@ -3761,11 +5220,31 @@ class IndexedDbQueueStorage {
|
|
|
3761
5220
|
const db = await this.getDb();
|
|
3762
5221
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3763
5222
|
const store = tx.objectStore(this.tableName);
|
|
3764
|
-
const
|
|
5223
|
+
const index = store.index("queue_status");
|
|
5224
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
3765
5225
|
return new Promise((resolve, reject) => {
|
|
3766
|
-
|
|
3767
|
-
request
|
|
5226
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, "\uFFFF"]);
|
|
5227
|
+
const request = index.openCursor(keyRange);
|
|
5228
|
+
request.onsuccess = (event) => {
|
|
5229
|
+
const cursor = event.target.result;
|
|
5230
|
+
if (cursor) {
|
|
5231
|
+
const job = cursor.value;
|
|
5232
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
5233
|
+
const deleteRequest = cursor.delete();
|
|
5234
|
+
deleteRequest.onsuccess = () => {
|
|
5235
|
+
cursor.continue();
|
|
5236
|
+
};
|
|
5237
|
+
deleteRequest.onerror = () => {
|
|
5238
|
+
cursor.continue();
|
|
5239
|
+
};
|
|
5240
|
+
} else {
|
|
5241
|
+
cursor.continue();
|
|
5242
|
+
}
|
|
5243
|
+
}
|
|
5244
|
+
};
|
|
5245
|
+
tx.oncomplete = () => resolve();
|
|
3768
5246
|
tx.onerror = () => reject(tx.error);
|
|
5247
|
+
request.onerror = () => reject(request.error);
|
|
3769
5248
|
});
|
|
3770
5249
|
}
|
|
3771
5250
|
async outputForInput(input) {
|
|
@@ -3773,10 +5252,23 @@ class IndexedDbQueueStorage {
|
|
|
3773
5252
|
const db = await this.getDb();
|
|
3774
5253
|
const tx = db.transaction(this.tableName, "readonly");
|
|
3775
5254
|
const store = tx.objectStore(this.tableName);
|
|
3776
|
-
const index = store.index("
|
|
3777
|
-
const
|
|
5255
|
+
const index = store.index("queue_fingerprint_status");
|
|
5256
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5257
|
+
const request = index.get([
|
|
5258
|
+
...prefixKeyValues,
|
|
5259
|
+
this.queueName,
|
|
5260
|
+
fingerprint,
|
|
5261
|
+
"COMPLETED" /* COMPLETED */
|
|
5262
|
+
]);
|
|
3778
5263
|
return new Promise((resolve, reject) => {
|
|
3779
|
-
request.onsuccess = () =>
|
|
5264
|
+
request.onsuccess = () => {
|
|
5265
|
+
const job = request.result;
|
|
5266
|
+
if (job && this.matchesPrefixes(job)) {
|
|
5267
|
+
resolve(job.output ?? null);
|
|
5268
|
+
} else {
|
|
5269
|
+
resolve(null);
|
|
5270
|
+
}
|
|
5271
|
+
};
|
|
3780
5272
|
request.onerror = () => reject(request.error);
|
|
3781
5273
|
tx.onerror = () => reject(tx.error);
|
|
3782
5274
|
});
|
|
@@ -3791,6 +5283,9 @@ class IndexedDbQueueStorage {
|
|
|
3791
5283
|
await this.complete(job);
|
|
3792
5284
|
}
|
|
3793
5285
|
async delete(id) {
|
|
5286
|
+
const job = await this.get(id);
|
|
5287
|
+
if (!job)
|
|
5288
|
+
return;
|
|
3794
5289
|
const db = await this.getDb();
|
|
3795
5290
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3796
5291
|
const store = tx.objectStore(this.tableName);
|
|
@@ -3805,15 +5300,17 @@ class IndexedDbQueueStorage {
|
|
|
3805
5300
|
const db = await this.getDb();
|
|
3806
5301
|
const tx = db.transaction(this.tableName, "readwrite");
|
|
3807
5302
|
const store = tx.objectStore(this.tableName);
|
|
3808
|
-
const index = store.index("
|
|
5303
|
+
const index = store.index("queue_status");
|
|
3809
5304
|
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
5305
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5306
|
+
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
3810
5307
|
return new Promise((resolve, reject) => {
|
|
3811
|
-
const request = index.openCursor();
|
|
5308
|
+
const request = index.openCursor(keyRange);
|
|
3812
5309
|
request.onsuccess = (event) => {
|
|
3813
5310
|
const cursor = event.target.result;
|
|
3814
5311
|
if (cursor) {
|
|
3815
5312
|
const job = cursor.value;
|
|
3816
|
-
if (job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
|
|
5313
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
|
|
3817
5314
|
cursor.delete();
|
|
3818
5315
|
}
|
|
3819
5316
|
cursor.continue();
|
|
@@ -3824,6 +5321,363 @@ class IndexedDbQueueStorage {
|
|
|
3824
5321
|
request.onerror = () => reject(request.error);
|
|
3825
5322
|
});
|
|
3826
5323
|
}
|
|
5324
|
+
async getAllJobs() {
|
|
5325
|
+
const db = await this.getDb();
|
|
5326
|
+
const tx = db.transaction(this.tableName, "readonly");
|
|
5327
|
+
const store = tx.objectStore(this.tableName);
|
|
5328
|
+
const index = store.index("queue_status");
|
|
5329
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5330
|
+
return new Promise((resolve, reject) => {
|
|
5331
|
+
const jobs = [];
|
|
5332
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, "\uFFFF"]);
|
|
5333
|
+
const request = index.openCursor(keyRange);
|
|
5334
|
+
request.onsuccess = (event) => {
|
|
5335
|
+
const cursor = event.target.result;
|
|
5336
|
+
if (cursor) {
|
|
5337
|
+
const job = cursor.value;
|
|
5338
|
+
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
5339
|
+
jobs.push(job);
|
|
5340
|
+
}
|
|
5341
|
+
cursor.continue();
|
|
5342
|
+
}
|
|
5343
|
+
};
|
|
5344
|
+
tx.oncomplete = () => resolve(jobs);
|
|
5345
|
+
tx.onerror = () => reject(tx.error);
|
|
5346
|
+
request.onerror = () => reject(request.error);
|
|
5347
|
+
});
|
|
5348
|
+
}
|
|
5349
|
+
async getAllJobsWithFilter(prefixFilter) {
|
|
5350
|
+
const db = await this.getDb();
|
|
5351
|
+
const tx = db.transaction(this.tableName, "readonly");
|
|
5352
|
+
const store = tx.objectStore(this.tableName);
|
|
5353
|
+
return new Promise((resolve, reject) => {
|
|
5354
|
+
const jobs = [];
|
|
5355
|
+
const request = store.openCursor();
|
|
5356
|
+
request.onsuccess = (event) => {
|
|
5357
|
+
const cursor = event.target.result;
|
|
5358
|
+
if (cursor) {
|
|
5359
|
+
const job = cursor.value;
|
|
5360
|
+
if (job.queue !== this.queueName) {
|
|
5361
|
+
cursor.continue();
|
|
5362
|
+
return;
|
|
5363
|
+
}
|
|
5364
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
5365
|
+
jobs.push(job);
|
|
5366
|
+
} else {
|
|
5367
|
+
let matches = true;
|
|
5368
|
+
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
5369
|
+
if (job[key] !== value) {
|
|
5370
|
+
matches = false;
|
|
5371
|
+
break;
|
|
5372
|
+
}
|
|
5373
|
+
}
|
|
5374
|
+
if (matches) {
|
|
5375
|
+
jobs.push(job);
|
|
5376
|
+
}
|
|
5377
|
+
}
|
|
5378
|
+
cursor.continue();
|
|
5379
|
+
}
|
|
5380
|
+
};
|
|
5381
|
+
tx.oncomplete = () => resolve(jobs);
|
|
5382
|
+
tx.onerror = () => reject(tx.error);
|
|
5383
|
+
request.onerror = () => reject(request.error);
|
|
5384
|
+
});
|
|
5385
|
+
}
|
|
5386
|
+
isCustomPrefixFilter(prefixFilter) {
|
|
5387
|
+
if (prefixFilter === undefined) {
|
|
5388
|
+
return false;
|
|
5389
|
+
}
|
|
5390
|
+
if (Object.keys(prefixFilter).length === 0) {
|
|
5391
|
+
return true;
|
|
5392
|
+
}
|
|
5393
|
+
const instanceKeys = Object.keys(this.prefixValues);
|
|
5394
|
+
const filterKeys = Object.keys(prefixFilter);
|
|
5395
|
+
if (instanceKeys.length !== filterKeys.length) {
|
|
5396
|
+
return true;
|
|
5397
|
+
}
|
|
5398
|
+
for (const key of instanceKeys) {
|
|
5399
|
+
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
5400
|
+
return true;
|
|
5401
|
+
}
|
|
5402
|
+
}
|
|
5403
|
+
return false;
|
|
5404
|
+
}
|
|
5405
|
+
getPollingManager() {
|
|
5406
|
+
if (!this.pollingManager) {
|
|
5407
|
+
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
5408
|
+
const jobs = await this.getAllJobs();
|
|
5409
|
+
return new Map(jobs.map((j) => [j.id, j]));
|
|
5410
|
+
}, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
|
|
5411
|
+
insert: (item) => ({ type: "INSERT", new: item }),
|
|
5412
|
+
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
5413
|
+
delete: (item) => ({ type: "DELETE", old: item })
|
|
5414
|
+
});
|
|
5415
|
+
}
|
|
5416
|
+
return this.pollingManager;
|
|
5417
|
+
}
|
|
5418
|
+
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
5419
|
+
let lastKnownJobs = new Map;
|
|
5420
|
+
let cancelled = false;
|
|
5421
|
+
const poll = async () => {
|
|
5422
|
+
if (cancelled)
|
|
5423
|
+
return;
|
|
5424
|
+
try {
|
|
5425
|
+
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
5426
|
+
if (cancelled)
|
|
5427
|
+
return;
|
|
5428
|
+
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
5429
|
+
for (const [id, job] of currentMap) {
|
|
5430
|
+
const old = lastKnownJobs.get(id);
|
|
5431
|
+
if (!old) {
|
|
5432
|
+
callback({ type: "INSERT", new: job });
|
|
5433
|
+
} else if (JSON.stringify(old) !== JSON.stringify(job)) {
|
|
5434
|
+
callback({ type: "UPDATE", old, new: job });
|
|
5435
|
+
}
|
|
5436
|
+
}
|
|
5437
|
+
for (const [id, job] of lastKnownJobs) {
|
|
5438
|
+
if (!currentMap.has(id)) {
|
|
5439
|
+
callback({ type: "DELETE", old: job });
|
|
5440
|
+
}
|
|
5441
|
+
}
|
|
5442
|
+
lastKnownJobs = currentMap;
|
|
5443
|
+
} catch {}
|
|
5444
|
+
};
|
|
5445
|
+
const intervalId = setInterval(poll, intervalMs);
|
|
5446
|
+
poll();
|
|
5447
|
+
return () => {
|
|
5448
|
+
cancelled = true;
|
|
5449
|
+
clearInterval(intervalId);
|
|
5450
|
+
};
|
|
5451
|
+
}
|
|
5452
|
+
subscribeToChanges(callback, options) {
|
|
5453
|
+
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
5454
|
+
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
5455
|
+
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
5456
|
+
}
|
|
5457
|
+
const manager = this.getPollingManager();
|
|
5458
|
+
return manager.subscribe(callback, { intervalMs });
|
|
5459
|
+
}
|
|
5460
|
+
}
|
|
5461
|
+
// src/limiter/IndexedDbRateLimiterStorage.ts
|
|
5462
|
+
import { createServiceToken as createServiceToken28 } from "@workglow/util";
|
|
5463
|
+
var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken28("ratelimiter.storage.indexedDb");
|
|
5464
|
+
|
|
5465
|
+
class IndexedDbRateLimiterStorage {
|
|
5466
|
+
executionDb;
|
|
5467
|
+
nextAvailableDb;
|
|
5468
|
+
executionTableName;
|
|
5469
|
+
nextAvailableTableName;
|
|
5470
|
+
migrationOptions;
|
|
5471
|
+
prefixes;
|
|
5472
|
+
prefixValues;
|
|
5473
|
+
constructor(options = {}) {
|
|
5474
|
+
this.migrationOptions = options;
|
|
5475
|
+
this.prefixes = options.prefixes ?? [];
|
|
5476
|
+
this.prefixValues = options.prefixValues ?? {};
|
|
5477
|
+
if (this.prefixes.length > 0) {
|
|
5478
|
+
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
5479
|
+
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
5480
|
+
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
5481
|
+
} else {
|
|
5482
|
+
this.executionTableName = "rate_limit_executions";
|
|
5483
|
+
this.nextAvailableTableName = "rate_limit_next_available";
|
|
5484
|
+
}
|
|
5485
|
+
}
|
|
5486
|
+
getPrefixColumnNames() {
|
|
5487
|
+
return this.prefixes.map((p) => p.name);
|
|
5488
|
+
}
|
|
5489
|
+
matchesPrefixes(record) {
|
|
5490
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5491
|
+
if (record[key] !== value) {
|
|
5492
|
+
return false;
|
|
5493
|
+
}
|
|
5494
|
+
}
|
|
5495
|
+
return true;
|
|
5496
|
+
}
|
|
5497
|
+
getPrefixKeyValues() {
|
|
5498
|
+
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
5499
|
+
}
|
|
5500
|
+
async getExecutionDb() {
|
|
5501
|
+
if (this.executionDb)
|
|
5502
|
+
return this.executionDb;
|
|
5503
|
+
await this.setupDatabase();
|
|
5504
|
+
return this.executionDb;
|
|
5505
|
+
}
|
|
5506
|
+
async getNextAvailableDb() {
|
|
5507
|
+
if (this.nextAvailableDb)
|
|
5508
|
+
return this.nextAvailableDb;
|
|
5509
|
+
await this.setupDatabase();
|
|
5510
|
+
return this.nextAvailableDb;
|
|
5511
|
+
}
|
|
5512
|
+
async setupDatabase() {
|
|
5513
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
5514
|
+
const buildKeyPath = (basePath) => {
|
|
5515
|
+
return [...prefixColumnNames, ...basePath];
|
|
5516
|
+
};
|
|
5517
|
+
const executionIndexes = [
|
|
5518
|
+
{
|
|
5519
|
+
name: "queue_executed_at",
|
|
5520
|
+
keyPath: buildKeyPath(["queue_name", "executed_at"]),
|
|
5521
|
+
options: { unique: false }
|
|
5522
|
+
}
|
|
5523
|
+
];
|
|
5524
|
+
this.executionDb = await ensureIndexedDbTable(this.executionTableName, "id", executionIndexes, this.migrationOptions);
|
|
5525
|
+
const nextAvailableIndexes = [
|
|
5526
|
+
{
|
|
5527
|
+
name: "queue_name",
|
|
5528
|
+
keyPath: buildKeyPath(["queue_name"]),
|
|
5529
|
+
options: { unique: true }
|
|
5530
|
+
}
|
|
5531
|
+
];
|
|
5532
|
+
this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
|
|
5533
|
+
}
|
|
5534
|
+
async recordExecution(queueName) {
|
|
5535
|
+
const db = await this.getExecutionDb();
|
|
5536
|
+
const tx = db.transaction(this.executionTableName, "readwrite");
|
|
5537
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5538
|
+
const record = {
|
|
5539
|
+
id: crypto.randomUUID(),
|
|
5540
|
+
queue_name: queueName,
|
|
5541
|
+
executed_at: new Date().toISOString()
|
|
5542
|
+
};
|
|
5543
|
+
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5544
|
+
record[key] = value;
|
|
5545
|
+
}
|
|
5546
|
+
return new Promise((resolve, reject) => {
|
|
5547
|
+
const request = store.add(record);
|
|
5548
|
+
tx.oncomplete = () => resolve();
|
|
5549
|
+
tx.onerror = () => reject(tx.error);
|
|
5550
|
+
request.onerror = () => reject(request.error);
|
|
5551
|
+
});
|
|
5552
|
+
}
|
|
5553
|
+
async getExecutionCount(queueName, windowStartTime) {
|
|
5554
|
+
const db = await this.getExecutionDb();
|
|
5555
|
+
const tx = db.transaction(this.executionTableName, "readonly");
|
|
5556
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5557
|
+
const index = store.index("queue_executed_at");
|
|
5558
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5559
|
+
return new Promise((resolve, reject) => {
|
|
5560
|
+
let count = 0;
|
|
5561
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartTime], [...prefixKeyValues, queueName, "\uFFFF"], true, false);
|
|
5562
|
+
const request = index.openCursor(keyRange);
|
|
5563
|
+
request.onsuccess = (event) => {
|
|
5564
|
+
const cursor = event.target.result;
|
|
5565
|
+
if (cursor) {
|
|
5566
|
+
const record = cursor.value;
|
|
5567
|
+
if (this.matchesPrefixes(record)) {
|
|
5568
|
+
count++;
|
|
5569
|
+
}
|
|
5570
|
+
cursor.continue();
|
|
5571
|
+
}
|
|
5572
|
+
};
|
|
5573
|
+
tx.oncomplete = () => resolve(count);
|
|
5574
|
+
tx.onerror = () => reject(tx.error);
|
|
5575
|
+
request.onerror = () => reject(request.error);
|
|
5576
|
+
});
|
|
5577
|
+
}
|
|
5578
|
+
async getOldestExecutionAtOffset(queueName, offset) {
|
|
5579
|
+
const db = await this.getExecutionDb();
|
|
5580
|
+
const tx = db.transaction(this.executionTableName, "readonly");
|
|
5581
|
+
const store = tx.objectStore(this.executionTableName);
|
|
5582
|
+
const index = store.index("queue_executed_at");
|
|
5583
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5584
|
+
return new Promise((resolve, reject) => {
|
|
5585
|
+
const executions = [];
|
|
5586
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, "\uFFFF"]);
|
|
5587
|
+
const request = index.openCursor(keyRange);
|
|
5588
|
+
request.onsuccess = (event) => {
|
|
5589
|
+
const cursor = event.target.result;
|
|
5590
|
+
if (cursor) {
|
|
5591
|
+
const record = cursor.value;
|
|
5592
|
+
if (this.matchesPrefixes(record)) {
|
|
5593
|
+
executions.push(record.executed_at);
|
|
5594
|
+
}
|
|
5595
|
+
cursor.continue();
|
|
5596
|
+
}
|
|
5597
|
+
};
|
|
5598
|
+
tx.oncomplete = () => {
|
|
5599
|
+
executions.sort();
|
|
5600
|
+
resolve(executions[offset]);
|
|
5601
|
+
};
|
|
5602
|
+
tx.onerror = () => reject(tx.error);
|
|
5603
|
+
request.onerror = () => reject(request.error);
|
|
5604
|
+
});
|
|
5605
|
+
}
|
|
5606
|
+
async getNextAvailableTime(queueName) {
|
|
5607
|
+
const db = await this.getNextAvailableDb();
|
|
5608
|
+
const tx = db.transaction(this.nextAvailableTableName, "readonly");
|
|
5609
|
+
const store = tx.objectStore(this.nextAvailableTableName);
|
|
5610
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5611
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5612
|
+
return new Promise((resolve, reject) => {
|
|
5613
|
+
const request = store.get(key);
|
|
5614
|
+
request.onsuccess = () => {
|
|
5615
|
+
const record = request.result;
|
|
5616
|
+
if (record && this.matchesPrefixes(record)) {
|
|
5617
|
+
resolve(record.next_available_at);
|
|
5618
|
+
} else {
|
|
5619
|
+
resolve(undefined);
|
|
5620
|
+
}
|
|
5621
|
+
};
|
|
5622
|
+
request.onerror = () => reject(request.error);
|
|
5623
|
+
tx.onerror = () => reject(tx.error);
|
|
5624
|
+
});
|
|
5625
|
+
}
|
|
5626
|
+
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
5627
|
+
const db = await this.getNextAvailableDb();
|
|
5628
|
+
const tx = db.transaction(this.nextAvailableTableName, "readwrite");
|
|
5629
|
+
const store = tx.objectStore(this.nextAvailableTableName);
|
|
5630
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5631
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5632
|
+
const record = {
|
|
5633
|
+
queue_name: queueName,
|
|
5634
|
+
next_available_at: nextAvailableAt
|
|
5635
|
+
};
|
|
5636
|
+
for (const [k, value] of Object.entries(this.prefixValues)) {
|
|
5637
|
+
record[k] = value;
|
|
5638
|
+
}
|
|
5639
|
+
record[this.getPrefixColumnNames().concat(["queue_name"]).join("_")] = key;
|
|
5640
|
+
return new Promise((resolve, reject) => {
|
|
5641
|
+
const request = store.put(record);
|
|
5642
|
+
tx.oncomplete = () => resolve();
|
|
5643
|
+
tx.onerror = () => reject(tx.error);
|
|
5644
|
+
request.onerror = () => reject(request.error);
|
|
5645
|
+
});
|
|
5646
|
+
}
|
|
5647
|
+
async clear(queueName) {
|
|
5648
|
+
const execDb = await this.getExecutionDb();
|
|
5649
|
+
const execTx = execDb.transaction(this.executionTableName, "readwrite");
|
|
5650
|
+
const execStore = execTx.objectStore(this.executionTableName);
|
|
5651
|
+
const execIndex = execStore.index("queue_executed_at");
|
|
5652
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5653
|
+
await new Promise((resolve, reject) => {
|
|
5654
|
+
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, "\uFFFF"]);
|
|
5655
|
+
const request = execIndex.openCursor(keyRange);
|
|
5656
|
+
request.onsuccess = (event) => {
|
|
5657
|
+
const cursor = event.target.result;
|
|
5658
|
+
if (cursor) {
|
|
5659
|
+
const record = cursor.value;
|
|
5660
|
+
if (this.matchesPrefixes(record)) {
|
|
5661
|
+
cursor.delete();
|
|
5662
|
+
}
|
|
5663
|
+
cursor.continue();
|
|
5664
|
+
}
|
|
5665
|
+
};
|
|
5666
|
+
execTx.oncomplete = () => resolve();
|
|
5667
|
+
execTx.onerror = () => reject(execTx.error);
|
|
5668
|
+
request.onerror = () => reject(request.error);
|
|
5669
|
+
});
|
|
5670
|
+
const nextDb = await this.getNextAvailableDb();
|
|
5671
|
+
const nextTx = nextDb.transaction(this.nextAvailableTableName, "readwrite");
|
|
5672
|
+
const nextStore = nextTx.objectStore(this.nextAvailableTableName);
|
|
5673
|
+
const key = [...prefixKeyValues, queueName].join("_");
|
|
5674
|
+
await new Promise((resolve, reject) => {
|
|
5675
|
+
const request = nextStore.delete(key);
|
|
5676
|
+
nextTx.oncomplete = () => resolve();
|
|
5677
|
+
nextTx.onerror = () => reject(nextTx.error);
|
|
5678
|
+
request.onerror = () => reject(request.error);
|
|
5679
|
+
});
|
|
5680
|
+
}
|
|
3827
5681
|
}
|
|
3828
5682
|
export {
|
|
3829
5683
|
ensureIndexedDbTable,
|
|
@@ -3831,22 +5685,29 @@ export {
|
|
|
3831
5685
|
TabularRepository,
|
|
3832
5686
|
TABULAR_REPOSITORY,
|
|
3833
5687
|
SupabaseTabularRepository,
|
|
5688
|
+
SupabaseRateLimiterStorage,
|
|
3834
5689
|
SupabaseQueueStorage,
|
|
3835
5690
|
SupabaseKvRepository,
|
|
3836
5691
|
SqliteTabularRepository,
|
|
5692
|
+
SqliteRateLimiterStorage,
|
|
3837
5693
|
SqliteQueueStorage,
|
|
3838
5694
|
SqliteKvRepository,
|
|
3839
5695
|
SUPABASE_TABULAR_REPOSITORY,
|
|
5696
|
+
SUPABASE_RATE_LIMITER_STORAGE,
|
|
3840
5697
|
SUPABASE_QUEUE_STORAGE,
|
|
3841
5698
|
SUPABASE_KV_REPOSITORY,
|
|
3842
5699
|
SQLITE_TABULAR_REPOSITORY,
|
|
5700
|
+
SQLITE_RATE_LIMITER_STORAGE,
|
|
3843
5701
|
SQLITE_QUEUE_STORAGE,
|
|
3844
5702
|
SQLITE_KV_REPOSITORY,
|
|
5703
|
+
RATE_LIMITER_STORAGE,
|
|
3845
5704
|
QUEUE_STORAGE,
|
|
3846
5705
|
PostgresTabularRepository,
|
|
5706
|
+
PostgresRateLimiterStorage,
|
|
3847
5707
|
PostgresQueueStorage,
|
|
3848
5708
|
PostgresKvRepository,
|
|
3849
5709
|
POSTGRES_TABULAR_REPOSITORY,
|
|
5710
|
+
POSTGRES_RATE_LIMITER_STORAGE,
|
|
3850
5711
|
POSTGRES_QUEUE_STORAGE,
|
|
3851
5712
|
POSTGRES_KV_REPOSITORY,
|
|
3852
5713
|
MEMORY_TABULAR_REPOSITORY,
|
|
@@ -3856,12 +5717,16 @@ export {
|
|
|
3856
5717
|
KV_REPOSITORY,
|
|
3857
5718
|
JobStatus,
|
|
3858
5719
|
IndexedDbTabularRepository,
|
|
5720
|
+
IndexedDbRateLimiterStorage,
|
|
3859
5721
|
IndexedDbQueueStorage,
|
|
3860
5722
|
IndexedDbKvRepository,
|
|
3861
5723
|
InMemoryTabularRepository,
|
|
5724
|
+
InMemoryRateLimiterStorage,
|
|
3862
5725
|
InMemoryQueueStorage,
|
|
3863
5726
|
InMemoryKvRepository,
|
|
5727
|
+
IN_MEMORY_RATE_LIMITER_STORAGE,
|
|
3864
5728
|
IN_MEMORY_QUEUE_STORAGE,
|
|
5729
|
+
INDEXED_DB_RATE_LIMITER_STORAGE,
|
|
3865
5730
|
INDEXED_DB_QUEUE_STORAGE,
|
|
3866
5731
|
IDB_TABULAR_REPOSITORY,
|
|
3867
5732
|
IDB_KV_REPOSITORY,
|
|
@@ -3877,4 +5742,4 @@ export {
|
|
|
3877
5742
|
CACHED_TABULAR_REPOSITORY
|
|
3878
5743
|
};
|
|
3879
5744
|
|
|
3880
|
-
//# debugId=
|
|
5745
|
+
//# debugId=69EE3CC6572AC11564756E2164756E21
|