@workglow/storage 0.0.57 → 0.0.58

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/browser.js +1266 -121
  2. package/dist/browser.js.map +15 -10
  3. package/dist/bun.js +2017 -252
  4. package/dist/bun.js.map +19 -12
  5. package/dist/common-server.d.ts +4 -0
  6. package/dist/common-server.d.ts.map +1 -1
  7. package/dist/common.d.ts +2 -0
  8. package/dist/common.d.ts.map +1 -1
  9. package/dist/limiter/IRateLimiterStorage.d.ts +81 -0
  10. package/dist/limiter/IRateLimiterStorage.d.ts.map +1 -0
  11. package/dist/limiter/InMemoryRateLimiterStorage.d.ts +32 -0
  12. package/dist/limiter/InMemoryRateLimiterStorage.d.ts.map +1 -0
  13. package/dist/limiter/IndexedDbRateLimiterStorage.d.ts +52 -0
  14. package/dist/limiter/IndexedDbRateLimiterStorage.d.ts.map +1 -0
  15. package/dist/limiter/PostgresRateLimiterStorage.d.ts +54 -0
  16. package/dist/limiter/PostgresRateLimiterStorage.d.ts.map +1 -0
  17. package/dist/limiter/SqliteRateLimiterStorage.d.ts +53 -0
  18. package/dist/limiter/SqliteRateLimiterStorage.d.ts.map +1 -0
  19. package/dist/limiter/SupabaseRateLimiterStorage.d.ts +53 -0
  20. package/dist/limiter/SupabaseRateLimiterStorage.d.ts.map +1 -0
  21. package/dist/node.js +2017 -252
  22. package/dist/node.js.map +19 -12
  23. package/dist/queue/IQueueStorage.d.ts +72 -1
  24. package/dist/queue/IQueueStorage.d.ts.map +1 -1
  25. package/dist/queue/InMemoryQueueStorage.d.ts +44 -11
  26. package/dist/queue/InMemoryQueueStorage.d.ts.map +1 -1
  27. package/dist/queue/IndexedDbQueueStorage.d.ts +70 -5
  28. package/dist/queue/IndexedDbQueueStorage.d.ts.map +1 -1
  29. package/dist/queue/PostgresQueueStorage.d.ts +80 -8
  30. package/dist/queue/PostgresQueueStorage.d.ts.map +1 -1
  31. package/dist/queue/SqliteQueueStorage.d.ts +90 -34
  32. package/dist/queue/SqliteQueueStorage.d.ts.map +1 -1
  33. package/dist/queue/SupabaseQueueStorage.d.ts +98 -4
  34. package/dist/queue/SupabaseQueueStorage.d.ts.map +1 -1
  35. package/dist/tabular/ITabularRepository.d.ts +18 -0
  36. package/dist/tabular/ITabularRepository.d.ts.map +1 -1
  37. package/dist/tabular/InMemoryTabularRepository.d.ts +9 -1
  38. package/dist/tabular/InMemoryTabularRepository.d.ts.map +1 -1
  39. package/dist/tabular/SupabaseTabularRepository.d.ts +21 -1
  40. package/dist/tabular/SupabaseTabularRepository.d.ts.map +1 -1
  41. package/dist/tabular/TabularRepository.d.ts +10 -1
  42. package/dist/tabular/TabularRepository.d.ts.map +1 -1
  43. package/dist/util/PollingSubscriptionManager.d.ts +112 -0
  44. package/dist/util/PollingSubscriptionManager.d.ts.map +1 -0
  45. package/package.json +5 -5
package/dist/bun.js CHANGED
@@ -107,6 +107,9 @@ class TabularRepository {
107
107
  waitOn(name) {
108
108
  return this.events.waitOn(name);
109
109
  }
110
+ subscribeToChanges(_callback) {
111
+ throw new Error(`subscribeToChanges is not implemented for ${this.constructor.name}. ` + `Use InMemoryTabularRepository or SupabaseTabularRepository for subscription support.`);
112
+ }
110
113
  primaryKeyColumns() {
111
114
  const columns = [];
112
115
  for (const key of Object.keys(this.primaryKeySchema.properties)) {
@@ -274,12 +277,30 @@ class InMemoryTabularRepository extends TabularRepository {
274
277
  return false;
275
278
  }
276
279
  });
277
- for (const [id, _] of entriesToDelete) {
280
+ for (const [id, entity] of entriesToDelete) {
278
281
  this.values.delete(id);
282
+ const { key } = this.separateKeyValueFromCombined(entity);
283
+ this.events.emit("delete", key);
279
284
  }
280
- if (Array.from(entriesToDelete).length > 0) {
281
- this.events.emit("delete", column);
282
- }
285
+ }
286
+ subscribeToChanges(callback) {
287
+ const handlePut = (entity) => {
288
+ callback({ type: "UPDATE", new: entity });
289
+ };
290
+ const handleDelete = (_key) => {
291
+ callback({ type: "DELETE" });
292
+ };
293
+ const handleClearAll = () => {
294
+ callback({ type: "DELETE" });
295
+ };
296
+ this.events.on("put", handlePut);
297
+ this.events.on("delete", handleDelete);
298
+ this.events.on("clearall", handleClearAll);
299
+ return () => {
300
+ this.events.off("put", handlePut);
301
+ this.events.off("delete", handleDelete);
302
+ this.events.off("clearall", handleClearAll);
303
+ };
283
304
  }
284
305
  destroy() {
285
306
  this.values.clear();
@@ -541,7 +562,7 @@ class InMemoryKvRepository extends KvViaTabularRepository {
541
562
  }
542
563
  }
543
564
  // src/queue/InMemoryQueueStorage.ts
544
- import { createServiceToken as createServiceToken7, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
565
+ import { createServiceToken as createServiceToken7, EventEmitter as EventEmitter3, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
545
566
 
546
567
  // src/queue/IQueueStorage.ts
547
568
  import { createServiceToken as createServiceToken6 } from "@workglow/util";
@@ -561,63 +582,88 @@ var IN_MEMORY_QUEUE_STORAGE = createServiceToken7("jobqueue.storage.inMemory");
561
582
 
562
583
  class InMemoryQueueStorage {
563
584
  queueName;
564
- constructor(queueName) {
585
+ prefixValues;
586
+ events = new EventEmitter3;
587
+ constructor(queueName, options) {
565
588
  this.queueName = queueName;
566
589
  this.jobQueue = [];
590
+ this.prefixValues = options?.prefixValues ?? {};
567
591
  }
568
592
  jobQueue;
593
+ matchesPrefixes(job) {
594
+ for (const [key, value] of Object.entries(this.prefixValues)) {
595
+ if (job[key] !== value) {
596
+ return false;
597
+ }
598
+ }
599
+ return true;
600
+ }
569
601
  pendingQueue() {
570
602
  const now = new Date().toISOString();
571
- return this.jobQueue.filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
603
+ return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
572
604
  }
573
605
  async add(job) {
574
606
  await sleep(0);
575
607
  const now = new Date().toISOString();
576
- job.id = job.id ?? uuid4();
577
- job.job_run_id = job.job_run_id ?? uuid4();
578
- job.queue = this.queueName;
579
- job.fingerprint = await makeFingerprint4(job.input);
580
- job.status = "PENDING" /* PENDING */;
581
- job.progress = 0;
582
- job.progress_message = "";
583
- job.progress_details = null;
584
- job.created_at = now;
585
- job.run_after = now;
586
- this.jobQueue.push(job);
587
- return job.id;
608
+ const jobWithPrefixes = job;
609
+ jobWithPrefixes.id = jobWithPrefixes.id ?? uuid4();
610
+ jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid4();
611
+ jobWithPrefixes.queue = this.queueName;
612
+ jobWithPrefixes.fingerprint = await makeFingerprint4(jobWithPrefixes.input);
613
+ jobWithPrefixes.status = "PENDING" /* PENDING */;
614
+ jobWithPrefixes.progress = 0;
615
+ jobWithPrefixes.progress_message = "";
616
+ jobWithPrefixes.progress_details = null;
617
+ jobWithPrefixes.created_at = now;
618
+ jobWithPrefixes.run_after = now;
619
+ for (const [key, value] of Object.entries(this.prefixValues)) {
620
+ jobWithPrefixes[key] = value;
621
+ }
622
+ this.jobQueue.push(jobWithPrefixes);
623
+ this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
624
+ return jobWithPrefixes.id;
588
625
  }
589
626
  async get(id) {
590
627
  await sleep(0);
591
- return this.jobQueue.find((j) => j.id === id);
628
+ const job = this.jobQueue.find((j) => j.id === id);
629
+ if (job && this.matchesPrefixes(job)) {
630
+ return job;
631
+ }
632
+ return;
592
633
  }
593
634
  async peek(status = "PENDING" /* PENDING */, num = 100) {
594
635
  await sleep(0);
595
636
  num = Number(num) || 100;
596
- return this.jobQueue.sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
637
+ return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
597
638
  }
598
- async next() {
639
+ async next(workerId) {
599
640
  await sleep(0);
600
641
  const top = this.pendingQueue();
601
642
  const job = top[0];
602
643
  if (job) {
644
+ const oldJob = { ...job };
603
645
  job.status = "PROCESSING" /* PROCESSING */;
604
646
  job.last_ran_at = new Date().toISOString();
647
+ job.worker_id = workerId ?? null;
648
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
605
649
  return job;
606
650
  }
607
651
  }
608
652
  async size(status = "PENDING" /* PENDING */) {
609
653
  await sleep(0);
610
- return this.jobQueue.filter((j) => j.status === status).length;
654
+ return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
611
655
  }
612
656
  async saveProgress(id, progress, message, details) {
613
657
  await sleep(0);
614
- const job = this.jobQueue.find((j) => j.id === id);
658
+ const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
615
659
  if (!job) {
616
660
  throw new Error(`Job ${id} not found`);
617
661
  }
662
+ const oldJob = { ...job };
618
663
  job.progress = progress;
619
664
  job.progress_message = message;
620
665
  job.progress_details = details;
666
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
621
667
  }
622
668
  async complete(job) {
623
669
  await sleep(0);
@@ -627,44 +673,149 @@ class InMemoryQueueStorage {
627
673
  const currentAttempts = existing?.run_attempts ?? 0;
628
674
  job.run_attempts = currentAttempts + 1;
629
675
  this.jobQueue[index] = job;
676
+ this.events.emit("change", { type: "UPDATE", old: existing, new: job });
630
677
  }
631
678
  }
632
679
  async abort(id) {
633
680
  await sleep(0);
634
- const job = this.jobQueue.find((j) => j.id === id);
681
+ const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
635
682
  if (job) {
683
+ const oldJob = { ...job };
636
684
  job.status = "ABORTING" /* ABORTING */;
685
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
637
686
  }
638
687
  }
639
688
  async getByRunId(runId) {
640
689
  await sleep(0);
641
- return this.jobQueue.filter((job) => job.job_run_id === runId);
690
+ return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
642
691
  }
643
692
  async deleteAll() {
644
693
  await sleep(0);
645
- this.jobQueue = [];
694
+ const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
695
+ this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
696
+ for (const job of deletedJobs) {
697
+ this.events.emit("change", { type: "DELETE", old: job });
698
+ }
646
699
  }
647
700
  async outputForInput(input) {
648
701
  await sleep(0);
649
702
  const fingerprint = await makeFingerprint4(input);
650
- return this.jobQueue.find((j) => j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
703
+ return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
651
704
  }
652
705
  async delete(id) {
653
706
  await sleep(0);
654
- this.jobQueue = this.jobQueue.filter((job) => job.id !== id);
707
+ const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
708
+ this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
709
+ if (deletedJob) {
710
+ this.events.emit("change", { type: "DELETE", old: deletedJob });
711
+ }
655
712
  }
656
713
  async deleteJobsByStatusAndAge(status, olderThanMs) {
657
714
  await sleep(0);
658
715
  const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
659
- this.jobQueue = this.jobQueue.filter((job) => job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
716
+ const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
717
+ this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
718
+ for (const job of deletedJobs) {
719
+ this.events.emit("change", { type: "DELETE", old: job });
720
+ }
721
+ }
722
+ async setupDatabase() {}
723
+ matchesPrefixFilter(job, prefixFilter) {
724
+ if (prefixFilter && Object.keys(prefixFilter).length === 0) {
725
+ return true;
726
+ }
727
+ const filterValues = prefixFilter ?? this.prefixValues;
728
+ if (Object.keys(filterValues).length === 0) {
729
+ return true;
730
+ }
731
+ const jobWithPrefixes = job;
732
+ for (const [key, value] of Object.entries(filterValues)) {
733
+ if (jobWithPrefixes[key] !== value) {
734
+ return false;
735
+ }
736
+ }
737
+ return true;
738
+ }
739
+ subscribeToChanges(callback, options) {
740
+ const prefixFilter = options?.prefixFilter;
741
+ const filteredCallback = (change) => {
742
+ const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
743
+ const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
744
+ if (!newMatches && !oldMatches) {
745
+ return;
746
+ }
747
+ callback(change);
748
+ };
749
+ return this.events.subscribe("change", filteredCallback);
750
+ }
751
+ }
752
+ // src/limiter/IRateLimiterStorage.ts
753
+ import { createServiceToken as createServiceToken8 } from "@workglow/util";
754
+ var RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage");
755
+ // src/limiter/InMemoryRateLimiterStorage.ts
756
+ import { createServiceToken as createServiceToken9, sleep as sleep2 } from "@workglow/util";
757
+ var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken9("ratelimiter.storage.inMemory");
758
+
759
+ class InMemoryRateLimiterStorage {
760
+ prefixValues;
761
+ executions = new Map;
762
+ nextAvailableTimes = new Map;
763
+ constructor(options) {
764
+ this.prefixValues = options?.prefixValues ?? {};
765
+ }
766
+ makeKey(queueName) {
767
+ const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
768
+ return prefixPart ? `${prefixPart}|${queueName}` : queueName;
660
769
  }
661
770
  async setupDatabase() {}
771
+ async recordExecution(queueName) {
772
+ await sleep2(0);
773
+ const key = this.makeKey(queueName);
774
+ const executions = this.executions.get(key) ?? [];
775
+ executions.push({
776
+ queueName,
777
+ executedAt: new Date
778
+ });
779
+ this.executions.set(key, executions);
780
+ }
781
+ async getExecutionCount(queueName, windowStartTime) {
782
+ await sleep2(0);
783
+ const key = this.makeKey(queueName);
784
+ const executions = this.executions.get(key) ?? [];
785
+ const windowStart = new Date(windowStartTime);
786
+ return executions.filter((e) => e.executedAt > windowStart).length;
787
+ }
788
+ async getOldestExecutionAtOffset(queueName, offset) {
789
+ await sleep2(0);
790
+ const key = this.makeKey(queueName);
791
+ const executions = this.executions.get(key) ?? [];
792
+ const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
793
+ const execution = sorted[offset];
794
+ return execution?.executedAt.toISOString();
795
+ }
796
+ async getNextAvailableTime(queueName) {
797
+ await sleep2(0);
798
+ const key = this.makeKey(queueName);
799
+ const time = this.nextAvailableTimes.get(key);
800
+ return time?.toISOString();
801
+ }
802
+ async setNextAvailableTime(queueName, nextAvailableAt) {
803
+ await sleep2(0);
804
+ const key = this.makeKey(queueName);
805
+ this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
806
+ }
807
+ async clear(queueName) {
808
+ await sleep2(0);
809
+ const key = this.makeKey(queueName);
810
+ this.executions.delete(key);
811
+ this.nextAvailableTimes.delete(key);
812
+ }
662
813
  }
663
814
  // src/tabular/FsFolderTabularRepository.ts
664
- import { createServiceToken as createServiceToken8, sleep as sleep2 } from "@workglow/util";
815
+ import { createServiceToken as createServiceToken10, sleep as sleep3 } from "@workglow/util";
665
816
  import { mkdir, readdir, readFile, rm, writeFile } from "fs/promises";
666
817
  import path from "path";
667
- var FS_FOLDER_TABULAR_REPOSITORY = createServiceToken8("storage.tabularRepository.fsFolder");
818
+ var FS_FOLDER_TABULAR_REPOSITORY = createServiceToken10("storage.tabularRepository.fsFolder");
668
819
 
669
820
  class FsFolderTabularRepository extends TabularRepository {
670
821
  folderPath;
@@ -689,7 +840,7 @@ class FsFolderTabularRepository extends TabularRepository {
689
840
  await writeFile(filePath, JSON.stringify(entity));
690
841
  } catch (error) {
691
842
  try {
692
- await sleep2(1);
843
+ await sleep3(1);
693
844
  await writeFile(filePath, JSON.stringify(entity));
694
845
  } catch (error2) {
695
846
  console.error("Error writing file", filePath, error2);
@@ -773,7 +924,7 @@ class FsFolderTabularRepository extends TabularRepository {
773
924
  }
774
925
  }
775
926
  // src/tabular/PostgresTabularRepository.ts
776
- import { createServiceToken as createServiceToken9 } from "@workglow/util";
927
+ import { createServiceToken as createServiceToken11 } from "@workglow/util";
777
928
 
778
929
  // src/tabular/BaseSqlTabularRepository.ts
779
930
  class BaseSqlTabularRepository extends TabularRepository {
@@ -959,7 +1110,7 @@ class BaseSqlTabularRepository extends TabularRepository {
959
1110
  }
960
1111
 
961
1112
  // src/tabular/PostgresTabularRepository.ts
962
- var POSTGRES_TABULAR_REPOSITORY = createServiceToken9("storage.tabularRepository.postgres");
1113
+ var POSTGRES_TABULAR_REPOSITORY = createServiceToken11("storage.tabularRepository.postgres");
963
1114
 
964
1115
  class PostgresTabularRepository extends BaseSqlTabularRepository {
965
1116
  db;
@@ -1317,8 +1468,8 @@ class PostgresTabularRepository extends BaseSqlTabularRepository {
1317
1468
  }
1318
1469
  // src/tabular/SqliteTabularRepository.ts
1319
1470
  import { Sqlite } from "@workglow/sqlite";
1320
- import { createServiceToken as createServiceToken10 } from "@workglow/util";
1321
- var SQLITE_TABULAR_REPOSITORY = createServiceToken10("storage.tabularRepository.sqlite");
1471
+ import { createServiceToken as createServiceToken12 } from "@workglow/util";
1472
+ var SQLITE_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.sqlite");
1322
1473
  var Database = Sqlite.Database;
1323
1474
 
1324
1475
  class SqliteTabularRepository extends BaseSqlTabularRepository {
@@ -1595,11 +1746,12 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
1595
1746
  }
1596
1747
  }
1597
1748
  // src/tabular/SupabaseTabularRepository.ts
1598
- import { createServiceToken as createServiceToken11 } from "@workglow/util";
1599
- var SUPABASE_TABULAR_REPOSITORY = createServiceToken11("storage.tabularRepository.supabase");
1749
+ import { createServiceToken as createServiceToken13 } from "@workglow/util";
1750
+ var SUPABASE_TABULAR_REPOSITORY = createServiceToken13("storage.tabularRepository.supabase");
1600
1751
 
1601
1752
  class SupabaseTabularRepository extends BaseSqlTabularRepository {
1602
1753
  client;
1754
+ realtimeChannel = null;
1603
1755
  constructor(client, table = "tabular_store", schema, primaryKeyNames, indexes = []) {
1604
1756
  super(table, schema, primaryKeyNames, indexes);
1605
1757
  this.client = client;
@@ -1972,10 +2124,44 @@ class SupabaseTabularRepository extends BaseSqlTabularRepository {
1972
2124
  throw error;
1973
2125
  this.events.emit("delete", column);
1974
2126
  }
2127
+ convertRealtimeRow(row) {
2128
+ const entity = { ...row };
2129
+ for (const key in this.schema.properties) {
2130
+ entity[key] = this.sqlToJsValue(key, row[key]);
2131
+ }
2132
+ return entity;
2133
+ }
2134
+ subscribeToChanges(callback) {
2135
+ const channelName = `tabular-${this.table}-${Date.now()}`;
2136
+ this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
2137
+ event: "*",
2138
+ schema: "public",
2139
+ table: this.table
2140
+ }, (payload) => {
2141
+ const change = {
2142
+ type: payload.eventType.toUpperCase(),
2143
+ old: payload.old && Object.keys(payload.old).length > 0 ? this.convertRealtimeRow(payload.old) : undefined,
2144
+ new: payload.new && Object.keys(payload.new).length > 0 ? this.convertRealtimeRow(payload.new) : undefined
2145
+ };
2146
+ callback(change);
2147
+ }).subscribe();
2148
+ return () => {
2149
+ if (this.realtimeChannel) {
2150
+ this.client.removeChannel(this.realtimeChannel);
2151
+ this.realtimeChannel = null;
2152
+ }
2153
+ };
2154
+ }
2155
+ destroy() {
2156
+ if (this.realtimeChannel) {
2157
+ this.client.removeChannel(this.realtimeChannel);
2158
+ this.realtimeChannel = null;
2159
+ }
2160
+ }
1975
2161
  }
1976
2162
  // src/kv/FsFolderJsonKvRepository.ts
1977
- import { createServiceToken as createServiceToken12 } from "@workglow/util";
1978
- var FS_FOLDER_JSON_KV_REPOSITORY = createServiceToken12("storage.kvRepository.fsFolderJson");
2163
+ import { createServiceToken as createServiceToken14 } from "@workglow/util";
2164
+ var FS_FOLDER_JSON_KV_REPOSITORY = createServiceToken14("storage.kvRepository.fsFolderJson");
1979
2165
 
1980
2166
  class FsFolderJsonKvRepository extends KvViaTabularRepository {
1981
2167
  folderPath;
@@ -1987,10 +2173,10 @@ class FsFolderJsonKvRepository extends KvViaTabularRepository {
1987
2173
  }
1988
2174
  }
1989
2175
  // src/kv/FsFolderKvRepository.ts
1990
- import { createServiceToken as createServiceToken13 } from "@workglow/util";
2176
+ import { createServiceToken as createServiceToken15 } from "@workglow/util";
1991
2177
  import { mkdir as mkdir2, readFile as readFile2, rm as rm2, unlink, writeFile as writeFile2 } from "fs/promises";
1992
2178
  import path2 from "path";
1993
- var FS_FOLDER_KV_REPOSITORY = createServiceToken13("storage.kvRepository.fsFolder");
2179
+ var FS_FOLDER_KV_REPOSITORY = createServiceToken15("storage.kvRepository.fsFolder");
1994
2180
 
1995
2181
  class FsFolderKvRepository extends KvRepository {
1996
2182
  folderPath;
@@ -2067,8 +2253,8 @@ class FsFolderKvRepository extends KvRepository {
2067
2253
  }
2068
2254
  }
2069
2255
  // src/kv/PostgresKvRepository.ts
2070
- import { createServiceToken as createServiceToken14 } from "@workglow/util";
2071
- var POSTGRES_KV_REPOSITORY = createServiceToken14("storage.kvRepository.postgres");
2256
+ import { createServiceToken as createServiceToken16 } from "@workglow/util";
2257
+ var POSTGRES_KV_REPOSITORY = createServiceToken16("storage.kvRepository.postgres");
2072
2258
 
2073
2259
  class PostgresKvRepository extends KvViaTabularRepository {
2074
2260
  db;
@@ -2082,8 +2268,8 @@ class PostgresKvRepository extends KvViaTabularRepository {
2082
2268
  }
2083
2269
  }
2084
2270
  // src/kv/SqliteKvRepository.ts
2085
- import { createServiceToken as createServiceToken15 } from "@workglow/util";
2086
- var SQLITE_KV_REPOSITORY = createServiceToken15("storage.kvRepository.sqlite");
2271
+ import { createServiceToken as createServiceToken17 } from "@workglow/util";
2272
+ var SQLITE_KV_REPOSITORY = createServiceToken17("storage.kvRepository.sqlite");
2087
2273
 
2088
2274
  class SqliteKvRepository extends KvViaTabularRepository {
2089
2275
  db;
@@ -2097,8 +2283,8 @@ class SqliteKvRepository extends KvViaTabularRepository {
2097
2283
  }
2098
2284
  }
2099
2285
  // src/kv/SupabaseKvRepository.ts
2100
- import { createServiceToken as createServiceToken16 } from "@workglow/util";
2101
- var SUPABASE_KV_REPOSITORY = createServiceToken16("storage.kvRepository.supabase");
2286
+ import { createServiceToken as createServiceToken18 } from "@workglow/util";
2287
+ var SUPABASE_KV_REPOSITORY = createServiceToken18("storage.kvRepository.supabase");
2102
2288
 
2103
2289
  class SupabaseKvRepository extends KvViaTabularRepository {
2104
2290
  client;
@@ -2112,15 +2298,169 @@ class SupabaseKvRepository extends KvViaTabularRepository {
2112
2298
  }
2113
2299
  }
2114
2300
  // src/queue/PostgresQueueStorage.ts
2115
- import { createServiceToken as createServiceToken17, makeFingerprint as makeFingerprint5, uuid4 as uuid42 } from "@workglow/util";
2116
- var POSTGRES_QUEUE_STORAGE = createServiceToken17("jobqueue.storage.postgres");
2301
+ import { createServiceToken as createServiceToken19, makeFingerprint as makeFingerprint5, uuid4 as uuid42 } from "@workglow/util";
2302
+
2303
+ // src/util/PollingSubscriptionManager.ts
2304
+ class PollingSubscriptionManager {
2305
+ intervals = new Map;
2306
+ lastKnownState = new Map;
2307
+ initialized = false;
2308
+ fetchState;
2309
+ compareItems;
2310
+ payloadFactory;
2311
+ defaultIntervalMs;
2312
+ constructor(fetchState, compareItems, payloadFactory, options) {
2313
+ this.fetchState = fetchState;
2314
+ this.compareItems = compareItems;
2315
+ this.payloadFactory = payloadFactory;
2316
+ this.defaultIntervalMs = options?.defaultIntervalMs ?? 1000;
2317
+ }
2318
+ subscribe(callback, options) {
2319
+ const interval = options?.intervalMs ?? this.defaultIntervalMs;
2320
+ const subscription = {
2321
+ callback,
2322
+ intervalMs: interval
2323
+ };
2324
+ let intervalGroup = this.intervals.get(interval);
2325
+ if (!intervalGroup) {
2326
+ const subscribers = new Set;
2327
+ const intervalId = setInterval(() => this.poll(subscribers), interval);
2328
+ intervalGroup = { intervalId, subscribers };
2329
+ this.intervals.set(interval, intervalGroup);
2330
+ if (!this.initialized) {
2331
+ this.initialized = true;
2332
+ this.initAndPoll(subscribers, subscription);
2333
+ } else {
2334
+ this.pollForNewSubscriber(subscription);
2335
+ }
2336
+ } else {
2337
+ this.pollForNewSubscriber(subscription);
2338
+ }
2339
+ intervalGroup.subscribers.add(subscription);
2340
+ return () => {
2341
+ const group = this.intervals.get(interval);
2342
+ if (group) {
2343
+ group.subscribers.delete(subscription);
2344
+ if (group.subscribers.size === 0) {
2345
+ clearInterval(group.intervalId);
2346
+ this.intervals.delete(interval);
2347
+ }
2348
+ }
2349
+ };
2350
+ }
2351
+ async initAndPoll(subscribers, newSubscription) {
2352
+ try {
2353
+ this.lastKnownState = await this.fetchState();
2354
+ for (const [, item] of this.lastKnownState) {
2355
+ const payload = this.payloadFactory.insert(item);
2356
+ try {
2357
+ newSubscription.callback(payload);
2358
+ } catch {}
2359
+ }
2360
+ } catch {}
2361
+ }
2362
+ pollForNewSubscriber(subscription) {
2363
+ for (const [, item] of this.lastKnownState) {
2364
+ const payload = this.payloadFactory.insert(item);
2365
+ try {
2366
+ subscription.callback(payload);
2367
+ } catch {}
2368
+ }
2369
+ }
2370
+ async poll(subscribers) {
2371
+ if (subscribers.size === 0)
2372
+ return;
2373
+ try {
2374
+ const currentState = await this.fetchState();
2375
+ const changes = [];
2376
+ for (const [key, item] of currentState) {
2377
+ const oldItem = this.lastKnownState.get(key);
2378
+ if (!oldItem) {
2379
+ changes.push(this.payloadFactory.insert(item));
2380
+ } else if (!this.compareItems(oldItem, item)) {
2381
+ changes.push(this.payloadFactory.update(oldItem, item));
2382
+ }
2383
+ }
2384
+ for (const [key, item] of this.lastKnownState) {
2385
+ if (!currentState.has(key)) {
2386
+ changes.push(this.payloadFactory.delete(item));
2387
+ }
2388
+ }
2389
+ this.lastKnownState = currentState;
2390
+ for (const change of changes) {
2391
+ for (const sub of subscribers) {
2392
+ try {
2393
+ sub.callback(change);
2394
+ } catch {}
2395
+ }
2396
+ }
2397
+ } catch {}
2398
+ }
2399
+ get subscriptionCount() {
2400
+ let count = 0;
2401
+ for (const group of this.intervals.values()) {
2402
+ count += group.subscribers.size;
2403
+ }
2404
+ return count;
2405
+ }
2406
+ get hasSubscriptions() {
2407
+ return this.intervals.size > 0;
2408
+ }
2409
+ destroy() {
2410
+ for (const group of this.intervals.values()) {
2411
+ clearInterval(group.intervalId);
2412
+ }
2413
+ this.intervals.clear();
2414
+ this.lastKnownState.clear();
2415
+ this.initialized = false;
2416
+ }
2417
+ }
2418
+
2419
+ // src/queue/PostgresQueueStorage.ts
2420
+ var POSTGRES_QUEUE_STORAGE = createServiceToken19("jobqueue.storage.postgres");
2117
2421
 
2118
2422
  class PostgresQueueStorage {
2119
2423
  db;
2120
2424
  queueName;
2121
- constructor(db, queueName) {
2425
+ prefixes;
2426
+ prefixValues;
2427
+ tableName;
2428
+ pollingManager = null;
2429
+ constructor(db, queueName, options) {
2122
2430
  this.db = db;
2123
2431
  this.queueName = queueName;
2432
+ this.prefixes = options?.prefixes ?? [];
2433
+ this.prefixValues = options?.prefixValues ?? {};
2434
+ if (this.prefixes.length > 0) {
2435
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
2436
+ this.tableName = `job_queue_${prefixNames}`;
2437
+ } else {
2438
+ this.tableName = "job_queue";
2439
+ }
2440
+ }
2441
+ getPrefixColumnType(type) {
2442
+ return type === "uuid" ? "UUID" : "INTEGER";
2443
+ }
2444
+ buildPrefixColumnsSql() {
2445
+ if (this.prefixes.length === 0)
2446
+ return "";
2447
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
2448
+ `) + `,
2449
+ `;
2450
+ }
2451
+ getPrefixColumnNames() {
2452
+ return this.prefixes.map((p) => p.name);
2453
+ }
2454
+ buildPrefixWhereClause(startParam) {
2455
+ if (this.prefixes.length === 0) {
2456
+ return { conditions: "", params: [] };
2457
+ }
2458
+ const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
2459
+ const params = this.prefixes.map((p) => this.prefixValues[p.name]);
2460
+ return { conditions: " AND " + conditions, params };
2461
+ }
2462
+ getPrefixParamValues() {
2463
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
2124
2464
  }
2125
2465
  async setupDatabase() {
2126
2466
  let sql;
@@ -2131,10 +2471,13 @@ class PostgresQueueStorage {
2131
2471
  if (e.code !== "42710")
2132
2472
  throw e;
2133
2473
  }
2474
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
2475
+ const prefixColumnNames = this.getPrefixColumnNames();
2476
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
2134
2477
  sql = `
2135
- CREATE TABLE IF NOT EXISTS job_queue (
2478
+ CREATE TABLE IF NOT EXISTS ${this.tableName} (
2136
2479
  id SERIAL NOT NULL,
2137
- fingerprint text NOT NULL,
2480
+ ${prefixColumnsSql}fingerprint text NOT NULL,
2138
2481
  queue text NOT NULL,
2139
2482
  job_run_id text NOT NULL,
2140
2483
  status job_status NOT NULL default 'PENDING',
@@ -2151,20 +2494,22 @@ class PostgresQueueStorage {
2151
2494
  error_code text,
2152
2495
  progress real DEFAULT 0,
2153
2496
  progress_message text DEFAULT '',
2154
- progress_details jsonb
2497
+ progress_details jsonb,
2498
+ worker_id text
2155
2499
  )`;
2156
2500
  await this.db.query(sql);
2501
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
2157
2502
  sql = `
2158
- CREATE INDEX IF NOT EXISTS job_fetcher_idx
2159
- ON job_queue (id, status, run_after)`;
2503
+ CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx
2504
+ ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`;
2160
2505
  await this.db.query(sql);
2161
2506
  sql = `
2162
- CREATE INDEX IF NOT EXISTS job_queue_fetcher_idx
2163
- ON job_queue (queue, status, run_after)`;
2507
+ CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx
2508
+ ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`;
2164
2509
  await this.db.query(sql);
2165
2510
  sql = `
2166
- CREATE INDEX IF NOT EXISTS jobs_fingerprint_unique_idx
2167
- ON job_queue (queue, fingerprint, status)`;
2511
+ CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx
2512
+ ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`;
2168
2513
  await this.db.query(sql);
2169
2514
  }
2170
2515
  async add(job) {
@@ -2178,9 +2523,14 @@ class PostgresQueueStorage {
2178
2523
  job.progress_details = null;
2179
2524
  job.created_at = now;
2180
2525
  job.run_after = now;
2526
+ const prefixColumnNames = this.getPrefixColumnNames();
2527
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
2528
+ const prefixParamValues = this.getPrefixParamValues();
2529
+ const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(",") + "," : "";
2530
+ const baseParamStart = prefixColumnNames.length + 1;
2181
2531
  const sql = `
2182
- INSERT INTO job_queue(
2183
- queue,
2532
+ INSERT INTO ${this.tableName}(
2533
+ ${prefixColumnsInsert}queue,
2184
2534
  fingerprint,
2185
2535
  input,
2186
2536
  run_after,
@@ -2193,9 +2543,10 @@ class PostgresQueueStorage {
2193
2543
  progress_details
2194
2544
  )
2195
2545
  VALUES
2196
- ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
2546
+ (${prefixParamPlaceholders}$${baseParamStart},$${baseParamStart + 1},$${baseParamStart + 2},$${baseParamStart + 3},$${baseParamStart + 4},$${baseParamStart + 5},$${baseParamStart + 6},$${baseParamStart + 7},$${baseParamStart + 8},$${baseParamStart + 9},$${baseParamStart + 10})
2197
2547
  RETURNING id`;
2198
2548
  const params = [
2549
+ ...prefixParamValues,
2199
2550
  job.queue,
2200
2551
  job.fingerprint,
2201
2552
  JSON.stringify(job.input),
@@ -2215,68 +2566,75 @@ class PostgresQueueStorage {
2215
2566
  return job.id;
2216
2567
  }
2217
2568
  async get(id) {
2569
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2218
2570
  const result = await this.db.query(`SELECT *
2219
- FROM job_queue
2220
- WHERE id = $1 AND queue = $2
2571
+ FROM ${this.tableName}
2572
+ WHERE id = $1 AND queue = $2${prefixConditions}
2221
2573
  FOR UPDATE SKIP LOCKED
2222
- LIMIT 1`, [id, this.queueName]);
2574
+ LIMIT 1`, [id, this.queueName, ...prefixParams]);
2223
2575
  if (!result || result.rows.length === 0)
2224
2576
  return;
2225
2577
  return result.rows[0];
2226
2578
  }
2227
2579
  async peek(status = "PENDING" /* PENDING */, num = 100) {
2228
2580
  num = Number(num) || 100;
2581
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
2229
2582
  const result = await this.db.query(`
2230
2583
  SELECT *
2231
- FROM job_queue
2584
+ FROM ${this.tableName}
2232
2585
  WHERE queue = $1
2233
- AND status = $2
2586
+ AND status = $2${prefixConditions}
2234
2587
  ORDER BY run_after ASC
2235
2588
  LIMIT $3
2236
- FOR UPDATE SKIP LOCKED`, [this.queueName, status, num]);
2589
+ FOR UPDATE SKIP LOCKED`, [this.queueName, status, num, ...prefixParams]);
2237
2590
  if (!result)
2238
2591
  return [];
2239
2592
  return result.rows;
2240
2593
  }
2241
- async next() {
2594
+ async next(workerId) {
2595
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
2242
2596
  const result = await this.db.query(`
2243
- UPDATE job_queue
2244
- SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC'
2597
+ UPDATE ${this.tableName}
2598
+ SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = $4
2245
2599
  WHERE id = (
2246
2600
  SELECT id
2247
- FROM job_queue
2601
+ FROM ${this.tableName}
2248
2602
  WHERE queue = $2
2249
- AND status = $3
2603
+ AND status = $3${prefixConditions}
2250
2604
  AND run_after <= NOW() AT TIME ZONE 'UTC'
2251
2605
  ORDER BY run_after ASC
2252
2606
  FOR UPDATE SKIP LOCKED
2253
2607
  LIMIT 1
2254
2608
  )
2255
- RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING */]);
2609
+ RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING */, workerId ?? null, ...prefixParams]);
2256
2610
  return result?.rows?.[0] ?? undefined;
2257
2611
  }
2258
2612
  async size(status = "PENDING" /* PENDING */) {
2613
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2259
2614
  const result = await this.db.query(`
2260
2615
  SELECT COUNT(*) as count
2261
- FROM job_queue
2616
+ FROM ${this.tableName}
2262
2617
  WHERE queue = $1
2263
- AND status = $2`, [this.queueName, status]);
2618
+ AND status = $2${prefixConditions}`, [this.queueName, status, ...prefixParams]);
2264
2619
  if (!result)
2265
2620
  return 0;
2266
2621
  return parseInt(result.rows[0].count, 10);
2267
2622
  }
2268
2623
  async complete(jobDetails) {
2624
+ const prefixParams = this.getPrefixParamValues();
2269
2625
  if (jobDetails.status === "DISABLED" /* DISABLED */) {
2270
- await this.db.query(`UPDATE job_queue
2626
+ const { conditions: prefixConditions } = this.buildPrefixWhereClause(4);
2627
+ await this.db.query(`UPDATE ${this.tableName}
2271
2628
  SET
2272
2629
  status = $1,
2273
2630
  progress = 100,
2274
2631
  progress_message = '',
2275
2632
  progress_details = NULL,
2276
2633
  completed_at = NOW() AT TIME ZONE 'UTC'
2277
- WHERE id = $2 AND queue = $3`, [jobDetails.status, jobDetails.id, this.queueName]);
2634
+ WHERE id = $2 AND queue = $3${prefixConditions}`, [jobDetails.status, jobDetails.id, this.queueName, ...prefixParams]);
2278
2635
  } else if (jobDetails.status === "PENDING" /* PENDING */) {
2279
- await this.db.query(`UPDATE job_queue
2636
+ const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
2637
+ await this.db.query(`UPDATE ${this.tableName}
2280
2638
  SET
2281
2639
  error = $1,
2282
2640
  error_code = $2,
@@ -2287,17 +2645,19 @@ class PostgresQueueStorage {
2287
2645
  progress_details = NULL,
2288
2646
  run_attempts = run_attempts + 1,
2289
2647
  last_ran_at = NOW() AT TIME ZONE 'UTC'
2290
- WHERE id = $5 AND queue = $6`, [
2648
+ WHERE id = $5 AND queue = $6${prefixConditions}`, [
2291
2649
  jobDetails.error,
2292
2650
  jobDetails.error_code,
2293
2651
  jobDetails.status,
2294
2652
  jobDetails.run_after,
2295
2653
  jobDetails.id,
2296
- this.queueName
2654
+ this.queueName,
2655
+ ...prefixParams
2297
2656
  ]);
2298
2657
  } else {
2658
+ const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
2299
2659
  await this.db.query(`
2300
- UPDATE job_queue
2660
+ UPDATE ${this.tableName}
2301
2661
  SET
2302
2662
  output = $1,
2303
2663
  error = $2,
@@ -2309,86 +2669,231 @@ class PostgresQueueStorage {
2309
2669
  run_attempts = run_attempts + 1,
2310
2670
  completed_at = NOW() AT TIME ZONE 'UTC',
2311
2671
  last_ran_at = NOW() AT TIME ZONE 'UTC'
2312
- WHERE id = $5 AND queue = $6`, [
2672
+ WHERE id = $5 AND queue = $6${prefixConditions}`, [
2313
2673
  jobDetails.output ? JSON.stringify(jobDetails.output) : null,
2314
2674
  jobDetails.error ?? null,
2315
2675
  jobDetails.error_code ?? null,
2316
2676
  jobDetails.status,
2317
2677
  jobDetails.id,
2318
- this.queueName
2678
+ this.queueName,
2679
+ ...prefixParams
2319
2680
  ]);
2320
2681
  }
2321
2682
  }
2322
2683
  async deleteAll() {
2684
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
2323
2685
  await this.db.query(`
2324
- DELETE FROM job_queue
2325
- WHERE queue = $1`, [this.queueName]);
2686
+ DELETE FROM ${this.tableName}
2687
+ WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
2326
2688
  }
2327
2689
  async outputForInput(input) {
2328
2690
  const fingerprint = await makeFingerprint5(input);
2691
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2329
2692
  const result = await this.db.query(`
2330
2693
  SELECT output
2331
- FROM job_queue
2332
- WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'`, [fingerprint, this.queueName]);
2333
- if (!result)
2694
+ FROM ${this.tableName}
2695
+ WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'${prefixConditions}`, [fingerprint, this.queueName, ...prefixParams]);
2696
+ if (!result || result.rows.length === 0)
2334
2697
  return null;
2335
2698
  return result.rows[0].output;
2336
2699
  }
2337
2700
  async abort(jobId) {
2338
- const result = await this.db.query(`
2339
- UPDATE job_queue
2701
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2702
+ await this.db.query(`
2703
+ UPDATE ${this.tableName}
2340
2704
  SET status = 'ABORTING'
2341
- WHERE id = $1 AND queue = $2`, [jobId, this.queueName]);
2705
+ WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
2342
2706
  }
2343
2707
  async getByRunId(job_run_id) {
2708
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2344
2709
  const result = await this.db.query(`
2345
- SELECT * FROM job_queue WHERE job_run_id = $1 AND queue = $2`, [job_run_id, this.queueName]);
2710
+ SELECT * FROM ${this.tableName} WHERE job_run_id = $1 AND queue = $2${prefixConditions}`, [job_run_id, this.queueName, ...prefixParams]);
2346
2711
  if (!result)
2347
2712
  return [];
2348
2713
  return result.rows;
2349
2714
  }
2350
2715
  async saveProgress(jobId, progress, message, details) {
2716
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(6);
2351
2717
  await this.db.query(`
2352
- UPDATE job_queue
2718
+ UPDATE ${this.tableName}
2353
2719
  SET progress = $1,
2354
2720
  progress_message = $2,
2355
2721
  progress_details = $3
2356
- WHERE id = $4 AND queue = $5`, [progress, message, details ? JSON.stringify(details) : null, jobId, this.queueName]);
2357
- }
2358
- async delete(jobId) {
2359
- await this.db.query("DELETE FROM job_queue WHERE id = $1 AND queue = $2", [
2722
+ WHERE id = $4 AND queue = $5${prefixConditions}`, [
2723
+ progress,
2724
+ message,
2725
+ details ? JSON.stringify(details) : null,
2360
2726
  jobId,
2361
- this.queueName
2727
+ this.queueName,
2728
+ ...prefixParams
2362
2729
  ]);
2363
2730
  }
2731
+ async delete(jobId) {
2732
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2733
+ await this.db.query(`DELETE FROM ${this.tableName} WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
2734
+ }
2364
2735
  async deleteJobsByStatusAndAge(status, olderThanMs) {
2365
2736
  const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
2366
- await this.db.query(`DELETE FROM job_queue
2737
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
2738
+ await this.db.query(`DELETE FROM ${this.tableName}
2367
2739
  WHERE queue = $1
2368
2740
  AND status = $2
2369
2741
  AND completed_at IS NOT NULL
2370
- AND completed_at <= $3`, [this.queueName, status, cutoffDate]);
2742
+ AND completed_at <= $3${prefixConditions}`, [this.queueName, status, cutoffDate, ...prefixParams]);
2743
+ }
2744
+ async getAllJobs() {
2745
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
2746
+ const result = await this.db.query(`SELECT * FROM ${this.tableName} WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
2747
+ if (!result)
2748
+ return [];
2749
+ return result.rows;
2750
+ }
2751
+ async getAllJobsWithFilter(prefixFilter) {
2752
+ const filterEntries = Object.entries(prefixFilter);
2753
+ let query = `SELECT * FROM ${this.tableName} WHERE queue = $1`;
2754
+ const params = [this.queueName];
2755
+ filterEntries.forEach(([key, value], index) => {
2756
+ query += ` AND ${key} = $${index + 2}`;
2757
+ params.push(value);
2758
+ });
2759
+ const result = await this.db.query(query, params);
2760
+ if (!result)
2761
+ return [];
2762
+ return result.rows;
2763
+ }
2764
+ isCustomPrefixFilter(prefixFilter) {
2765
+ if (prefixFilter === undefined) {
2766
+ return false;
2767
+ }
2768
+ if (Object.keys(prefixFilter).length === 0) {
2769
+ return true;
2770
+ }
2771
+ const instanceKeys = Object.keys(this.prefixValues);
2772
+ const filterKeys = Object.keys(prefixFilter);
2773
+ if (instanceKeys.length !== filterKeys.length) {
2774
+ return true;
2775
+ }
2776
+ for (const key of instanceKeys) {
2777
+ if (this.prefixValues[key] !== prefixFilter[key]) {
2778
+ return true;
2779
+ }
2780
+ }
2781
+ return false;
2782
+ }
2783
+ getPollingManager() {
2784
+ if (!this.pollingManager) {
2785
+ this.pollingManager = new PollingSubscriptionManager(async () => {
2786
+ const jobs = await this.getAllJobs();
2787
+ return new Map(jobs.map((j) => [j.id, j]));
2788
+ }, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
2789
+ insert: (item) => ({ type: "INSERT", new: item }),
2790
+ update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
2791
+ delete: (item) => ({ type: "DELETE", old: item })
2792
+ });
2793
+ }
2794
+ return this.pollingManager;
2795
+ }
2796
+ subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
2797
+ let lastKnownJobs = new Map;
2798
+ let cancelled = false;
2799
+ const poll = async () => {
2800
+ if (cancelled)
2801
+ return;
2802
+ try {
2803
+ const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
2804
+ if (cancelled)
2805
+ return;
2806
+ const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
2807
+ for (const [id, job] of currentMap) {
2808
+ const old = lastKnownJobs.get(id);
2809
+ if (!old) {
2810
+ callback({ type: "INSERT", new: job });
2811
+ } else if (JSON.stringify(old) !== JSON.stringify(job)) {
2812
+ callback({ type: "UPDATE", old, new: job });
2813
+ }
2814
+ }
2815
+ for (const [id, job] of lastKnownJobs) {
2816
+ if (!currentMap.has(id)) {
2817
+ callback({ type: "DELETE", old: job });
2818
+ }
2819
+ }
2820
+ lastKnownJobs = currentMap;
2821
+ } catch {}
2822
+ };
2823
+ const intervalId = setInterval(poll, intervalMs);
2824
+ poll();
2825
+ return () => {
2826
+ cancelled = true;
2827
+ clearInterval(intervalId);
2828
+ };
2829
+ }
2830
+ subscribeToChanges(callback, options) {
2831
+ const intervalMs = options?.pollingIntervalMs ?? 1000;
2832
+ if (this.isCustomPrefixFilter(options?.prefixFilter)) {
2833
+ return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
2834
+ }
2835
+ const manager = this.getPollingManager();
2836
+ return manager.subscribe(callback, { intervalMs });
2371
2837
  }
2372
2838
  }
2373
2839
  // src/queue/SqliteQueueStorage.ts
2374
- import { createServiceToken as createServiceToken18, makeFingerprint as makeFingerprint6, sleep as sleep3, uuid4 as uuid43 } from "@workglow/util";
2375
- var SQLITE_QUEUE_STORAGE = createServiceToken18("jobqueue.storage.sqlite");
2840
+ import { createServiceToken as createServiceToken20, makeFingerprint as makeFingerprint6, sleep as sleep4, uuid4 as uuid43 } from "@workglow/util";
2841
+ var SQLITE_QUEUE_STORAGE = createServiceToken20("jobqueue.storage.sqlite");
2376
2842
 
2377
2843
  class SqliteQueueStorage {
2378
2844
  db;
2379
2845
  queueName;
2380
2846
  options;
2847
+ prefixes;
2848
+ prefixValues;
2849
+ tableName;
2850
+ pollingManager = null;
2381
2851
  constructor(db, queueName, options) {
2382
2852
  this.db = db;
2383
2853
  this.queueName = queueName;
2384
2854
  this.options = options;
2855
+ this.prefixes = options?.prefixes ?? [];
2856
+ this.prefixValues = options?.prefixValues ?? {};
2857
+ if (this.prefixes.length > 0) {
2858
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
2859
+ this.tableName = `job_queue_${prefixNames}`;
2860
+ } else {
2861
+ this.tableName = "job_queue";
2862
+ }
2863
+ }
2864
+ getPrefixColumnType(type) {
2865
+ return type === "uuid" ? "TEXT" : "INTEGER";
2866
+ }
2867
+ buildPrefixColumnsSql() {
2868
+ if (this.prefixes.length === 0)
2869
+ return "";
2870
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
2871
+ `) + `,
2872
+ `;
2873
+ }
2874
+ getPrefixColumnNames() {
2875
+ return this.prefixes.map((p) => p.name);
2876
+ }
2877
+ buildPrefixWhereClause() {
2878
+ if (this.prefixes.length === 0) {
2879
+ return "";
2880
+ }
2881
+ const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
2882
+ return " AND " + conditions;
2883
+ }
2884
+ getPrefixParamValues() {
2885
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
2385
2886
  }
2386
2887
  async setupDatabase() {
2387
- await sleep3(0);
2888
+ await sleep4(0);
2889
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
2890
+ const prefixColumnNames = this.getPrefixColumnNames();
2891
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
2892
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
2388
2893
  this.db.exec(`
2389
- CREATE TABLE IF NOT EXISTS job_queue (
2894
+ CREATE TABLE IF NOT EXISTS ${this.tableName} (
2390
2895
  id INTEGER PRIMARY KEY,
2391
- fingerprint text NOT NULL,
2896
+ ${prefixColumnsSql}fingerprint text NOT NULL,
2392
2897
  queue text NOT NULL,
2393
2898
  job_run_id text NOT NULL,
2394
2899
  status TEXT NOT NULL default 'PENDING',
@@ -2405,12 +2910,13 @@ class SqliteQueueStorage {
2405
2910
  error_code TEXT,
2406
2911
  progress REAL DEFAULT 0,
2407
2912
  progress_message TEXT DEFAULT '',
2408
- progress_details TEXT NULL
2913
+ progress_details TEXT NULL,
2914
+ worker_id TEXT
2409
2915
  );
2410
2916
 
2411
- CREATE INDEX IF NOT EXISTS job_queue_fetcher_idx ON job_queue (queue, status, run_after);
2412
- CREATE INDEX IF NOT EXISTS job_queue_fingerprint_idx ON job_queue (queue, fingerprint, status);
2413
- CREATE INDEX IF NOT EXISTS job_queue_job_run_id_idx ON job_queue (queue, job_run_id);
2917
+ CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after);
2918
+ CREATE INDEX IF NOT EXISTS job_queue_fingerprint${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status);
2919
+ CREATE INDEX IF NOT EXISTS job_queue_job_run_id${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, job_run_id);
2414
2920
  `);
2415
2921
  }
2416
2922
  async add(job) {
@@ -2424,9 +2930,13 @@ class SqliteQueueStorage {
2424
2930
  job.progress_details = null;
2425
2931
  job.created_at = now;
2426
2932
  job.run_after = now;
2933
+ const prefixColumnNames = this.getPrefixColumnNames();
2934
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
2935
+ const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
2936
+ const prefixParamValues = this.getPrefixParamValues();
2427
2937
  const AddQuery = `
2428
- INSERT INTO job_queue(
2429
- queue,
2938
+ INSERT INTO ${this.tableName}(
2939
+ ${prefixColumnsInsert}queue,
2430
2940
  fingerprint,
2431
2941
  input,
2432
2942
  run_after,
@@ -2438,21 +2948,23 @@ class SqliteQueueStorage {
2438
2948
  progress_details,
2439
2949
  created_at
2440
2950
  )
2441
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
2951
+ VALUES (${prefixPlaceholders}?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
2442
2952
  RETURNING id`;
2443
2953
  const stmt = this.db.prepare(AddQuery);
2444
- const result = stmt.get(job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
2954
+ const result = stmt.get(...prefixParamValues, job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
2445
2955
  job.id = result?.id;
2446
2956
  return result?.id;
2447
2957
  }
2448
2958
  async get(id) {
2959
+ const prefixConditions = this.buildPrefixWhereClause();
2960
+ const prefixParams = this.getPrefixParamValues();
2449
2961
  const JobQuery = `
2450
2962
  SELECT *
2451
- FROM job_queue
2452
- WHERE id = ? AND queue = ?
2963
+ FROM ${this.tableName}
2964
+ WHERE id = ? AND queue = ?${prefixConditions}
2453
2965
  LIMIT 1`;
2454
2966
  const stmt = this.db.prepare(JobQuery);
2455
- const result = stmt.get(id, this.queueName);
2967
+ const result = stmt.get(String(id), this.queueName, ...prefixParams);
2456
2968
  if (!result)
2457
2969
  return;
2458
2970
  if (result.input)
@@ -2465,15 +2977,17 @@ class SqliteQueueStorage {
2465
2977
  }
2466
2978
  async peek(status = "PENDING" /* PENDING */, num = 100) {
2467
2979
  num = Number(num) || 100;
2980
+ const prefixConditions = this.buildPrefixWhereClause();
2981
+ const prefixParams = this.getPrefixParamValues();
2468
2982
  const FutureJobQuery = `
2469
2983
  SELECT *
2470
- FROM job_queue
2984
+ FROM ${this.tableName}
2471
2985
  WHERE queue = ?
2472
- AND status = ?
2986
+ AND status = ?${prefixConditions}
2473
2987
  ORDER BY run_after ASC
2474
2988
  LIMIT ${num}`;
2475
2989
  const stmt = this.db.prepare(FutureJobQuery);
2476
- const result = stmt.all(this.queueName, status);
2990
+ const result = stmt.all(this.queueName, status, ...prefixParams);
2477
2991
  return (result || []).map((details) => {
2478
2992
  if (details.input)
2479
2993
  details.input = JSON.parse(details.input);
@@ -2485,20 +2999,24 @@ class SqliteQueueStorage {
2485
2999
  });
2486
3000
  }
2487
3001
  async abort(jobId) {
3002
+ const prefixConditions = this.buildPrefixWhereClause();
3003
+ const prefixParams = this.getPrefixParamValues();
2488
3004
  const AbortQuery = `
2489
- UPDATE job_queue
3005
+ UPDATE ${this.tableName}
2490
3006
  SET status = ?
2491
- WHERE id = ? AND queue = ?`;
3007
+ WHERE id = ? AND queue = ?${prefixConditions}`;
2492
3008
  const stmt = this.db.prepare(AbortQuery);
2493
- stmt.run("ABORTING" /* ABORTING */, jobId, this.queueName);
3009
+ stmt.run("ABORTING" /* ABORTING */, String(jobId), this.queueName, ...prefixParams);
2494
3010
  }
2495
3011
  async getByRunId(job_run_id) {
3012
+ const prefixConditions = this.buildPrefixWhereClause();
3013
+ const prefixParams = this.getPrefixParamValues();
2496
3014
  const JobsByRunIdQuery = `
2497
3015
  SELECT *
2498
- FROM job_queue
2499
- WHERE job_run_id = ? AND queue = ?`;
3016
+ FROM ${this.tableName}
3017
+ WHERE job_run_id = ? AND queue = ?${prefixConditions}`;
2500
3018
  const stmt = this.db.prepare(JobsByRunIdQuery);
2501
- const result = stmt.all(job_run_id, this.queueName);
3019
+ const result = stmt.all(job_run_id, this.queueName, ...prefixParams);
2502
3020
  return (result || []).map((details) => {
2503
3021
  if (details.input)
2504
3022
  details.input = JSON.parse(details.input);
@@ -2509,22 +3027,24 @@ class SqliteQueueStorage {
2509
3027
  return details;
2510
3028
  });
2511
3029
  }
2512
- async next() {
3030
+ async next(workerId) {
2513
3031
  const now = new Date().toISOString();
3032
+ const prefixConditions = this.buildPrefixWhereClause();
3033
+ const prefixParams = this.getPrefixParamValues();
2514
3034
  const stmt = this.db.prepare(`
2515
- UPDATE job_queue
2516
- SET status = ?, last_ran_at = ?
3035
+ UPDATE ${this.tableName}
3036
+ SET status = ?, last_ran_at = ?, worker_id = ?
2517
3037
  WHERE id = (
2518
3038
  SELECT id
2519
- FROM job_queue
3039
+ FROM ${this.tableName}
2520
3040
  WHERE queue = ?
2521
- AND status = ?
3041
+ AND status = ?${prefixConditions}
2522
3042
  AND run_after <= ?
2523
3043
  ORDER BY run_after ASC
2524
3044
  LIMIT 1
2525
3045
  )
2526
3046
  RETURNING *`);
2527
- const result = stmt.get("PROCESSING" /* PROCESSING */, now, this.queueName, "PENDING" /* PENDING */, now);
3047
+ const result = stmt.get("PROCESSING" /* PROCESSING */, now, workerId ?? null, this.queueName, "PENDING" /* PENDING */, ...prefixParams, now);
2528
3048
  if (!result)
2529
3049
  return;
2530
3050
  if (result.input)
@@ -2536,33 +3056,37 @@ class SqliteQueueStorage {
2536
3056
  return result;
2537
3057
  }
2538
3058
  async size(status = "PENDING" /* PENDING */) {
3059
+ const prefixConditions = this.buildPrefixWhereClause();
3060
+ const prefixParams = this.getPrefixParamValues();
2539
3061
  const sizeQuery = `
2540
3062
  SELECT COUNT(*) as count
2541
- FROM job_queue
3063
+ FROM ${this.tableName}
2542
3064
  WHERE queue = ?
2543
- AND status = ?`;
3065
+ AND status = ?${prefixConditions}`;
2544
3066
  const stmt = this.db.prepare(sizeQuery);
2545
- const result = stmt.get(this.queueName, status);
3067
+ const result = stmt.get(this.queueName, status, ...prefixParams);
2546
3068
  return result.count;
2547
3069
  }
2548
3070
  async complete(job) {
2549
3071
  const now = new Date().toISOString();
3072
+ const prefixConditions = this.buildPrefixWhereClause();
3073
+ const prefixParams = this.getPrefixParamValues();
2550
3074
  let updateQuery;
2551
3075
  let params;
2552
3076
  if (job.status === "DISABLED" /* DISABLED */) {
2553
3077
  updateQuery = `
2554
- UPDATE job_queue
3078
+ UPDATE ${this.tableName}
2555
3079
  SET
2556
3080
  status = ?,
2557
3081
  progress = 100,
2558
3082
  progress_message = '',
2559
3083
  progress_details = NULL,
2560
3084
  completed_at = ?
2561
- WHERE id = ? AND queue = ?`;
2562
- params = [job.status, now, job.id, this.queueName];
3085
+ WHERE id = ? AND queue = ?${prefixConditions}`;
3086
+ params = [job.status, now, job.id, this.queueName, ...prefixParams];
2563
3087
  } else {
2564
3088
  updateQuery = `
2565
- UPDATE job_queue
3089
+ UPDATE ${this.tableName}
2566
3090
  SET
2567
3091
  output = ?,
2568
3092
  error = ?,
@@ -2574,7 +3098,7 @@ class SqliteQueueStorage {
2574
3098
  last_ran_at = ?,
2575
3099
  completed_at = ?,
2576
3100
  run_attempts = run_attempts + 1
2577
- WHERE id = ? AND queue = ?`;
3101
+ WHERE id = ? AND queue = ?${prefixConditions}`;
2578
3102
  params = [
2579
3103
  job.output ? JSON.stringify(job.output) : null,
2580
3104
  job.error ?? null,
@@ -2583,68 +3107,227 @@ class SqliteQueueStorage {
2583
3107
  now,
2584
3108
  now,
2585
3109
  job.id,
2586
- this.queueName
3110
+ this.queueName,
3111
+ ...prefixParams
2587
3112
  ];
2588
3113
  }
2589
3114
  const stmt = this.db.prepare(updateQuery);
2590
3115
  stmt.run(...params);
2591
3116
  }
2592
3117
  async deleteAll() {
3118
+ const prefixConditions = this.buildPrefixWhereClause();
3119
+ const prefixParams = this.getPrefixParamValues();
2593
3120
  const ClearQuery = `
2594
- DELETE FROM job_queue
2595
- WHERE queue = ?`;
3121
+ DELETE FROM ${this.tableName}
3122
+ WHERE queue = ?${prefixConditions}`;
2596
3123
  const stmt = this.db.prepare(ClearQuery);
2597
- stmt.run(this.queueName);
3124
+ stmt.run(this.queueName, ...prefixParams);
2598
3125
  }
2599
3126
  async outputForInput(input) {
2600
3127
  const fingerprint = await makeFingerprint6(input);
3128
+ const prefixConditions = this.buildPrefixWhereClause();
3129
+ const prefixParams = this.getPrefixParamValues();
2601
3130
  const OutputQuery = `
2602
3131
  SELECT output
2603
- FROM job_queue
2604
- WHERE queue = ? AND fingerprint = ? AND status = ?`;
3132
+ FROM ${this.tableName}
3133
+ WHERE queue = ? AND fingerprint = ? AND status = ?${prefixConditions}`;
2605
3134
  const stmt = this.db.prepare(OutputQuery);
2606
- const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED */);
3135
+ const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED */, ...prefixParams);
2607
3136
  return result?.output ? JSON.parse(result.output) : null;
2608
3137
  }
2609
3138
  async saveProgress(jobId, progress, message, details) {
3139
+ const prefixConditions = this.buildPrefixWhereClause();
3140
+ const prefixParams = this.getPrefixParamValues();
2610
3141
  const UpdateProgressQuery = `
2611
- UPDATE job_queue
3142
+ UPDATE ${this.tableName}
2612
3143
  SET progress = ?,
2613
3144
  progress_message = ?,
2614
3145
  progress_details = ?
2615
- WHERE id = ? AND queue = ?`;
3146
+ WHERE id = ? AND queue = ?${prefixConditions}`;
2616
3147
  const stmt = this.db.prepare(UpdateProgressQuery);
2617
- stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName);
3148
+ stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName, ...prefixParams);
2618
3149
  }
2619
3150
  async delete(jobId) {
3151
+ const prefixConditions = this.buildPrefixWhereClause();
3152
+ const prefixParams = this.getPrefixParamValues();
2620
3153
  const DeleteQuery = `
2621
- DELETE FROM job_queue
2622
- WHERE id = ? AND queue = ?`;
3154
+ DELETE FROM ${this.tableName}
3155
+ WHERE id = ? AND queue = ?${prefixConditions}`;
2623
3156
  const stmt = this.db.prepare(DeleteQuery);
2624
- stmt.run(String(jobId), this.queueName);
3157
+ stmt.run(String(jobId), this.queueName, ...prefixParams);
2625
3158
  }
2626
3159
  async deleteJobsByStatusAndAge(status, olderThanMs) {
2627
3160
  const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
3161
+ const prefixConditions = this.buildPrefixWhereClause();
3162
+ const prefixParams = this.getPrefixParamValues();
2628
3163
  const DeleteQuery = `
2629
- DELETE FROM job_queue
3164
+ DELETE FROM ${this.tableName}
2630
3165
  WHERE queue = ?
2631
3166
  AND status = ?
2632
3167
  AND completed_at IS NOT NULL
2633
- AND completed_at <= ?`;
3168
+ AND completed_at <= ?${prefixConditions}`;
2634
3169
  const stmt = this.db.prepare(DeleteQuery);
2635
- stmt.run(this.queueName, status, cutoffDate);
3170
+ stmt.run(this.queueName, status, cutoffDate, ...prefixParams);
3171
+ }
3172
+ getAllJobs() {
3173
+ const prefixConditions = this.buildPrefixWhereClause();
3174
+ const prefixParams = this.getPrefixParamValues();
3175
+ const AllJobsQuery = `
3176
+ SELECT *
3177
+ FROM ${this.tableName}
3178
+ WHERE queue = ?${prefixConditions}`;
3179
+ const stmt = this.db.prepare(AllJobsQuery);
3180
+ const result = stmt.all(this.queueName, ...prefixParams);
3181
+ return (result || []).map((details) => {
3182
+ if (details.input)
3183
+ details.input = JSON.parse(details.input);
3184
+ if (details.output)
3185
+ details.output = JSON.parse(details.output);
3186
+ if (details.progress_details)
3187
+ details.progress_details = JSON.parse(details.progress_details);
3188
+ return details;
3189
+ });
3190
+ }
3191
+ getAllJobsWithFilter(prefixFilter) {
3192
+ const filterEntries = Object.entries(prefixFilter);
3193
+ let whereClause = "WHERE queue = ?";
3194
+ const params = [this.queueName];
3195
+ for (const [key, value] of filterEntries) {
3196
+ whereClause += ` AND ${key} = ?`;
3197
+ params.push(value);
3198
+ }
3199
+ const AllJobsQuery = `SELECT * FROM ${this.tableName} ${whereClause}`;
3200
+ const stmt = this.db.prepare(AllJobsQuery);
3201
+ const result = stmt.all(...params);
3202
+ return (result || []).map((details) => {
3203
+ if (details.input)
3204
+ details.input = JSON.parse(details.input);
3205
+ if (details.output)
3206
+ details.output = JSON.parse(details.output);
3207
+ if (details.progress_details)
3208
+ details.progress_details = JSON.parse(details.progress_details);
3209
+ return details;
3210
+ });
3211
+ }
3212
+ isCustomPrefixFilter(prefixFilter) {
3213
+ if (prefixFilter === undefined) {
3214
+ return false;
3215
+ }
3216
+ if (Object.keys(prefixFilter).length === 0) {
3217
+ return true;
3218
+ }
3219
+ const instanceKeys = Object.keys(this.prefixValues);
3220
+ const filterKeys = Object.keys(prefixFilter);
3221
+ if (instanceKeys.length !== filterKeys.length) {
3222
+ return true;
3223
+ }
3224
+ for (const key of instanceKeys) {
3225
+ if (this.prefixValues[key] !== prefixFilter[key]) {
3226
+ return true;
3227
+ }
3228
+ }
3229
+ return false;
3230
+ }
3231
+ getPollingManager() {
3232
+ if (!this.pollingManager) {
3233
+ this.pollingManager = new PollingSubscriptionManager(async () => {
3234
+ const jobs = this.getAllJobs();
3235
+ return new Map(jobs.map((j) => [j.id, j]));
3236
+ }, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
3237
+ insert: (item) => ({ type: "INSERT", new: item }),
3238
+ update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
3239
+ delete: (item) => ({ type: "DELETE", old: item })
3240
+ });
3241
+ }
3242
+ return this.pollingManager;
3243
+ }
3244
+ subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
3245
+ let lastKnownJobs = new Map;
3246
+ const poll = () => {
3247
+ try {
3248
+ const currentJobs = this.getAllJobsWithFilter(prefixFilter);
3249
+ const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
3250
+ for (const [id, job] of currentMap) {
3251
+ const old = lastKnownJobs.get(id);
3252
+ if (!old) {
3253
+ callback({ type: "INSERT", new: job });
3254
+ } else if (JSON.stringify(old) !== JSON.stringify(job)) {
3255
+ callback({ type: "UPDATE", old, new: job });
3256
+ }
3257
+ }
3258
+ for (const [id, job] of lastKnownJobs) {
3259
+ if (!currentMap.has(id)) {
3260
+ callback({ type: "DELETE", old: job });
3261
+ }
3262
+ }
3263
+ lastKnownJobs = currentMap;
3264
+ } catch {}
3265
+ };
3266
+ const intervalId = setInterval(poll, intervalMs);
3267
+ poll();
3268
+ return () => {
3269
+ clearInterval(intervalId);
3270
+ };
3271
+ }
3272
+ subscribeToChanges(callback, options) {
3273
+ const intervalMs = options?.pollingIntervalMs ?? 1000;
3274
+ if (this.isCustomPrefixFilter(options?.prefixFilter)) {
3275
+ return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
3276
+ }
3277
+ const manager = this.getPollingManager();
3278
+ return manager.subscribe(callback, { intervalMs });
2636
3279
  }
2637
3280
  }
2638
3281
  // src/queue/SupabaseQueueStorage.ts
2639
- import { createServiceToken as createServiceToken19, makeFingerprint as makeFingerprint7, uuid4 as uuid44 } from "@workglow/util";
2640
- var SUPABASE_QUEUE_STORAGE = createServiceToken19("jobqueue.storage.supabase");
3282
+ import { createServiceToken as createServiceToken21, makeFingerprint as makeFingerprint7, uuid4 as uuid44 } from "@workglow/util";
3283
+ var SUPABASE_QUEUE_STORAGE = createServiceToken21("jobqueue.storage.supabase");
2641
3284
 
2642
3285
  class SupabaseQueueStorage {
2643
3286
  client;
2644
3287
  queueName;
2645
- constructor(client, queueName) {
3288
+ prefixes;
3289
+ prefixValues;
3290
+ tableName;
3291
+ realtimeChannel = null;
3292
+ pollingManager = null;
3293
+ constructor(client, queueName, options) {
2646
3294
  this.client = client;
2647
3295
  this.queueName = queueName;
3296
+ this.prefixes = options?.prefixes ?? [];
3297
+ this.prefixValues = options?.prefixValues ?? {};
3298
+ if (this.prefixes.length > 0) {
3299
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
3300
+ this.tableName = `job_queue_${prefixNames}`;
3301
+ } else {
3302
+ this.tableName = "job_queue";
3303
+ }
3304
+ }
3305
+ getPrefixColumnType(type) {
3306
+ return type === "uuid" ? "UUID" : "INTEGER";
3307
+ }
3308
+ buildPrefixColumnsSql() {
3309
+ if (this.prefixes.length === 0)
3310
+ return "";
3311
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
3312
+ `) + `,
3313
+ `;
3314
+ }
3315
+ getPrefixColumnNames() {
3316
+ return this.prefixes.map((p) => p.name);
3317
+ }
3318
+ applyPrefixFilters(query) {
3319
+ let result = query;
3320
+ for (const prefix of this.prefixes) {
3321
+ result = result.eq(prefix.name, this.prefixValues[prefix.name]);
3322
+ }
3323
+ return result;
3324
+ }
3325
+ getPrefixInsertValues() {
3326
+ const values = {};
3327
+ for (const prefix of this.prefixes) {
3328
+ values[prefix.name] = this.prefixValues[prefix.name];
3329
+ }
3330
+ return values;
2648
3331
  }
2649
3332
  async setupDatabase() {
2650
3333
  const createTypeSql = `CREATE TYPE job_status AS ENUM (${Object.values(JobStatus).map((v) => `'${v}'`).join(",")})`;
@@ -2652,10 +3335,14 @@ class SupabaseQueueStorage {
2652
3335
  if (typeError && typeError.code !== "42710") {
2653
3336
  throw typeError;
2654
3337
  }
3338
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
3339
+ const prefixColumnNames = this.getPrefixColumnNames();
3340
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3341
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
2655
3342
  const createTableSql = `
2656
- CREATE TABLE IF NOT EXISTS job_queue (
3343
+ CREATE TABLE IF NOT EXISTS ${this.tableName} (
2657
3344
  id SERIAL NOT NULL,
2658
- fingerprint text NOT NULL,
3345
+ ${prefixColumnsSql}fingerprint text NOT NULL,
2659
3346
  queue text NOT NULL,
2660
3347
  job_run_id text NOT NULL,
2661
3348
  status job_status NOT NULL default 'PENDING',
@@ -2672,7 +3359,8 @@ class SupabaseQueueStorage {
2672
3359
  error_code text,
2673
3360
  progress real DEFAULT 0,
2674
3361
  progress_message text DEFAULT '',
2675
- progress_details jsonb
3362
+ progress_details jsonb,
3363
+ worker_id text
2676
3364
  )`;
2677
3365
  const { error: tableError } = await this.client.rpc("exec_sql", { query: createTableSql });
2678
3366
  if (tableError) {
@@ -2681,12 +3369,12 @@ class SupabaseQueueStorage {
2681
3369
  }
2682
3370
  }
2683
3371
  const indexes = [
2684
- `CREATE INDEX IF NOT EXISTS job_fetcher_idx ON job_queue (id, status, run_after)`,
2685
- `CREATE INDEX IF NOT EXISTS job_queue_fetcher_idx ON job_queue (queue, status, run_after)`,
2686
- `CREATE INDEX IF NOT EXISTS jobs_fingerprint_unique_idx ON job_queue (queue, fingerprint, status)`
3372
+ `CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`,
3373
+ `CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`,
3374
+ `CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`
2687
3375
  ];
2688
3376
  for (const indexSql of indexes) {
2689
- const { error: indexError } = await this.client.rpc("exec_sql", { query: indexSql });
3377
+ await this.client.rpc("exec_sql", { query: indexSql });
2690
3378
  }
2691
3379
  }
2692
3380
  async add(job) {
@@ -2700,7 +3388,9 @@ class SupabaseQueueStorage {
2700
3388
  job.progress_details = null;
2701
3389
  job.created_at = now;
2702
3390
  job.run_after = now;
2703
- const { data, error } = await this.client.from("job_queue").insert({
3391
+ const prefixInsertValues = this.getPrefixInsertValues();
3392
+ const { data, error } = await this.client.from(this.tableName).insert({
3393
+ ...prefixInsertValues,
2704
3394
  queue: job.queue,
2705
3395
  fingerprint: job.fingerprint,
2706
3396
  input: job.input,
@@ -2721,7 +3411,9 @@ class SupabaseQueueStorage {
2721
3411
  return job.id;
2722
3412
  }
2723
3413
  async get(id) {
2724
- const { data, error } = await this.client.from("job_queue").select("*").eq("id", id).eq("queue", this.queueName).single();
3414
+ let query = this.client.from(this.tableName).select("*").eq("id", id).eq("queue", this.queueName);
3415
+ query = this.applyPrefixFilters(query);
3416
+ const { data, error } = await query.single();
2725
3417
  if (error) {
2726
3418
  if (error.code === "PGRST116")
2727
3419
  return;
@@ -2731,36 +3423,53 @@ class SupabaseQueueStorage {
2731
3423
  }
2732
3424
  async peek(status = "PENDING" /* PENDING */, num = 100) {
2733
3425
  num = Number(num) || 100;
2734
- const { data, error } = await this.client.from("job_queue").select("*").eq("queue", this.queueName).eq("status", status).order("run_after", { ascending: true }).limit(num);
3426
+ let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", status);
3427
+ query = this.applyPrefixFilters(query);
3428
+ const { data, error } = await query.order("run_after", { ascending: true }).limit(num);
2735
3429
  if (error)
2736
3430
  throw error;
2737
3431
  return data ?? [];
2738
3432
  }
2739
- async next() {
2740
- const { data: jobs, error: selectError } = await this.client.from("job_queue").select("*").eq("queue", this.queueName).eq("status", "PENDING" /* PENDING */).lte("run_after", new Date().toISOString()).order("run_after", { ascending: true }).limit(1);
3433
+ async next(workerId) {
3434
+ let selectQuery = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", "PENDING" /* PENDING */).lte("run_after", new Date().toISOString());
3435
+ selectQuery = this.applyPrefixFilters(selectQuery);
3436
+ const { data: jobs, error: selectError } = await selectQuery.order("run_after", { ascending: true }).limit(1);
2741
3437
  if (selectError)
2742
3438
  throw selectError;
2743
3439
  if (!jobs || jobs.length === 0)
2744
3440
  return;
2745
3441
  const job = jobs[0];
2746
- const { data: updatedJob, error: updateError } = await this.client.from("job_queue").update({
3442
+ let updateQuery = this.client.from(this.tableName).update({
2747
3443
  status: "PROCESSING" /* PROCESSING */,
2748
- last_ran_at: new Date().toISOString()
2749
- }).eq("id", job.id).eq("queue", this.queueName).select().single();
3444
+ last_ran_at: new Date().toISOString(),
3445
+ worker_id: workerId ?? null
3446
+ }).eq("id", job.id).eq("queue", this.queueName);
3447
+ updateQuery = this.applyPrefixFilters(updateQuery);
3448
+ const { data: updatedJob, error: updateError } = await updateQuery.select().single();
2750
3449
  if (updateError)
2751
3450
  throw updateError;
2752
3451
  return updatedJob;
2753
3452
  }
2754
3453
  async size(status = "PENDING" /* PENDING */) {
2755
- const { count, error } = await this.client.from("job_queue").select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
3454
+ let query = this.client.from(this.tableName).select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
3455
+ query = this.applyPrefixFilters(query);
3456
+ const { count, error } = await query;
2756
3457
  if (error)
2757
3458
  throw error;
2758
3459
  return count ?? 0;
2759
3460
  }
3461
+ async getAllJobs() {
3462
+ let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
3463
+ query = this.applyPrefixFilters(query);
3464
+ const { data, error } = await query;
3465
+ if (error)
3466
+ throw error;
3467
+ return data ?? [];
3468
+ }
2760
3469
  async complete(jobDetails) {
2761
3470
  const now = new Date().toISOString();
2762
3471
  if (jobDetails.status === "DISABLED" /* DISABLED */) {
2763
- const { error: error2 } = await this.client.from("job_queue").update({
3472
+ let query2 = this.client.from(this.tableName).update({
2764
3473
  status: jobDetails.status,
2765
3474
  progress: 100,
2766
3475
  progress_message: "",
@@ -2768,16 +3477,20 @@ class SupabaseQueueStorage {
2768
3477
  completed_at: now,
2769
3478
  last_ran_at: now
2770
3479
  }).eq("id", jobDetails.id).eq("queue", this.queueName);
3480
+ query2 = this.applyPrefixFilters(query2);
3481
+ const { error: error2 } = await query2;
2771
3482
  if (error2)
2772
3483
  throw error2;
2773
3484
  return;
2774
3485
  }
2775
- const { data: current, error: getError } = await this.client.from("job_queue").select("run_attempts").eq("id", jobDetails.id).eq("queue", this.queueName).single();
3486
+ let getQuery = this.client.from(this.tableName).select("run_attempts").eq("id", jobDetails.id).eq("queue", this.queueName);
3487
+ getQuery = this.applyPrefixFilters(getQuery);
3488
+ const { data: current, error: getError } = await getQuery.single();
2776
3489
  if (getError)
2777
3490
  throw getError;
2778
3491
  const nextAttempts = (current?.run_attempts ?? 0) + 1;
2779
3492
  if (jobDetails.status === "PENDING" /* PENDING */) {
2780
- const { error: error2 } = await this.client.from("job_queue").update({
3493
+ let query2 = this.client.from(this.tableName).update({
2781
3494
  error: jobDetails.error ?? null,
2782
3495
  error_code: jobDetails.error_code ?? null,
2783
3496
  status: jobDetails.status,
@@ -2788,12 +3501,14 @@ class SupabaseQueueStorage {
2788
3501
  run_attempts: nextAttempts,
2789
3502
  last_ran_at: now
2790
3503
  }).eq("id", jobDetails.id).eq("queue", this.queueName);
3504
+ query2 = this.applyPrefixFilters(query2);
3505
+ const { error: error2 } = await query2;
2791
3506
  if (error2)
2792
3507
  throw error2;
2793
3508
  return;
2794
3509
  }
2795
3510
  if (jobDetails.status === "COMPLETED" /* COMPLETED */ || jobDetails.status === "FAILED" /* FAILED */) {
2796
- const { error: error2 } = await this.client.from("job_queue").update({
3511
+ let query2 = this.client.from(this.tableName).update({
2797
3512
  output: jobDetails.output ?? null,
2798
3513
  error: jobDetails.error ?? null,
2799
3514
  error_code: jobDetails.error_code ?? null,
@@ -2805,11 +3520,13 @@ class SupabaseQueueStorage {
2805
3520
  completed_at: now,
2806
3521
  last_ran_at: now
2807
3522
  }).eq("id", jobDetails.id).eq("queue", this.queueName);
3523
+ query2 = this.applyPrefixFilters(query2);
3524
+ const { error: error2 } = await query2;
2808
3525
  if (error2)
2809
3526
  throw error2;
2810
3527
  return;
2811
3528
  }
2812
- const { error } = await this.client.from("job_queue").update({
3529
+ let query = this.client.from(this.tableName).update({
2813
3530
  status: jobDetails.status,
2814
3531
  output: jobDetails.output ?? null,
2815
3532
  error: jobDetails.error ?? null,
@@ -2818,17 +3535,23 @@ class SupabaseQueueStorage {
2818
3535
  run_attempts: nextAttempts,
2819
3536
  last_ran_at: now
2820
3537
  }).eq("id", jobDetails.id).eq("queue", this.queueName);
3538
+ query = this.applyPrefixFilters(query);
3539
+ const { error } = await query;
2821
3540
  if (error)
2822
3541
  throw error;
2823
3542
  }
2824
3543
  async deleteAll() {
2825
- const { error } = await this.client.from("job_queue").delete().eq("queue", this.queueName);
3544
+ let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
3545
+ query = this.applyPrefixFilters(query);
3546
+ const { error } = await query;
2826
3547
  if (error)
2827
3548
  throw error;
2828
3549
  }
2829
3550
  async outputForInput(input) {
2830
3551
  const fingerprint = await makeFingerprint7(input);
2831
- const { data, error } = await this.client.from("job_queue").select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", "COMPLETED" /* COMPLETED */).single();
3552
+ let query = this.client.from(this.tableName).select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", "COMPLETED" /* COMPLETED */);
3553
+ query = this.applyPrefixFilters(query);
3554
+ const { data, error } = await query.single();
2832
3555
  if (error) {
2833
3556
  if (error.code === "PGRST116")
2834
3557
  return null;
@@ -2837,47 +3560,622 @@ class SupabaseQueueStorage {
2837
3560
  return data?.output ?? null;
2838
3561
  }
2839
3562
  async abort(jobId) {
2840
- const { error } = await this.client.from("job_queue").update({ status: "ABORTING" /* ABORTING */ }).eq("id", jobId).eq("queue", this.queueName);
3563
+ let query = this.client.from(this.tableName).update({ status: "ABORTING" /* ABORTING */ }).eq("id", jobId).eq("queue", this.queueName);
3564
+ query = this.applyPrefixFilters(query);
3565
+ const { error } = await query;
2841
3566
  if (error)
2842
3567
  throw error;
2843
3568
  }
2844
3569
  async getByRunId(job_run_id) {
2845
- const { data, error } = await this.client.from("job_queue").select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
3570
+ let query = this.client.from(this.tableName).select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
3571
+ query = this.applyPrefixFilters(query);
3572
+ const { data, error } = await query;
2846
3573
  if (error)
2847
3574
  throw error;
2848
3575
  return data ?? [];
2849
3576
  }
2850
3577
  async saveProgress(jobId, progress, message, details) {
2851
- const { error } = await this.client.from("job_queue").update({
3578
+ let query = this.client.from(this.tableName).update({
2852
3579
  progress,
2853
3580
  progress_message: message,
2854
3581
  progress_details: details
2855
3582
  }).eq("id", jobId).eq("queue", this.queueName);
3583
+ query = this.applyPrefixFilters(query);
3584
+ const { error } = await query;
2856
3585
  if (error)
2857
3586
  throw error;
2858
3587
  }
2859
3588
  async delete(jobId) {
2860
- const { error } = await this.client.from("job_queue").delete().eq("id", jobId).eq("queue", this.queueName);
3589
+ let query = this.client.from(this.tableName).delete().eq("id", jobId).eq("queue", this.queueName);
3590
+ query = this.applyPrefixFilters(query);
3591
+ const { error } = await query;
2861
3592
  if (error)
2862
3593
  throw error;
2863
3594
  }
2864
3595
  async deleteJobsByStatusAndAge(status, olderThanMs) {
2865
3596
  const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
2866
- const { error } = await this.client.from("job_queue").delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
3597
+ let query = this.client.from(this.tableName).delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
3598
+ query = this.applyPrefixFilters(query);
3599
+ const { error } = await query;
2867
3600
  if (error)
2868
3601
  throw error;
2869
3602
  }
2870
- }
2871
- // src/kv/IndexedDbKvRepository.ts
2872
- import { createServiceToken as createServiceToken21 } from "@workglow/util";
2873
-
2874
- // src/tabular/IndexedDbTabularRepository.ts
2875
- import { createServiceToken as createServiceToken20 } from "@workglow/util";
2876
-
2877
- // src/util/IndexedDbTable.ts
2878
- var METADATA_STORE_NAME = "__schema_metadata__";
2879
- async function saveSchemaMetadata(db, tableName, snapshot) {
2880
- return new Promise((resolve, reject) => {
3603
+ matchesPrefixFilter(job, prefixFilter) {
3604
+ if (!job)
3605
+ return false;
3606
+ if (job.queue !== this.queueName) {
3607
+ return false;
3608
+ }
3609
+ if (prefixFilter && Object.keys(prefixFilter).length === 0) {
3610
+ return true;
3611
+ }
3612
+ const filterValues = prefixFilter ?? this.prefixValues;
3613
+ if (Object.keys(filterValues).length === 0) {
3614
+ return true;
3615
+ }
3616
+ for (const [key, value] of Object.entries(filterValues)) {
3617
+ if (job[key] !== value) {
3618
+ return false;
3619
+ }
3620
+ }
3621
+ return true;
3622
+ }
3623
+ isCustomPrefixFilter(prefixFilter) {
3624
+ if (prefixFilter === undefined) {
3625
+ return false;
3626
+ }
3627
+ if (Object.keys(prefixFilter).length === 0) {
3628
+ return true;
3629
+ }
3630
+ const instanceKeys = Object.keys(this.prefixValues);
3631
+ const filterKeys = Object.keys(prefixFilter);
3632
+ if (instanceKeys.length !== filterKeys.length) {
3633
+ return true;
3634
+ }
3635
+ for (const key of instanceKeys) {
3636
+ if (this.prefixValues[key] !== prefixFilter[key]) {
3637
+ return true;
3638
+ }
3639
+ }
3640
+ return false;
3641
+ }
3642
+ async getAllJobsWithFilter(prefixFilter) {
3643
+ let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
3644
+ for (const [key, value] of Object.entries(prefixFilter)) {
3645
+ query = query.eq(key, value);
3646
+ }
3647
+ const { data, error } = await query;
3648
+ if (error)
3649
+ throw error;
3650
+ return data ?? [];
3651
+ }
3652
+ subscribeToChanges(callback, options) {
3653
+ return this.subscribeToChangesWithRealtime(callback, options?.prefixFilter);
3654
+ }
3655
+ subscribeToChangesWithRealtime(callback, prefixFilter) {
3656
+ const channelName = `queue-${this.tableName}-${this.queueName}-${Date.now()}`;
3657
+ this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
3658
+ event: "*",
3659
+ schema: "public",
3660
+ table: this.tableName,
3661
+ filter: `queue=eq.${this.queueName}`
3662
+ }, (payload) => {
3663
+ const newJob = payload.new;
3664
+ const oldJob = payload.old;
3665
+ const newMatches = this.matchesPrefixFilter(newJob, prefixFilter);
3666
+ const oldMatches = this.matchesPrefixFilter(oldJob, prefixFilter);
3667
+ if (!newMatches && !oldMatches) {
3668
+ return;
3669
+ }
3670
+ callback({
3671
+ type: payload.eventType.toUpperCase(),
3672
+ old: oldJob && Object.keys(oldJob).length > 0 ? oldJob : undefined,
3673
+ new: newJob && Object.keys(newJob).length > 0 ? newJob : undefined
3674
+ });
3675
+ }).subscribe();
3676
+ return () => {
3677
+ if (this.realtimeChannel) {
3678
+ this.client.removeChannel(this.realtimeChannel);
3679
+ this.realtimeChannel = null;
3680
+ }
3681
+ };
3682
+ }
3683
+ getPollingManager() {
3684
+ if (!this.pollingManager) {
3685
+ this.pollingManager = new PollingSubscriptionManager(async () => {
3686
+ const jobs = await this.getAllJobs();
3687
+ return new Map(jobs.map((j) => [j.id, j]));
3688
+ }, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
3689
+ insert: (item) => ({ type: "INSERT", new: item }),
3690
+ update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
3691
+ delete: (item) => ({ type: "DELETE", old: item })
3692
+ });
3693
+ }
3694
+ return this.pollingManager;
3695
+ }
3696
+ subscribeWithCustomPrefixFilterPolling(callback, prefixFilter, intervalMs) {
3697
+ let lastKnownJobs = new Map;
3698
+ let cancelled = false;
3699
+ const poll = async () => {
3700
+ if (cancelled)
3701
+ return;
3702
+ try {
3703
+ const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
3704
+ if (cancelled)
3705
+ return;
3706
+ const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
3707
+ for (const [id, job] of currentMap) {
3708
+ const old = lastKnownJobs.get(id);
3709
+ if (!old) {
3710
+ callback({ type: "INSERT", new: job });
3711
+ } else if (JSON.stringify(old) !== JSON.stringify(job)) {
3712
+ callback({ type: "UPDATE", old, new: job });
3713
+ }
3714
+ }
3715
+ for (const [id, job] of lastKnownJobs) {
3716
+ if (!currentMap.has(id)) {
3717
+ callback({ type: "DELETE", old: job });
3718
+ }
3719
+ }
3720
+ lastKnownJobs = currentMap;
3721
+ } catch {}
3722
+ };
3723
+ const intervalId = setInterval(poll, intervalMs);
3724
+ poll();
3725
+ return () => {
3726
+ cancelled = true;
3727
+ clearInterval(intervalId);
3728
+ };
3729
+ }
3730
+ subscribeToChangesWithPolling(callback, options) {
3731
+ const intervalMs = options?.pollingIntervalMs ?? 1000;
3732
+ if (this.isCustomPrefixFilter(options?.prefixFilter)) {
3733
+ return this.subscribeWithCustomPrefixFilterPolling(callback, options.prefixFilter, intervalMs);
3734
+ }
3735
+ const manager = this.getPollingManager();
3736
+ return manager.subscribe(callback, { intervalMs });
3737
+ }
3738
+ }
3739
+ // src/limiter/PostgresRateLimiterStorage.ts
3740
+ import { createServiceToken as createServiceToken22 } from "@workglow/util";
3741
+ var POSTGRES_RATE_LIMITER_STORAGE = createServiceToken22("ratelimiter.storage.postgres");
3742
+
3743
+ class PostgresRateLimiterStorage {
3744
+ db;
3745
+ prefixes;
3746
+ prefixValues;
3747
+ executionTableName;
3748
+ nextAvailableTableName;
3749
+ constructor(db, options) {
3750
+ this.db = db;
3751
+ this.prefixes = options?.prefixes ?? [];
3752
+ this.prefixValues = options?.prefixValues ?? {};
3753
+ if (this.prefixes.length > 0) {
3754
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
3755
+ this.executionTableName = `rate_limit_executions_${prefixNames}`;
3756
+ this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
3757
+ } else {
3758
+ this.executionTableName = "rate_limit_executions";
3759
+ this.nextAvailableTableName = "rate_limit_next_available";
3760
+ }
3761
+ }
3762
+ getPrefixColumnType(type) {
3763
+ return type === "uuid" ? "UUID" : "INTEGER";
3764
+ }
3765
+ buildPrefixColumnsSql() {
3766
+ if (this.prefixes.length === 0)
3767
+ return "";
3768
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
3769
+ `) + `,
3770
+ `;
3771
+ }
3772
+ getPrefixColumnNames() {
3773
+ return this.prefixes.map((p) => p.name);
3774
+ }
3775
+ buildPrefixWhereClause(startParam) {
3776
+ if (this.prefixes.length === 0) {
3777
+ return { conditions: "", params: [] };
3778
+ }
3779
+ const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
3780
+ const params = this.prefixes.map((p) => this.prefixValues[p.name]);
3781
+ return { conditions: " AND " + conditions, params };
3782
+ }
3783
+ getPrefixParamValues() {
3784
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
3785
+ }
3786
+ async setupDatabase() {
3787
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
3788
+ const prefixColumnNames = this.getPrefixColumnNames();
3789
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3790
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
3791
+ await this.db.query(`
3792
+ CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
3793
+ id SERIAL PRIMARY KEY,
3794
+ ${prefixColumnsSql}queue_name TEXT NOT NULL,
3795
+ executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
3796
+ )
3797
+ `);
3798
+ await this.db.query(`
3799
+ CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
3800
+ ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
3801
+ `);
3802
+ const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
3803
+ await this.db.query(`
3804
+ CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
3805
+ ${prefixColumnsSql}queue_name TEXT NOT NULL,
3806
+ next_available_at TIMESTAMP WITH TIME ZONE,
3807
+ PRIMARY KEY (${primaryKeyColumns})
3808
+ )
3809
+ `);
3810
+ }
3811
+ async recordExecution(queueName) {
3812
+ const prefixColumnNames = this.getPrefixColumnNames();
3813
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3814
+ const prefixParamValues = this.getPrefixParamValues();
3815
+ const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
3816
+ const queueParamNum = prefixColumnNames.length + 1;
3817
+ await this.db.query(`
3818
+ INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
3819
+ VALUES (${prefixParamPlaceholders}$${queueParamNum})
3820
+ `, [...prefixParamValues, queueName]);
3821
+ }
3822
+ async getExecutionCount(queueName, windowStartTime) {
3823
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
3824
+ const result = await this.db.query(`
3825
+ SELECT COUNT(*) AS count
3826
+ FROM ${this.executionTableName}
3827
+ WHERE queue_name = $1 AND executed_at > $2${prefixConditions}
3828
+ `, [queueName, windowStartTime, ...prefixParams]);
3829
+ return parseInt(result.rows[0]?.count ?? "0", 10);
3830
+ }
3831
+ async getOldestExecutionAtOffset(queueName, offset) {
3832
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
3833
+ const result = await this.db.query(`
3834
+ SELECT executed_at
3835
+ FROM ${this.executionTableName}
3836
+ WHERE queue_name = $1${prefixConditions}
3837
+ ORDER BY executed_at ASC
3838
+ LIMIT 1 OFFSET $2
3839
+ `, [queueName, offset, ...prefixParams]);
3840
+ const executedAt = result.rows[0]?.executed_at;
3841
+ if (!executedAt)
3842
+ return;
3843
+ return new Date(executedAt).toISOString();
3844
+ }
3845
+ async getNextAvailableTime(queueName) {
3846
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
3847
+ const result = await this.db.query(`
3848
+ SELECT next_available_at
3849
+ FROM ${this.nextAvailableTableName}
3850
+ WHERE queue_name = $1${prefixConditions}
3851
+ `, [queueName, ...prefixParams]);
3852
+ const nextAvailableAt = result.rows[0]?.next_available_at;
3853
+ if (!nextAvailableAt)
3854
+ return;
3855
+ return new Date(nextAvailableAt).toISOString();
3856
+ }
3857
+ async setNextAvailableTime(queueName, nextAvailableAt) {
3858
+ const prefixColumnNames = this.getPrefixColumnNames();
3859
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3860
+ const prefixParamValues = this.getPrefixParamValues();
3861
+ const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
3862
+ const baseParamStart = prefixColumnNames.length + 1;
3863
+ const conflictColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
3864
+ await this.db.query(`
3865
+ INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
3866
+ VALUES (${prefixParamPlaceholders}$${baseParamStart}, $${baseParamStart + 1})
3867
+ ON CONFLICT (${conflictColumns})
3868
+ DO UPDATE SET next_available_at = EXCLUDED.next_available_at
3869
+ `, [...prefixParamValues, queueName, nextAvailableAt]);
3870
+ }
3871
+ async clear(queueName) {
3872
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
3873
+ await this.db.query(`DELETE FROM ${this.executionTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
3874
+ await this.db.query(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
3875
+ }
3876
+ }
3877
+ // src/limiter/SqliteRateLimiterStorage.ts
3878
+ import { createServiceToken as createServiceToken23, sleep as sleep5, toSQLiteTimestamp } from "@workglow/util";
3879
+ var SQLITE_RATE_LIMITER_STORAGE = createServiceToken23("ratelimiter.storage.sqlite");
3880
+
3881
+ class SqliteRateLimiterStorage {
3882
+ db;
3883
+ prefixes;
3884
+ prefixValues;
3885
+ executionTableName;
3886
+ nextAvailableTableName;
3887
+ constructor(db, options) {
3888
+ this.db = db;
3889
+ this.prefixes = options?.prefixes ?? [];
3890
+ this.prefixValues = options?.prefixValues ?? {};
3891
+ if (this.prefixes.length > 0) {
3892
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
3893
+ this.executionTableName = `rate_limit_executions_${prefixNames}`;
3894
+ this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
3895
+ } else {
3896
+ this.executionTableName = "rate_limit_executions";
3897
+ this.nextAvailableTableName = "rate_limit_next_available";
3898
+ }
3899
+ }
3900
+ getPrefixColumnType(type) {
3901
+ return type === "uuid" ? "TEXT" : "INTEGER";
3902
+ }
3903
+ buildPrefixColumnsSql() {
3904
+ if (this.prefixes.length === 0)
3905
+ return "";
3906
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
3907
+ `) + `,
3908
+ `;
3909
+ }
3910
+ getPrefixColumnNames() {
3911
+ return this.prefixes.map((p) => p.name);
3912
+ }
3913
+ buildPrefixWhereClause() {
3914
+ if (this.prefixes.length === 0) {
3915
+ return "";
3916
+ }
3917
+ const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
3918
+ return " AND " + conditions;
3919
+ }
3920
+ getPrefixParamValues() {
3921
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
3922
+ }
3923
+ async setupDatabase() {
3924
+ await sleep5(0);
3925
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
3926
+ const prefixColumnNames = this.getPrefixColumnNames();
3927
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3928
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
3929
+ this.db.exec(`
3930
+ CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
3931
+ id INTEGER PRIMARY KEY,
3932
+ ${prefixColumnsSql}queue_name TEXT NOT NULL,
3933
+ executed_at TEXT DEFAULT CURRENT_TIMESTAMP
3934
+ );
3935
+
3936
+ CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
3937
+ ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at);
3938
+ `);
3939
+ this.db.exec(`
3940
+ CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
3941
+ ${prefixColumnsSql}queue_name TEXT PRIMARY KEY,
3942
+ next_available_at TEXT
3943
+ );
3944
+ `);
3945
+ }
3946
+ async recordExecution(queueName) {
3947
+ const prefixColumnNames = this.getPrefixColumnNames();
3948
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3949
+ const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
3950
+ const prefixParamValues = this.getPrefixParamValues();
3951
+ const stmt = this.db.prepare(`
3952
+ INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
3953
+ VALUES (${prefixPlaceholders}?)
3954
+ `);
3955
+ stmt.run(...prefixParamValues, queueName);
3956
+ }
3957
+ async getExecutionCount(queueName, windowStartTime) {
3958
+ const prefixConditions = this.buildPrefixWhereClause();
3959
+ const prefixParams = this.getPrefixParamValues();
3960
+ const thresholdTime = toSQLiteTimestamp(new Date(windowStartTime));
3961
+ const stmt = this.db.prepare(`
3962
+ SELECT COUNT(*) AS count
3963
+ FROM ${this.executionTableName}
3964
+ WHERE queue_name = ? AND executed_at > ?${prefixConditions}
3965
+ `);
3966
+ const result = stmt.get(queueName, thresholdTime, ...prefixParams);
3967
+ return result?.count ?? 0;
3968
+ }
3969
+ async getOldestExecutionAtOffset(queueName, offset) {
3970
+ const prefixConditions = this.buildPrefixWhereClause();
3971
+ const prefixParams = this.getPrefixParamValues();
3972
+ const stmt = this.db.prepare(`
3973
+ SELECT executed_at
3974
+ FROM ${this.executionTableName}
3975
+ WHERE queue_name = ?${prefixConditions}
3976
+ ORDER BY executed_at ASC
3977
+ LIMIT 1 OFFSET ?
3978
+ `);
3979
+ const result = stmt.get(queueName, ...prefixParams, offset);
3980
+ if (!result)
3981
+ return;
3982
+ return result.executed_at + "Z";
3983
+ }
3984
+ async getNextAvailableTime(queueName) {
3985
+ const prefixConditions = this.buildPrefixWhereClause();
3986
+ const prefixParams = this.getPrefixParamValues();
3987
+ const stmt = this.db.prepare(`
3988
+ SELECT next_available_at
3989
+ FROM ${this.nextAvailableTableName}
3990
+ WHERE queue_name = ?${prefixConditions}
3991
+ `);
3992
+ const result = stmt.get(queueName, ...prefixParams);
3993
+ if (!result?.next_available_at)
3994
+ return;
3995
+ return result.next_available_at + "Z";
3996
+ }
3997
+ async setNextAvailableTime(queueName, nextAvailableAt) {
3998
+ const prefixColumnNames = this.getPrefixColumnNames();
3999
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
4000
+ const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
4001
+ const prefixParamValues = this.getPrefixParamValues();
4002
+ const stmt = this.db.prepare(`
4003
+ INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
4004
+ VALUES (${prefixPlaceholders}?, ?)
4005
+ ON CONFLICT(queue_name) DO UPDATE SET next_available_at = excluded.next_available_at
4006
+ `);
4007
+ stmt.run(...prefixParamValues, queueName, nextAvailableAt);
4008
+ }
4009
+ async clear(queueName) {
4010
+ const prefixConditions = this.buildPrefixWhereClause();
4011
+ const prefixParams = this.getPrefixParamValues();
4012
+ this.db.prepare(`DELETE FROM ${this.executionTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
4013
+ this.db.prepare(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
4014
+ }
4015
+ }
4016
+ // src/limiter/SupabaseRateLimiterStorage.ts
4017
+ import { createServiceToken as createServiceToken24 } from "@workglow/util";
4018
+ var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken24("ratelimiter.storage.supabase");
4019
+
4020
+ class SupabaseRateLimiterStorage {
4021
+ client;
4022
+ prefixes;
4023
+ prefixValues;
4024
+ executionTableName;
4025
+ nextAvailableTableName;
4026
+ constructor(client, options) {
4027
+ this.client = client;
4028
+ this.prefixes = options?.prefixes ?? [];
4029
+ this.prefixValues = options?.prefixValues ?? {};
4030
+ if (this.prefixes.length > 0) {
4031
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
4032
+ this.executionTableName = `rate_limit_executions_${prefixNames}`;
4033
+ this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
4034
+ } else {
4035
+ this.executionTableName = "rate_limit_executions";
4036
+ this.nextAvailableTableName = "rate_limit_next_available";
4037
+ }
4038
+ }
4039
+ getPrefixColumnType(type) {
4040
+ return type === "uuid" ? "UUID" : "INTEGER";
4041
+ }
4042
+ buildPrefixColumnsSql() {
4043
+ if (this.prefixes.length === 0)
4044
+ return "";
4045
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
4046
+ `) + `,
4047
+ `;
4048
+ }
4049
+ getPrefixColumnNames() {
4050
+ return this.prefixes.map((p) => p.name);
4051
+ }
4052
+ applyPrefixFilters(query) {
4053
+ let result = query;
4054
+ for (const prefix of this.prefixes) {
4055
+ result = result.eq(prefix.name, this.prefixValues[prefix.name]);
4056
+ }
4057
+ return result;
4058
+ }
4059
+ getPrefixInsertValues() {
4060
+ const values = {};
4061
+ for (const prefix of this.prefixes) {
4062
+ values[prefix.name] = this.prefixValues[prefix.name];
4063
+ }
4064
+ return values;
4065
+ }
4066
+ async setupDatabase() {
4067
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
4068
+ const prefixColumnNames = this.getPrefixColumnNames();
4069
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
4070
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
4071
+ const createExecTableSql = `
4072
+ CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
4073
+ id SERIAL PRIMARY KEY,
4074
+ ${prefixColumnsSql}queue_name TEXT NOT NULL,
4075
+ executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
4076
+ )
4077
+ `;
4078
+ const { error: execTableError } = await this.client.rpc("exec_sql", {
4079
+ query: createExecTableSql
4080
+ });
4081
+ if (execTableError && execTableError.code !== "42P07") {
4082
+ throw execTableError;
4083
+ }
4084
+ const createExecIndexSql = `
4085
+ CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
4086
+ ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
4087
+ `;
4088
+ await this.client.rpc("exec_sql", { query: createExecIndexSql });
4089
+ const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
4090
+ const createNextTableSql = `
4091
+ CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
4092
+ ${prefixColumnsSql}queue_name TEXT NOT NULL,
4093
+ next_available_at TIMESTAMP WITH TIME ZONE,
4094
+ PRIMARY KEY (${primaryKeyColumns})
4095
+ )
4096
+ `;
4097
+ const { error: nextTableError } = await this.client.rpc("exec_sql", {
4098
+ query: createNextTableSql
4099
+ });
4100
+ if (nextTableError && nextTableError.code !== "42P07") {
4101
+ throw nextTableError;
4102
+ }
4103
+ }
4104
+ async recordExecution(queueName) {
4105
+ const prefixInsertValues = this.getPrefixInsertValues();
4106
+ const { error } = await this.client.from(this.executionTableName).insert({
4107
+ ...prefixInsertValues,
4108
+ queue_name: queueName
4109
+ });
4110
+ if (error)
4111
+ throw error;
4112
+ }
4113
+ async getExecutionCount(queueName, windowStartTime) {
4114
+ let query = this.client.from(this.executionTableName).select("*", { count: "exact", head: true }).eq("queue_name", queueName).gt("executed_at", windowStartTime);
4115
+ query = this.applyPrefixFilters(query);
4116
+ const { count, error } = await query;
4117
+ if (error)
4118
+ throw error;
4119
+ return count ?? 0;
4120
+ }
4121
+ async getOldestExecutionAtOffset(queueName, offset) {
4122
+ let query = this.client.from(this.executionTableName).select("executed_at").eq("queue_name", queueName);
4123
+ query = this.applyPrefixFilters(query);
4124
+ const { data, error } = await query.order("executed_at", { ascending: true }).range(offset, offset);
4125
+ if (error)
4126
+ throw error;
4127
+ if (!data || data.length === 0)
4128
+ return;
4129
+ return new Date(data[0].executed_at).toISOString();
4130
+ }
4131
+ async getNextAvailableTime(queueName) {
4132
+ let query = this.client.from(this.nextAvailableTableName).select("next_available_at").eq("queue_name", queueName);
4133
+ query = this.applyPrefixFilters(query);
4134
+ const { data, error } = await query.single();
4135
+ if (error) {
4136
+ if (error.code === "PGRST116")
4137
+ return;
4138
+ throw error;
4139
+ }
4140
+ if (!data?.next_available_at)
4141
+ return;
4142
+ return new Date(data.next_available_at).toISOString();
4143
+ }
4144
+ async setNextAvailableTime(queueName, nextAvailableAt) {
4145
+ const prefixInsertValues = this.getPrefixInsertValues();
4146
+ const { error } = await this.client.from(this.nextAvailableTableName).upsert({
4147
+ ...prefixInsertValues,
4148
+ queue_name: queueName,
4149
+ next_available_at: nextAvailableAt
4150
+ }, {
4151
+ onConflict: this.prefixes.length > 0 ? `${this.getPrefixColumnNames().join(",")},queue_name` : "queue_name"
4152
+ });
4153
+ if (error)
4154
+ throw error;
4155
+ }
4156
+ async clear(queueName) {
4157
+ let execQuery = this.client.from(this.executionTableName).delete().eq("queue_name", queueName);
4158
+ execQuery = this.applyPrefixFilters(execQuery);
4159
+ const { error: execError } = await execQuery;
4160
+ if (execError)
4161
+ throw execError;
4162
+ let nextQuery = this.client.from(this.nextAvailableTableName).delete().eq("queue_name", queueName);
4163
+ nextQuery = this.applyPrefixFilters(nextQuery);
4164
+ const { error: nextError } = await nextQuery;
4165
+ if (nextError)
4166
+ throw nextError;
4167
+ }
4168
+ }
4169
+ // src/kv/IndexedDbKvRepository.ts
4170
+ import { createServiceToken as createServiceToken26 } from "@workglow/util";
4171
+
4172
+ // src/tabular/IndexedDbTabularRepository.ts
4173
+ import { createServiceToken as createServiceToken25 } from "@workglow/util";
4174
+
4175
+ // src/util/IndexedDbTable.ts
4176
+ var METADATA_STORE_NAME = "__schema_metadata__";
4177
+ async function saveSchemaMetadata(db, tableName, snapshot) {
4178
+ return new Promise((resolve, reject) => {
2881
4179
  try {
2882
4180
  const transaction = db.transaction(METADATA_STORE_NAME, "readwrite");
2883
4181
  const store = transaction.objectStore(METADATA_STORE_NAME);
@@ -3212,7 +4510,7 @@ async function dropIndexedDbTable(tableName) {
3212
4510
  }
3213
4511
 
3214
4512
  // src/tabular/IndexedDbTabularRepository.ts
3215
- var IDB_TABULAR_REPOSITORY = createServiceToken20("storage.tabularRepository.indexedDb");
4513
+ var IDB_TABULAR_REPOSITORY = createServiceToken25("storage.tabularRepository.indexedDb");
3216
4514
 
3217
4515
  class IndexedDbTabularRepository extends TabularRepository {
3218
4516
  table;
@@ -3548,7 +4846,7 @@ class IndexedDbTabularRepository extends TabularRepository {
3548
4846
  }
3549
4847
 
3550
4848
  // src/kv/IndexedDbKvRepository.ts
3551
- var IDB_KV_REPOSITORY = createServiceToken21("storage.kvRepository.indexedDb");
4849
+ var IDB_KV_REPOSITORY = createServiceToken26("storage.kvRepository.indexedDb");
3552
4850
 
3553
4851
  class IndexedDbKvRepository extends KvViaTabularRepository {
3554
4852
  dbName;
@@ -3560,18 +4858,42 @@ class IndexedDbKvRepository extends KvViaTabularRepository {
3560
4858
  }
3561
4859
  }
3562
4860
  // src/queue/IndexedDbQueueStorage.ts
3563
- import { createServiceToken as createServiceToken22, makeFingerprint as makeFingerprint8, uuid4 as uuid45 } from "@workglow/util";
3564
- var INDEXED_DB_QUEUE_STORAGE = createServiceToken22("jobqueue.storage.indexedDb");
4861
+ import { createServiceToken as createServiceToken27, makeFingerprint as makeFingerprint8, uuid4 as uuid45 } from "@workglow/util";
4862
+ var INDEXED_DB_QUEUE_STORAGE = createServiceToken27("jobqueue.storage.indexedDb");
3565
4863
 
3566
4864
  class IndexedDbQueueStorage {
3567
4865
  queueName;
3568
4866
  db;
3569
4867
  tableName;
3570
4868
  migrationOptions;
3571
- constructor(queueName, migrationOptions = {}) {
4869
+ prefixes;
4870
+ prefixValues;
4871
+ pollingManager = null;
4872
+ constructor(queueName, options = {}) {
3572
4873
  this.queueName = queueName;
3573
- this.tableName = `jobs_${queueName}`;
3574
- this.migrationOptions = migrationOptions;
4874
+ this.migrationOptions = options;
4875
+ this.prefixes = options.prefixes ?? [];
4876
+ this.prefixValues = options.prefixValues ?? {};
4877
+ if (this.prefixes.length > 0) {
4878
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
4879
+ this.tableName = `jobs_${prefixNames}`;
4880
+ } else {
4881
+ this.tableName = "jobs";
4882
+ }
4883
+ }
4884
+ getPrefixColumnNames() {
4885
+ return this.prefixes.map((p) => p.name);
4886
+ }
4887
+ matchesPrefixes(job) {
4888
+ for (const [key, value] of Object.entries(this.prefixValues)) {
4889
+ if (job[key] !== value) {
4890
+ return false;
4891
+ }
4892
+ }
4893
+ return true;
4894
+ }
4895
+ getPrefixKeyValues() {
4896
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
3575
4897
  }
3576
4898
  async getDb() {
3577
4899
  if (this.db)
@@ -3580,25 +4902,29 @@ class IndexedDbQueueStorage {
3580
4902
  return this.db;
3581
4903
  }
3582
4904
  async setupDatabase() {
4905
+ const prefixColumnNames = this.getPrefixColumnNames();
4906
+ const buildKeyPath = (basePath) => {
4907
+ return [...prefixColumnNames, ...basePath];
4908
+ };
3583
4909
  const expectedIndexes = [
3584
4910
  {
3585
- name: "status",
3586
- keyPath: `status`,
4911
+ name: "queue_status",
4912
+ keyPath: buildKeyPath(["queue", "status"]),
3587
4913
  options: { unique: false }
3588
4914
  },
3589
4915
  {
3590
- name: "status_run_after",
3591
- keyPath: ["status", "run_after"],
4916
+ name: "queue_status_run_after",
4917
+ keyPath: buildKeyPath(["queue", "status", "run_after"]),
3592
4918
  options: { unique: false }
3593
4919
  },
3594
4920
  {
3595
- name: "job_run_id",
3596
- keyPath: `job_run_id`,
4921
+ name: "queue_job_run_id",
4922
+ keyPath: buildKeyPath(["queue", "job_run_id"]),
3597
4923
  options: { unique: false }
3598
4924
  },
3599
4925
  {
3600
- name: "fingerprint_status",
3601
- keyPath: ["fingerprint", "status"],
4926
+ name: "queue_fingerprint_status",
4927
+ keyPath: buildKeyPath(["queue", "fingerprint", "status"]),
3602
4928
  options: { unique: false }
3603
4929
  }
3604
4930
  ];
@@ -3607,21 +4933,25 @@ class IndexedDbQueueStorage {
3607
4933
  async add(job) {
3608
4934
  const db = await this.getDb();
3609
4935
  const now = new Date().toISOString();
3610
- job.id = job.id ?? uuid45();
3611
- job.job_run_id = job.job_run_id ?? uuid45();
3612
- job.queue = this.queueName;
3613
- job.fingerprint = await makeFingerprint8(job.input);
3614
- job.status = "PENDING" /* PENDING */;
3615
- job.progress = 0;
3616
- job.progress_message = "";
3617
- job.progress_details = null;
3618
- job.created_at = now;
3619
- job.run_after = now;
4936
+ const jobWithPrefixes = job;
4937
+ jobWithPrefixes.id = jobWithPrefixes.id ?? uuid45();
4938
+ jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid45();
4939
+ jobWithPrefixes.queue = this.queueName;
4940
+ jobWithPrefixes.fingerprint = await makeFingerprint8(jobWithPrefixes.input);
4941
+ jobWithPrefixes.status = "PENDING" /* PENDING */;
4942
+ jobWithPrefixes.progress = 0;
4943
+ jobWithPrefixes.progress_message = "";
4944
+ jobWithPrefixes.progress_details = null;
4945
+ jobWithPrefixes.created_at = now;
4946
+ jobWithPrefixes.run_after = now;
4947
+ for (const [key, value] of Object.entries(this.prefixValues)) {
4948
+ jobWithPrefixes[key] = value;
4949
+ }
3620
4950
  const tx = db.transaction(this.tableName, "readwrite");
3621
4951
  const store = tx.objectStore(this.tableName);
3622
4952
  return new Promise((resolve, reject) => {
3623
- const request = store.add(job);
3624
- tx.oncomplete = () => resolve(job.id);
4953
+ const request = store.add(jobWithPrefixes);
4954
+ tx.oncomplete = () => resolve(jobWithPrefixes.id);
3625
4955
  tx.onerror = () => reject(tx.error);
3626
4956
  request.onerror = () => reject(request.error);
3627
4957
  });
@@ -3632,7 +4962,14 @@ class IndexedDbQueueStorage {
3632
4962
  const store = tx.objectStore(this.tableName);
3633
4963
  const request = store.get(id);
3634
4964
  return new Promise((resolve, reject) => {
3635
- request.onsuccess = () => resolve(request.result);
4965
+ request.onsuccess = () => {
4966
+ const job = request.result;
4967
+ if (job && job.queue === this.queueName && this.matchesPrefixes(job)) {
4968
+ resolve(job);
4969
+ } else {
4970
+ resolve(undefined);
4971
+ }
4972
+ };
3636
4973
  request.onerror = () => reject(request.error);
3637
4974
  tx.onerror = () => reject(tx.error);
3638
4975
  });
@@ -3641,10 +4978,11 @@ class IndexedDbQueueStorage {
3641
4978
  const db = await this.getDb();
3642
4979
  const tx = db.transaction(this.tableName, "readonly");
3643
4980
  const store = tx.objectStore(this.tableName);
3644
- const index = store.index("status_run_after");
4981
+ const index = store.index("queue_status_run_after");
4982
+ const prefixKeyValues = this.getPrefixKeyValues();
3645
4983
  return new Promise((resolve, reject) => {
3646
4984
  const ret = new Map;
3647
- const keyRange = IDBKeyRange.bound([status, ""], [status, "\uFFFF"]);
4985
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, status, ""], [...prefixKeyValues, this.queueName, status, "\uFFFF"]);
3648
4986
  const cursorRequest = index.openCursor(keyRange);
3649
4987
  const handleCursor = (e) => {
3650
4988
  const cursor = e.target.result;
@@ -3652,7 +4990,10 @@ class IndexedDbQueueStorage {
3652
4990
  resolve(Array.from(ret.values()));
3653
4991
  return;
3654
4992
  }
3655
- ret.set(cursor.value.id, cursor.value);
4993
+ const job = cursor.value;
4994
+ if (this.matchesPrefixes(job)) {
4995
+ ret.set(cursor.value.id, cursor.value);
4996
+ }
3656
4997
  cursor.continue();
3657
4998
  };
3658
4999
  cursorRequest.onsuccess = handleCursor;
@@ -3660,14 +5001,15 @@ class IndexedDbQueueStorage {
3660
5001
  tx.onerror = () => reject(tx.error);
3661
5002
  });
3662
5003
  }
3663
- async next() {
5004
+ async next(workerId) {
3664
5005
  const db = await this.getDb();
3665
5006
  const tx = db.transaction(this.tableName, "readwrite");
3666
5007
  const store = tx.objectStore(this.tableName);
3667
- const index = store.index("status_run_after");
5008
+ const index = store.index("queue_status_run_after");
3668
5009
  const now = new Date().toISOString();
5010
+ const prefixKeyValues = this.getPrefixKeyValues();
3669
5011
  return new Promise((resolve, reject) => {
3670
- const cursorRequest = index.openCursor(IDBKeyRange.bound(["PENDING" /* PENDING */, ""], ["PENDING" /* PENDING */, now], false, true));
5012
+ const cursorRequest = index.openCursor(IDBKeyRange.bound([...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, ""], [...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, now], false, true));
3671
5013
  let jobToReturn;
3672
5014
  cursorRequest.onsuccess = (e) => {
3673
5015
  const cursor = e.target.result;
@@ -3680,12 +5022,13 @@ class IndexedDbQueueStorage {
3680
5022
  return;
3681
5023
  }
3682
5024
  const job = cursor.value;
3683
- if (job.status !== "PENDING" /* PENDING */) {
5025
+ if (job.queue !== this.queueName || job.status !== "PENDING" /* PENDING */ || !this.matchesPrefixes(job)) {
3684
5026
  cursor.continue();
3685
5027
  return;
3686
5028
  }
3687
5029
  job.status = "PROCESSING" /* PROCESSING */;
3688
5030
  job.last_ran_at = now;
5031
+ job.worker_id = workerId ?? null;
3689
5032
  try {
3690
5033
  const updateRequest = store.put(job);
3691
5034
  updateRequest.onsuccess = () => {
@@ -3709,11 +5052,13 @@ class IndexedDbQueueStorage {
3709
5052
  }
3710
5053
  async size(status = "PENDING" /* PENDING */) {
3711
5054
  const db = await this.getDb();
5055
+ const prefixKeyValues = this.getPrefixKeyValues();
3712
5056
  return new Promise((resolve, reject) => {
3713
5057
  const tx = db.transaction(this.tableName, "readonly");
3714
5058
  const store = tx.objectStore(this.tableName);
3715
- const index = store.index("status");
3716
- const request = index.count(status);
5059
+ const index = store.index("queue_status");
5060
+ const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
5061
+ const request = index.count(keyRange);
3717
5062
  request.onsuccess = () => resolve(request.result);
3718
5063
  request.onerror = () => reject(request.error);
3719
5064
  tx.onerror = () => reject(tx.error);
@@ -3727,9 +5072,18 @@ class IndexedDbQueueStorage {
3727
5072
  const getReq = store.get(job.id);
3728
5073
  getReq.onsuccess = () => {
3729
5074
  const existing = getReq.result;
3730
- const currentAttempts = existing?.run_attempts ?? 0;
5075
+ if (!existing || existing.queue !== this.queueName || !this.matchesPrefixes(existing)) {
5076
+ reject(new Error(`Job ${job.id} not found or does not belong to queue ${this.queueName}`));
5077
+ return;
5078
+ }
5079
+ const currentAttempts = existing.run_attempts ?? 0;
3731
5080
  job.run_attempts = currentAttempts + 1;
3732
- const putReq = store.put(job);
5081
+ job.queue = this.queueName;
5082
+ const jobWithPrefixes = job;
5083
+ for (const [key, value] of Object.entries(this.prefixValues)) {
5084
+ jobWithPrefixes[key] = value;
5085
+ }
5086
+ const putReq = store.put(jobWithPrefixes);
3733
5087
  putReq.onsuccess = () => {};
3734
5088
  putReq.onerror = () => reject(putReq.error);
3735
5089
  };
@@ -3749,10 +5103,15 @@ class IndexedDbQueueStorage {
3749
5103
  const db = await this.getDb();
3750
5104
  const tx = db.transaction(this.tableName, "readonly");
3751
5105
  const store = tx.objectStore(this.tableName);
3752
- const index = store.index("job_run_id");
3753
- const request = index.getAll(job_run_id);
5106
+ const index = store.index("queue_job_run_id");
5107
+ const prefixKeyValues = this.getPrefixKeyValues();
5108
+ const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, job_run_id]);
5109
+ const request = index.getAll(keyRange);
3754
5110
  return new Promise((resolve, reject) => {
3755
- request.onsuccess = () => resolve(request.result);
5111
+ request.onsuccess = () => {
5112
+ const results = (request.result || []).filter((job) => this.matchesPrefixes(job));
5113
+ resolve(results);
5114
+ };
3756
5115
  request.onerror = () => reject(request.error);
3757
5116
  tx.onerror = () => reject(tx.error);
3758
5117
  });
@@ -3761,11 +5120,31 @@ class IndexedDbQueueStorage {
3761
5120
  const db = await this.getDb();
3762
5121
  const tx = db.transaction(this.tableName, "readwrite");
3763
5122
  const store = tx.objectStore(this.tableName);
3764
- const request = store.clear();
5123
+ const index = store.index("queue_status");
5124
+ const prefixKeyValues = this.getPrefixKeyValues();
3765
5125
  return new Promise((resolve, reject) => {
3766
- request.onsuccess = () => resolve();
3767
- request.onerror = () => reject(request.error);
5126
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, "\uFFFF"]);
5127
+ const request = index.openCursor(keyRange);
5128
+ request.onsuccess = (event) => {
5129
+ const cursor = event.target.result;
5130
+ if (cursor) {
5131
+ const job = cursor.value;
5132
+ if (job.queue === this.queueName && this.matchesPrefixes(job)) {
5133
+ const deleteRequest = cursor.delete();
5134
+ deleteRequest.onsuccess = () => {
5135
+ cursor.continue();
5136
+ };
5137
+ deleteRequest.onerror = () => {
5138
+ cursor.continue();
5139
+ };
5140
+ } else {
5141
+ cursor.continue();
5142
+ }
5143
+ }
5144
+ };
5145
+ tx.oncomplete = () => resolve();
3768
5146
  tx.onerror = () => reject(tx.error);
5147
+ request.onerror = () => reject(request.error);
3769
5148
  });
3770
5149
  }
3771
5150
  async outputForInput(input) {
@@ -3773,10 +5152,23 @@ class IndexedDbQueueStorage {
3773
5152
  const db = await this.getDb();
3774
5153
  const tx = db.transaction(this.tableName, "readonly");
3775
5154
  const store = tx.objectStore(this.tableName);
3776
- const index = store.index("fingerprint_status");
3777
- const request = index.get([fingerprint, "COMPLETED" /* COMPLETED */]);
5155
+ const index = store.index("queue_fingerprint_status");
5156
+ const prefixKeyValues = this.getPrefixKeyValues();
5157
+ const request = index.get([
5158
+ ...prefixKeyValues,
5159
+ this.queueName,
5160
+ fingerprint,
5161
+ "COMPLETED" /* COMPLETED */
5162
+ ]);
3778
5163
  return new Promise((resolve, reject) => {
3779
- request.onsuccess = () => resolve(request.result?.output ?? null);
5164
+ request.onsuccess = () => {
5165
+ const job = request.result;
5166
+ if (job && this.matchesPrefixes(job)) {
5167
+ resolve(job.output ?? null);
5168
+ } else {
5169
+ resolve(null);
5170
+ }
5171
+ };
3780
5172
  request.onerror = () => reject(request.error);
3781
5173
  tx.onerror = () => reject(tx.error);
3782
5174
  });
@@ -3791,6 +5183,9 @@ class IndexedDbQueueStorage {
3791
5183
  await this.complete(job);
3792
5184
  }
3793
5185
  async delete(id) {
5186
+ const job = await this.get(id);
5187
+ if (!job)
5188
+ return;
3794
5189
  const db = await this.getDb();
3795
5190
  const tx = db.transaction(this.tableName, "readwrite");
3796
5191
  const store = tx.objectStore(this.tableName);
@@ -3805,15 +5200,17 @@ class IndexedDbQueueStorage {
3805
5200
  const db = await this.getDb();
3806
5201
  const tx = db.transaction(this.tableName, "readwrite");
3807
5202
  const store = tx.objectStore(this.tableName);
3808
- const index = store.index("status");
5203
+ const index = store.index("queue_status");
3809
5204
  const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
5205
+ const prefixKeyValues = this.getPrefixKeyValues();
5206
+ const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
3810
5207
  return new Promise((resolve, reject) => {
3811
- const request = index.openCursor();
5208
+ const request = index.openCursor(keyRange);
3812
5209
  request.onsuccess = (event) => {
3813
5210
  const cursor = event.target.result;
3814
5211
  if (cursor) {
3815
5212
  const job = cursor.value;
3816
- if (job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
5213
+ if (job.queue === this.queueName && this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
3817
5214
  cursor.delete();
3818
5215
  }
3819
5216
  cursor.continue();
@@ -3824,6 +5221,363 @@ class IndexedDbQueueStorage {
3824
5221
  request.onerror = () => reject(request.error);
3825
5222
  });
3826
5223
  }
5224
+ async getAllJobs() {
5225
+ const db = await this.getDb();
5226
+ const tx = db.transaction(this.tableName, "readonly");
5227
+ const store = tx.objectStore(this.tableName);
5228
+ const index = store.index("queue_status");
5229
+ const prefixKeyValues = this.getPrefixKeyValues();
5230
+ return new Promise((resolve, reject) => {
5231
+ const jobs = [];
5232
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, "\uFFFF"]);
5233
+ const request = index.openCursor(keyRange);
5234
+ request.onsuccess = (event) => {
5235
+ const cursor = event.target.result;
5236
+ if (cursor) {
5237
+ const job = cursor.value;
5238
+ if (job.queue === this.queueName && this.matchesPrefixes(job)) {
5239
+ jobs.push(job);
5240
+ }
5241
+ cursor.continue();
5242
+ }
5243
+ };
5244
+ tx.oncomplete = () => resolve(jobs);
5245
+ tx.onerror = () => reject(tx.error);
5246
+ request.onerror = () => reject(request.error);
5247
+ });
5248
+ }
5249
+ async getAllJobsWithFilter(prefixFilter) {
5250
+ const db = await this.getDb();
5251
+ const tx = db.transaction(this.tableName, "readonly");
5252
+ const store = tx.objectStore(this.tableName);
5253
+ return new Promise((resolve, reject) => {
5254
+ const jobs = [];
5255
+ const request = store.openCursor();
5256
+ request.onsuccess = (event) => {
5257
+ const cursor = event.target.result;
5258
+ if (cursor) {
5259
+ const job = cursor.value;
5260
+ if (job.queue !== this.queueName) {
5261
+ cursor.continue();
5262
+ return;
5263
+ }
5264
+ if (Object.keys(prefixFilter).length === 0) {
5265
+ jobs.push(job);
5266
+ } else {
5267
+ let matches = true;
5268
+ for (const [key, value] of Object.entries(prefixFilter)) {
5269
+ if (job[key] !== value) {
5270
+ matches = false;
5271
+ break;
5272
+ }
5273
+ }
5274
+ if (matches) {
5275
+ jobs.push(job);
5276
+ }
5277
+ }
5278
+ cursor.continue();
5279
+ }
5280
+ };
5281
+ tx.oncomplete = () => resolve(jobs);
5282
+ tx.onerror = () => reject(tx.error);
5283
+ request.onerror = () => reject(request.error);
5284
+ });
5285
+ }
5286
+ isCustomPrefixFilter(prefixFilter) {
5287
+ if (prefixFilter === undefined) {
5288
+ return false;
5289
+ }
5290
+ if (Object.keys(prefixFilter).length === 0) {
5291
+ return true;
5292
+ }
5293
+ const instanceKeys = Object.keys(this.prefixValues);
5294
+ const filterKeys = Object.keys(prefixFilter);
5295
+ if (instanceKeys.length !== filterKeys.length) {
5296
+ return true;
5297
+ }
5298
+ for (const key of instanceKeys) {
5299
+ if (this.prefixValues[key] !== prefixFilter[key]) {
5300
+ return true;
5301
+ }
5302
+ }
5303
+ return false;
5304
+ }
5305
+ getPollingManager() {
5306
+ if (!this.pollingManager) {
5307
+ this.pollingManager = new PollingSubscriptionManager(async () => {
5308
+ const jobs = await this.getAllJobs();
5309
+ return new Map(jobs.map((j) => [j.id, j]));
5310
+ }, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
5311
+ insert: (item) => ({ type: "INSERT", new: item }),
5312
+ update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
5313
+ delete: (item) => ({ type: "DELETE", old: item })
5314
+ });
5315
+ }
5316
+ return this.pollingManager;
5317
+ }
5318
+ subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
5319
+ let lastKnownJobs = new Map;
5320
+ let cancelled = false;
5321
+ const poll = async () => {
5322
+ if (cancelled)
5323
+ return;
5324
+ try {
5325
+ const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
5326
+ if (cancelled)
5327
+ return;
5328
+ const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
5329
+ for (const [id, job] of currentMap) {
5330
+ const old = lastKnownJobs.get(id);
5331
+ if (!old) {
5332
+ callback({ type: "INSERT", new: job });
5333
+ } else if (JSON.stringify(old) !== JSON.stringify(job)) {
5334
+ callback({ type: "UPDATE", old, new: job });
5335
+ }
5336
+ }
5337
+ for (const [id, job] of lastKnownJobs) {
5338
+ if (!currentMap.has(id)) {
5339
+ callback({ type: "DELETE", old: job });
5340
+ }
5341
+ }
5342
+ lastKnownJobs = currentMap;
5343
+ } catch {}
5344
+ };
5345
+ const intervalId = setInterval(poll, intervalMs);
5346
+ poll();
5347
+ return () => {
5348
+ cancelled = true;
5349
+ clearInterval(intervalId);
5350
+ };
5351
+ }
5352
+ subscribeToChanges(callback, options) {
5353
+ const intervalMs = options?.pollingIntervalMs ?? 1000;
5354
+ if (this.isCustomPrefixFilter(options?.prefixFilter)) {
5355
+ return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
5356
+ }
5357
+ const manager = this.getPollingManager();
5358
+ return manager.subscribe(callback, { intervalMs });
5359
+ }
5360
+ }
5361
+ // src/limiter/IndexedDbRateLimiterStorage.ts
5362
+ import { createServiceToken as createServiceToken28 } from "@workglow/util";
5363
+ var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken28("ratelimiter.storage.indexedDb");
5364
+
5365
+ class IndexedDbRateLimiterStorage {
5366
+ executionDb;
5367
+ nextAvailableDb;
5368
+ executionTableName;
5369
+ nextAvailableTableName;
5370
+ migrationOptions;
5371
+ prefixes;
5372
+ prefixValues;
5373
+ constructor(options = {}) {
5374
+ this.migrationOptions = options;
5375
+ this.prefixes = options.prefixes ?? [];
5376
+ this.prefixValues = options.prefixValues ?? {};
5377
+ if (this.prefixes.length > 0) {
5378
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
5379
+ this.executionTableName = `rate_limit_executions_${prefixNames}`;
5380
+ this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
5381
+ } else {
5382
+ this.executionTableName = "rate_limit_executions";
5383
+ this.nextAvailableTableName = "rate_limit_next_available";
5384
+ }
5385
+ }
5386
+ getPrefixColumnNames() {
5387
+ return this.prefixes.map((p) => p.name);
5388
+ }
5389
+ matchesPrefixes(record) {
5390
+ for (const [key, value] of Object.entries(this.prefixValues)) {
5391
+ if (record[key] !== value) {
5392
+ return false;
5393
+ }
5394
+ }
5395
+ return true;
5396
+ }
5397
+ getPrefixKeyValues() {
5398
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
5399
+ }
5400
+ async getExecutionDb() {
5401
+ if (this.executionDb)
5402
+ return this.executionDb;
5403
+ await this.setupDatabase();
5404
+ return this.executionDb;
5405
+ }
5406
+ async getNextAvailableDb() {
5407
+ if (this.nextAvailableDb)
5408
+ return this.nextAvailableDb;
5409
+ await this.setupDatabase();
5410
+ return this.nextAvailableDb;
5411
+ }
5412
+ async setupDatabase() {
5413
+ const prefixColumnNames = this.getPrefixColumnNames();
5414
+ const buildKeyPath = (basePath) => {
5415
+ return [...prefixColumnNames, ...basePath];
5416
+ };
5417
+ const executionIndexes = [
5418
+ {
5419
+ name: "queue_executed_at",
5420
+ keyPath: buildKeyPath(["queue_name", "executed_at"]),
5421
+ options: { unique: false }
5422
+ }
5423
+ ];
5424
+ this.executionDb = await ensureIndexedDbTable(this.executionTableName, "id", executionIndexes, this.migrationOptions);
5425
+ const nextAvailableIndexes = [
5426
+ {
5427
+ name: "queue_name",
5428
+ keyPath: buildKeyPath(["queue_name"]),
5429
+ options: { unique: true }
5430
+ }
5431
+ ];
5432
+ this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
5433
+ }
5434
+ async recordExecution(queueName) {
5435
+ const db = await this.getExecutionDb();
5436
+ const tx = db.transaction(this.executionTableName, "readwrite");
5437
+ const store = tx.objectStore(this.executionTableName);
5438
+ const record = {
5439
+ id: crypto.randomUUID(),
5440
+ queue_name: queueName,
5441
+ executed_at: new Date().toISOString()
5442
+ };
5443
+ for (const [key, value] of Object.entries(this.prefixValues)) {
5444
+ record[key] = value;
5445
+ }
5446
+ return new Promise((resolve, reject) => {
5447
+ const request = store.add(record);
5448
+ tx.oncomplete = () => resolve();
5449
+ tx.onerror = () => reject(tx.error);
5450
+ request.onerror = () => reject(request.error);
5451
+ });
5452
+ }
5453
+ async getExecutionCount(queueName, windowStartTime) {
5454
+ const db = await this.getExecutionDb();
5455
+ const tx = db.transaction(this.executionTableName, "readonly");
5456
+ const store = tx.objectStore(this.executionTableName);
5457
+ const index = store.index("queue_executed_at");
5458
+ const prefixKeyValues = this.getPrefixKeyValues();
5459
+ return new Promise((resolve, reject) => {
5460
+ let count = 0;
5461
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartTime], [...prefixKeyValues, queueName, "\uFFFF"], true, false);
5462
+ const request = index.openCursor(keyRange);
5463
+ request.onsuccess = (event) => {
5464
+ const cursor = event.target.result;
5465
+ if (cursor) {
5466
+ const record = cursor.value;
5467
+ if (this.matchesPrefixes(record)) {
5468
+ count++;
5469
+ }
5470
+ cursor.continue();
5471
+ }
5472
+ };
5473
+ tx.oncomplete = () => resolve(count);
5474
+ tx.onerror = () => reject(tx.error);
5475
+ request.onerror = () => reject(request.error);
5476
+ });
5477
+ }
5478
+ async getOldestExecutionAtOffset(queueName, offset) {
5479
+ const db = await this.getExecutionDb();
5480
+ const tx = db.transaction(this.executionTableName, "readonly");
5481
+ const store = tx.objectStore(this.executionTableName);
5482
+ const index = store.index("queue_executed_at");
5483
+ const prefixKeyValues = this.getPrefixKeyValues();
5484
+ return new Promise((resolve, reject) => {
5485
+ const executions = [];
5486
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, "\uFFFF"]);
5487
+ const request = index.openCursor(keyRange);
5488
+ request.onsuccess = (event) => {
5489
+ const cursor = event.target.result;
5490
+ if (cursor) {
5491
+ const record = cursor.value;
5492
+ if (this.matchesPrefixes(record)) {
5493
+ executions.push(record.executed_at);
5494
+ }
5495
+ cursor.continue();
5496
+ }
5497
+ };
5498
+ tx.oncomplete = () => {
5499
+ executions.sort();
5500
+ resolve(executions[offset]);
5501
+ };
5502
+ tx.onerror = () => reject(tx.error);
5503
+ request.onerror = () => reject(request.error);
5504
+ });
5505
+ }
5506
+ async getNextAvailableTime(queueName) {
5507
+ const db = await this.getNextAvailableDb();
5508
+ const tx = db.transaction(this.nextAvailableTableName, "readonly");
5509
+ const store = tx.objectStore(this.nextAvailableTableName);
5510
+ const prefixKeyValues = this.getPrefixKeyValues();
5511
+ const key = [...prefixKeyValues, queueName].join("_");
5512
+ return new Promise((resolve, reject) => {
5513
+ const request = store.get(key);
5514
+ request.onsuccess = () => {
5515
+ const record = request.result;
5516
+ if (record && this.matchesPrefixes(record)) {
5517
+ resolve(record.next_available_at);
5518
+ } else {
5519
+ resolve(undefined);
5520
+ }
5521
+ };
5522
+ request.onerror = () => reject(request.error);
5523
+ tx.onerror = () => reject(tx.error);
5524
+ });
5525
+ }
5526
+ async setNextAvailableTime(queueName, nextAvailableAt) {
5527
+ const db = await this.getNextAvailableDb();
5528
+ const tx = db.transaction(this.nextAvailableTableName, "readwrite");
5529
+ const store = tx.objectStore(this.nextAvailableTableName);
5530
+ const prefixKeyValues = this.getPrefixKeyValues();
5531
+ const key = [...prefixKeyValues, queueName].join("_");
5532
+ const record = {
5533
+ queue_name: queueName,
5534
+ next_available_at: nextAvailableAt
5535
+ };
5536
+ for (const [k, value] of Object.entries(this.prefixValues)) {
5537
+ record[k] = value;
5538
+ }
5539
+ record[this.getPrefixColumnNames().concat(["queue_name"]).join("_")] = key;
5540
+ return new Promise((resolve, reject) => {
5541
+ const request = store.put(record);
5542
+ tx.oncomplete = () => resolve();
5543
+ tx.onerror = () => reject(tx.error);
5544
+ request.onerror = () => reject(request.error);
5545
+ });
5546
+ }
5547
+ async clear(queueName) {
5548
+ const execDb = await this.getExecutionDb();
5549
+ const execTx = execDb.transaction(this.executionTableName, "readwrite");
5550
+ const execStore = execTx.objectStore(this.executionTableName);
5551
+ const execIndex = execStore.index("queue_executed_at");
5552
+ const prefixKeyValues = this.getPrefixKeyValues();
5553
+ await new Promise((resolve, reject) => {
5554
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, "\uFFFF"]);
5555
+ const request = execIndex.openCursor(keyRange);
5556
+ request.onsuccess = (event) => {
5557
+ const cursor = event.target.result;
5558
+ if (cursor) {
5559
+ const record = cursor.value;
5560
+ if (this.matchesPrefixes(record)) {
5561
+ cursor.delete();
5562
+ }
5563
+ cursor.continue();
5564
+ }
5565
+ };
5566
+ execTx.oncomplete = () => resolve();
5567
+ execTx.onerror = () => reject(execTx.error);
5568
+ request.onerror = () => reject(request.error);
5569
+ });
5570
+ const nextDb = await this.getNextAvailableDb();
5571
+ const nextTx = nextDb.transaction(this.nextAvailableTableName, "readwrite");
5572
+ const nextStore = nextTx.objectStore(this.nextAvailableTableName);
5573
+ const key = [...prefixKeyValues, queueName].join("_");
5574
+ await new Promise((resolve, reject) => {
5575
+ const request = nextStore.delete(key);
5576
+ nextTx.oncomplete = () => resolve();
5577
+ nextTx.onerror = () => reject(nextTx.error);
5578
+ request.onerror = () => reject(request.error);
5579
+ });
5580
+ }
3827
5581
  }
3828
5582
  export {
3829
5583
  ensureIndexedDbTable,
@@ -3831,22 +5585,29 @@ export {
3831
5585
  TabularRepository,
3832
5586
  TABULAR_REPOSITORY,
3833
5587
  SupabaseTabularRepository,
5588
+ SupabaseRateLimiterStorage,
3834
5589
  SupabaseQueueStorage,
3835
5590
  SupabaseKvRepository,
3836
5591
  SqliteTabularRepository,
5592
+ SqliteRateLimiterStorage,
3837
5593
  SqliteQueueStorage,
3838
5594
  SqliteKvRepository,
3839
5595
  SUPABASE_TABULAR_REPOSITORY,
5596
+ SUPABASE_RATE_LIMITER_STORAGE,
3840
5597
  SUPABASE_QUEUE_STORAGE,
3841
5598
  SUPABASE_KV_REPOSITORY,
3842
5599
  SQLITE_TABULAR_REPOSITORY,
5600
+ SQLITE_RATE_LIMITER_STORAGE,
3843
5601
  SQLITE_QUEUE_STORAGE,
3844
5602
  SQLITE_KV_REPOSITORY,
5603
+ RATE_LIMITER_STORAGE,
3845
5604
  QUEUE_STORAGE,
3846
5605
  PostgresTabularRepository,
5606
+ PostgresRateLimiterStorage,
3847
5607
  PostgresQueueStorage,
3848
5608
  PostgresKvRepository,
3849
5609
  POSTGRES_TABULAR_REPOSITORY,
5610
+ POSTGRES_RATE_LIMITER_STORAGE,
3850
5611
  POSTGRES_QUEUE_STORAGE,
3851
5612
  POSTGRES_KV_REPOSITORY,
3852
5613
  MEMORY_TABULAR_REPOSITORY,
@@ -3856,12 +5617,16 @@ export {
3856
5617
  KV_REPOSITORY,
3857
5618
  JobStatus,
3858
5619
  IndexedDbTabularRepository,
5620
+ IndexedDbRateLimiterStorage,
3859
5621
  IndexedDbQueueStorage,
3860
5622
  IndexedDbKvRepository,
3861
5623
  InMemoryTabularRepository,
5624
+ InMemoryRateLimiterStorage,
3862
5625
  InMemoryQueueStorage,
3863
5626
  InMemoryKvRepository,
5627
+ IN_MEMORY_RATE_LIMITER_STORAGE,
3864
5628
  IN_MEMORY_QUEUE_STORAGE,
5629
+ INDEXED_DB_RATE_LIMITER_STORAGE,
3865
5630
  INDEXED_DB_QUEUE_STORAGE,
3866
5631
  IDB_TABULAR_REPOSITORY,
3867
5632
  IDB_KV_REPOSITORY,
@@ -3877,4 +5642,4 @@ export {
3877
5642
  CACHED_TABULAR_REPOSITORY
3878
5643
  };
3879
5644
 
3880
- //# debugId=9E46692D2791266264756E2164756E21
5645
+ //# debugId=DBE095F06C6029D864756E2164756E21