@workglow/storage 0.0.56 → 0.0.58

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/browser.js +1266 -121
  2. package/dist/browser.js.map +15 -10
  3. package/dist/bun.js +2017 -252
  4. package/dist/bun.js.map +19 -12
  5. package/dist/common-server.d.ts +4 -0
  6. package/dist/common-server.d.ts.map +1 -1
  7. package/dist/common.d.ts +2 -0
  8. package/dist/common.d.ts.map +1 -1
  9. package/dist/limiter/IRateLimiterStorage.d.ts +81 -0
  10. package/dist/limiter/IRateLimiterStorage.d.ts.map +1 -0
  11. package/dist/limiter/InMemoryRateLimiterStorage.d.ts +32 -0
  12. package/dist/limiter/InMemoryRateLimiterStorage.d.ts.map +1 -0
  13. package/dist/limiter/IndexedDbRateLimiterStorage.d.ts +52 -0
  14. package/dist/limiter/IndexedDbRateLimiterStorage.d.ts.map +1 -0
  15. package/dist/limiter/PostgresRateLimiterStorage.d.ts +54 -0
  16. package/dist/limiter/PostgresRateLimiterStorage.d.ts.map +1 -0
  17. package/dist/limiter/SqliteRateLimiterStorage.d.ts +53 -0
  18. package/dist/limiter/SqliteRateLimiterStorage.d.ts.map +1 -0
  19. package/dist/limiter/SupabaseRateLimiterStorage.d.ts +53 -0
  20. package/dist/limiter/SupabaseRateLimiterStorage.d.ts.map +1 -0
  21. package/dist/node.js +2017 -252
  22. package/dist/node.js.map +19 -12
  23. package/dist/queue/IQueueStorage.d.ts +72 -1
  24. package/dist/queue/IQueueStorage.d.ts.map +1 -1
  25. package/dist/queue/InMemoryQueueStorage.d.ts +44 -11
  26. package/dist/queue/InMemoryQueueStorage.d.ts.map +1 -1
  27. package/dist/queue/IndexedDbQueueStorage.d.ts +70 -5
  28. package/dist/queue/IndexedDbQueueStorage.d.ts.map +1 -1
  29. package/dist/queue/PostgresQueueStorage.d.ts +80 -8
  30. package/dist/queue/PostgresQueueStorage.d.ts.map +1 -1
  31. package/dist/queue/SqliteQueueStorage.d.ts +90 -34
  32. package/dist/queue/SqliteQueueStorage.d.ts.map +1 -1
  33. package/dist/queue/SupabaseQueueStorage.d.ts +98 -4
  34. package/dist/queue/SupabaseQueueStorage.d.ts.map +1 -1
  35. package/dist/tabular/ITabularRepository.d.ts +18 -0
  36. package/dist/tabular/ITabularRepository.d.ts.map +1 -1
  37. package/dist/tabular/InMemoryTabularRepository.d.ts +9 -1
  38. package/dist/tabular/InMemoryTabularRepository.d.ts.map +1 -1
  39. package/dist/tabular/SupabaseTabularRepository.d.ts +21 -1
  40. package/dist/tabular/SupabaseTabularRepository.d.ts.map +1 -1
  41. package/dist/tabular/TabularRepository.d.ts +10 -1
  42. package/dist/tabular/TabularRepository.d.ts.map +1 -1
  43. package/dist/util/PollingSubscriptionManager.d.ts +112 -0
  44. package/dist/util/PollingSubscriptionManager.d.ts.map +1 -0
  45. package/package.json +5 -5
package/dist/node.js CHANGED
@@ -106,6 +106,9 @@ class TabularRepository {
106
106
  waitOn(name) {
107
107
  return this.events.waitOn(name);
108
108
  }
109
+ subscribeToChanges(_callback) {
110
+ throw new Error(`subscribeToChanges is not implemented for ${this.constructor.name}. ` + `Use InMemoryTabularRepository or SupabaseTabularRepository for subscription support.`);
111
+ }
109
112
  primaryKeyColumns() {
110
113
  const columns = [];
111
114
  for (const key of Object.keys(this.primaryKeySchema.properties)) {
@@ -273,12 +276,30 @@ class InMemoryTabularRepository extends TabularRepository {
273
276
  return false;
274
277
  }
275
278
  });
276
- for (const [id, _] of entriesToDelete) {
279
+ for (const [id, entity] of entriesToDelete) {
277
280
  this.values.delete(id);
281
+ const { key } = this.separateKeyValueFromCombined(entity);
282
+ this.events.emit("delete", key);
278
283
  }
279
- if (Array.from(entriesToDelete).length > 0) {
280
- this.events.emit("delete", column);
281
- }
284
+ }
285
+ subscribeToChanges(callback) {
286
+ const handlePut = (entity) => {
287
+ callback({ type: "UPDATE", new: entity });
288
+ };
289
+ const handleDelete = (_key) => {
290
+ callback({ type: "DELETE" });
291
+ };
292
+ const handleClearAll = () => {
293
+ callback({ type: "DELETE" });
294
+ };
295
+ this.events.on("put", handlePut);
296
+ this.events.on("delete", handleDelete);
297
+ this.events.on("clearall", handleClearAll);
298
+ return () => {
299
+ this.events.off("put", handlePut);
300
+ this.events.off("delete", handleDelete);
301
+ this.events.off("clearall", handleClearAll);
302
+ };
282
303
  }
283
304
  destroy() {
284
305
  this.values.clear();
@@ -540,7 +561,7 @@ class InMemoryKvRepository extends KvViaTabularRepository {
540
561
  }
541
562
  }
542
563
  // src/queue/InMemoryQueueStorage.ts
543
- import { createServiceToken as createServiceToken7, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
564
+ import { createServiceToken as createServiceToken7, EventEmitter as EventEmitter3, makeFingerprint as makeFingerprint4, sleep, uuid4 } from "@workglow/util";
544
565
 
545
566
  // src/queue/IQueueStorage.ts
546
567
  import { createServiceToken as createServiceToken6 } from "@workglow/util";
@@ -560,63 +581,88 @@ var IN_MEMORY_QUEUE_STORAGE = createServiceToken7("jobqueue.storage.inMemory");
560
581
 
561
582
  class InMemoryQueueStorage {
562
583
  queueName;
563
- constructor(queueName) {
584
+ prefixValues;
585
+ events = new EventEmitter3;
586
+ constructor(queueName, options) {
564
587
  this.queueName = queueName;
565
588
  this.jobQueue = [];
589
+ this.prefixValues = options?.prefixValues ?? {};
566
590
  }
567
591
  jobQueue;
592
+ matchesPrefixes(job) {
593
+ for (const [key, value] of Object.entries(this.prefixValues)) {
594
+ if (job[key] !== value) {
595
+ return false;
596
+ }
597
+ }
598
+ return true;
599
+ }
568
600
  pendingQueue() {
569
601
  const now = new Date().toISOString();
570
- return this.jobQueue.filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
602
+ return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === "PENDING" /* PENDING */).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
571
603
  }
572
604
  async add(job) {
573
605
  await sleep(0);
574
606
  const now = new Date().toISOString();
575
- job.id = job.id ?? uuid4();
576
- job.job_run_id = job.job_run_id ?? uuid4();
577
- job.queue = this.queueName;
578
- job.fingerprint = await makeFingerprint4(job.input);
579
- job.status = "PENDING" /* PENDING */;
580
- job.progress = 0;
581
- job.progress_message = "";
582
- job.progress_details = null;
583
- job.created_at = now;
584
- job.run_after = now;
585
- this.jobQueue.push(job);
586
- return job.id;
607
+ const jobWithPrefixes = job;
608
+ jobWithPrefixes.id = jobWithPrefixes.id ?? uuid4();
609
+ jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid4();
610
+ jobWithPrefixes.queue = this.queueName;
611
+ jobWithPrefixes.fingerprint = await makeFingerprint4(jobWithPrefixes.input);
612
+ jobWithPrefixes.status = "PENDING" /* PENDING */;
613
+ jobWithPrefixes.progress = 0;
614
+ jobWithPrefixes.progress_message = "";
615
+ jobWithPrefixes.progress_details = null;
616
+ jobWithPrefixes.created_at = now;
617
+ jobWithPrefixes.run_after = now;
618
+ for (const [key, value] of Object.entries(this.prefixValues)) {
619
+ jobWithPrefixes[key] = value;
620
+ }
621
+ this.jobQueue.push(jobWithPrefixes);
622
+ this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
623
+ return jobWithPrefixes.id;
587
624
  }
588
625
  async get(id) {
589
626
  await sleep(0);
590
- return this.jobQueue.find((j) => j.id === id);
627
+ const job = this.jobQueue.find((j) => j.id === id);
628
+ if (job && this.matchesPrefixes(job)) {
629
+ return job;
630
+ }
631
+ return;
591
632
  }
592
633
  async peek(status = "PENDING" /* PENDING */, num = 100) {
593
634
  await sleep(0);
594
635
  num = Number(num) || 100;
595
- return this.jobQueue.sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
636
+ return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
596
637
  }
597
- async next() {
638
+ async next(workerId) {
598
639
  await sleep(0);
599
640
  const top = this.pendingQueue();
600
641
  const job = top[0];
601
642
  if (job) {
643
+ const oldJob = { ...job };
602
644
  job.status = "PROCESSING" /* PROCESSING */;
603
645
  job.last_ran_at = new Date().toISOString();
646
+ job.worker_id = workerId ?? null;
647
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
604
648
  return job;
605
649
  }
606
650
  }
607
651
  async size(status = "PENDING" /* PENDING */) {
608
652
  await sleep(0);
609
- return this.jobQueue.filter((j) => j.status === status).length;
653
+ return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
610
654
  }
611
655
  async saveProgress(id, progress, message, details) {
612
656
  await sleep(0);
613
- const job = this.jobQueue.find((j) => j.id === id);
657
+ const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
614
658
  if (!job) {
615
659
  throw new Error(`Job ${id} not found`);
616
660
  }
661
+ const oldJob = { ...job };
617
662
  job.progress = progress;
618
663
  job.progress_message = message;
619
664
  job.progress_details = details;
665
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
620
666
  }
621
667
  async complete(job) {
622
668
  await sleep(0);
@@ -626,44 +672,149 @@ class InMemoryQueueStorage {
626
672
  const currentAttempts = existing?.run_attempts ?? 0;
627
673
  job.run_attempts = currentAttempts + 1;
628
674
  this.jobQueue[index] = job;
675
+ this.events.emit("change", { type: "UPDATE", old: existing, new: job });
629
676
  }
630
677
  }
631
678
  async abort(id) {
632
679
  await sleep(0);
633
- const job = this.jobQueue.find((j) => j.id === id);
680
+ const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
634
681
  if (job) {
682
+ const oldJob = { ...job };
635
683
  job.status = "ABORTING" /* ABORTING */;
684
+ this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
636
685
  }
637
686
  }
638
687
  async getByRunId(runId) {
639
688
  await sleep(0);
640
- return this.jobQueue.filter((job) => job.job_run_id === runId);
689
+ return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
641
690
  }
642
691
  async deleteAll() {
643
692
  await sleep(0);
644
- this.jobQueue = [];
693
+ const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
694
+ this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
695
+ for (const job of deletedJobs) {
696
+ this.events.emit("change", { type: "DELETE", old: job });
697
+ }
645
698
  }
646
699
  async outputForInput(input) {
647
700
  await sleep(0);
648
701
  const fingerprint = await makeFingerprint4(input);
649
- return this.jobQueue.find((j) => j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
702
+ return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === "COMPLETED" /* COMPLETED */)?.output ?? null;
650
703
  }
651
704
  async delete(id) {
652
705
  await sleep(0);
653
- this.jobQueue = this.jobQueue.filter((job) => job.id !== id);
706
+ const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
707
+ this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
708
+ if (deletedJob) {
709
+ this.events.emit("change", { type: "DELETE", old: deletedJob });
710
+ }
654
711
  }
655
712
  async deleteJobsByStatusAndAge(status, olderThanMs) {
656
713
  await sleep(0);
657
714
  const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
658
- this.jobQueue = this.jobQueue.filter((job) => job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
715
+ const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
716
+ this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
717
+ for (const job of deletedJobs) {
718
+ this.events.emit("change", { type: "DELETE", old: job });
719
+ }
720
+ }
721
+ async setupDatabase() {}
722
+ matchesPrefixFilter(job, prefixFilter) {
723
+ if (prefixFilter && Object.keys(prefixFilter).length === 0) {
724
+ return true;
725
+ }
726
+ const filterValues = prefixFilter ?? this.prefixValues;
727
+ if (Object.keys(filterValues).length === 0) {
728
+ return true;
729
+ }
730
+ const jobWithPrefixes = job;
731
+ for (const [key, value] of Object.entries(filterValues)) {
732
+ if (jobWithPrefixes[key] !== value) {
733
+ return false;
734
+ }
735
+ }
736
+ return true;
737
+ }
738
+ subscribeToChanges(callback, options) {
739
+ const prefixFilter = options?.prefixFilter;
740
+ const filteredCallback = (change) => {
741
+ const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
742
+ const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
743
+ if (!newMatches && !oldMatches) {
744
+ return;
745
+ }
746
+ callback(change);
747
+ };
748
+ return this.events.subscribe("change", filteredCallback);
749
+ }
750
+ }
751
+ // src/limiter/IRateLimiterStorage.ts
752
+ import { createServiceToken as createServiceToken8 } from "@workglow/util";
753
+ var RATE_LIMITER_STORAGE = createServiceToken8("ratelimiter.storage");
754
+ // src/limiter/InMemoryRateLimiterStorage.ts
755
+ import { createServiceToken as createServiceToken9, sleep as sleep2 } from "@workglow/util";
756
+ var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken9("ratelimiter.storage.inMemory");
757
+
758
+ class InMemoryRateLimiterStorage {
759
+ prefixValues;
760
+ executions = new Map;
761
+ nextAvailableTimes = new Map;
762
+ constructor(options) {
763
+ this.prefixValues = options?.prefixValues ?? {};
764
+ }
765
+ makeKey(queueName) {
766
+ const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
767
+ return prefixPart ? `${prefixPart}|${queueName}` : queueName;
659
768
  }
660
769
  async setupDatabase() {}
770
+ async recordExecution(queueName) {
771
+ await sleep2(0);
772
+ const key = this.makeKey(queueName);
773
+ const executions = this.executions.get(key) ?? [];
774
+ executions.push({
775
+ queueName,
776
+ executedAt: new Date
777
+ });
778
+ this.executions.set(key, executions);
779
+ }
780
+ async getExecutionCount(queueName, windowStartTime) {
781
+ await sleep2(0);
782
+ const key = this.makeKey(queueName);
783
+ const executions = this.executions.get(key) ?? [];
784
+ const windowStart = new Date(windowStartTime);
785
+ return executions.filter((e) => e.executedAt > windowStart).length;
786
+ }
787
+ async getOldestExecutionAtOffset(queueName, offset) {
788
+ await sleep2(0);
789
+ const key = this.makeKey(queueName);
790
+ const executions = this.executions.get(key) ?? [];
791
+ const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
792
+ const execution = sorted[offset];
793
+ return execution?.executedAt.toISOString();
794
+ }
795
+ async getNextAvailableTime(queueName) {
796
+ await sleep2(0);
797
+ const key = this.makeKey(queueName);
798
+ const time = this.nextAvailableTimes.get(key);
799
+ return time?.toISOString();
800
+ }
801
+ async setNextAvailableTime(queueName, nextAvailableAt) {
802
+ await sleep2(0);
803
+ const key = this.makeKey(queueName);
804
+ this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
805
+ }
806
+ async clear(queueName) {
807
+ await sleep2(0);
808
+ const key = this.makeKey(queueName);
809
+ this.executions.delete(key);
810
+ this.nextAvailableTimes.delete(key);
811
+ }
661
812
  }
662
813
  // src/tabular/FsFolderTabularRepository.ts
663
- import { createServiceToken as createServiceToken8, sleep as sleep2 } from "@workglow/util";
814
+ import { createServiceToken as createServiceToken10, sleep as sleep3 } from "@workglow/util";
664
815
  import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
665
816
  import path from "node:path";
666
- var FS_FOLDER_TABULAR_REPOSITORY = createServiceToken8("storage.tabularRepository.fsFolder");
817
+ var FS_FOLDER_TABULAR_REPOSITORY = createServiceToken10("storage.tabularRepository.fsFolder");
667
818
 
668
819
  class FsFolderTabularRepository extends TabularRepository {
669
820
  folderPath;
@@ -688,7 +839,7 @@ class FsFolderTabularRepository extends TabularRepository {
688
839
  await writeFile(filePath, JSON.stringify(entity));
689
840
  } catch (error) {
690
841
  try {
691
- await sleep2(1);
842
+ await sleep3(1);
692
843
  await writeFile(filePath, JSON.stringify(entity));
693
844
  } catch (error2) {
694
845
  console.error("Error writing file", filePath, error2);
@@ -772,7 +923,7 @@ class FsFolderTabularRepository extends TabularRepository {
772
923
  }
773
924
  }
774
925
  // src/tabular/PostgresTabularRepository.ts
775
- import { createServiceToken as createServiceToken9 } from "@workglow/util";
926
+ import { createServiceToken as createServiceToken11 } from "@workglow/util";
776
927
 
777
928
  // src/tabular/BaseSqlTabularRepository.ts
778
929
  class BaseSqlTabularRepository extends TabularRepository {
@@ -958,7 +1109,7 @@ class BaseSqlTabularRepository extends TabularRepository {
958
1109
  }
959
1110
 
960
1111
  // src/tabular/PostgresTabularRepository.ts
961
- var POSTGRES_TABULAR_REPOSITORY = createServiceToken9("storage.tabularRepository.postgres");
1112
+ var POSTGRES_TABULAR_REPOSITORY = createServiceToken11("storage.tabularRepository.postgres");
962
1113
 
963
1114
  class PostgresTabularRepository extends BaseSqlTabularRepository {
964
1115
  db;
@@ -1316,8 +1467,8 @@ class PostgresTabularRepository extends BaseSqlTabularRepository {
1316
1467
  }
1317
1468
  // src/tabular/SqliteTabularRepository.ts
1318
1469
  import { Sqlite } from "@workglow/sqlite";
1319
- import { createServiceToken as createServiceToken10 } from "@workglow/util";
1320
- var SQLITE_TABULAR_REPOSITORY = createServiceToken10("storage.tabularRepository.sqlite");
1470
+ import { createServiceToken as createServiceToken12 } from "@workglow/util";
1471
+ var SQLITE_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.sqlite");
1321
1472
  var Database = Sqlite.Database;
1322
1473
 
1323
1474
  class SqliteTabularRepository extends BaseSqlTabularRepository {
@@ -1594,11 +1745,12 @@ class SqliteTabularRepository extends BaseSqlTabularRepository {
1594
1745
  }
1595
1746
  }
1596
1747
  // src/tabular/SupabaseTabularRepository.ts
1597
- import { createServiceToken as createServiceToken11 } from "@workglow/util";
1598
- var SUPABASE_TABULAR_REPOSITORY = createServiceToken11("storage.tabularRepository.supabase");
1748
+ import { createServiceToken as createServiceToken13 } from "@workglow/util";
1749
+ var SUPABASE_TABULAR_REPOSITORY = createServiceToken13("storage.tabularRepository.supabase");
1599
1750
 
1600
1751
  class SupabaseTabularRepository extends BaseSqlTabularRepository {
1601
1752
  client;
1753
+ realtimeChannel = null;
1602
1754
  constructor(client, table = "tabular_store", schema, primaryKeyNames, indexes = []) {
1603
1755
  super(table, schema, primaryKeyNames, indexes);
1604
1756
  this.client = client;
@@ -1971,10 +2123,44 @@ class SupabaseTabularRepository extends BaseSqlTabularRepository {
1971
2123
  throw error;
1972
2124
  this.events.emit("delete", column);
1973
2125
  }
2126
+ convertRealtimeRow(row) {
2127
+ const entity = { ...row };
2128
+ for (const key in this.schema.properties) {
2129
+ entity[key] = this.sqlToJsValue(key, row[key]);
2130
+ }
2131
+ return entity;
2132
+ }
2133
+ subscribeToChanges(callback) {
2134
+ const channelName = `tabular-${this.table}-${Date.now()}`;
2135
+ this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
2136
+ event: "*",
2137
+ schema: "public",
2138
+ table: this.table
2139
+ }, (payload) => {
2140
+ const change = {
2141
+ type: payload.eventType.toUpperCase(),
2142
+ old: payload.old && Object.keys(payload.old).length > 0 ? this.convertRealtimeRow(payload.old) : undefined,
2143
+ new: payload.new && Object.keys(payload.new).length > 0 ? this.convertRealtimeRow(payload.new) : undefined
2144
+ };
2145
+ callback(change);
2146
+ }).subscribe();
2147
+ return () => {
2148
+ if (this.realtimeChannel) {
2149
+ this.client.removeChannel(this.realtimeChannel);
2150
+ this.realtimeChannel = null;
2151
+ }
2152
+ };
2153
+ }
2154
+ destroy() {
2155
+ if (this.realtimeChannel) {
2156
+ this.client.removeChannel(this.realtimeChannel);
2157
+ this.realtimeChannel = null;
2158
+ }
2159
+ }
1974
2160
  }
1975
2161
  // src/kv/FsFolderJsonKvRepository.ts
1976
- import { createServiceToken as createServiceToken12 } from "@workglow/util";
1977
- var FS_FOLDER_JSON_KV_REPOSITORY = createServiceToken12("storage.kvRepository.fsFolderJson");
2162
+ import { createServiceToken as createServiceToken14 } from "@workglow/util";
2163
+ var FS_FOLDER_JSON_KV_REPOSITORY = createServiceToken14("storage.kvRepository.fsFolderJson");
1978
2164
 
1979
2165
  class FsFolderJsonKvRepository extends KvViaTabularRepository {
1980
2166
  folderPath;
@@ -1986,10 +2172,10 @@ class FsFolderJsonKvRepository extends KvViaTabularRepository {
1986
2172
  }
1987
2173
  }
1988
2174
  // src/kv/FsFolderKvRepository.ts
1989
- import { createServiceToken as createServiceToken13 } from "@workglow/util";
2175
+ import { createServiceToken as createServiceToken15 } from "@workglow/util";
1990
2176
  import { mkdir as mkdir2, readFile as readFile2, rm as rm2, unlink, writeFile as writeFile2 } from "fs/promises";
1991
2177
  import path2 from "path";
1992
- var FS_FOLDER_KV_REPOSITORY = createServiceToken13("storage.kvRepository.fsFolder");
2178
+ var FS_FOLDER_KV_REPOSITORY = createServiceToken15("storage.kvRepository.fsFolder");
1993
2179
 
1994
2180
  class FsFolderKvRepository extends KvRepository {
1995
2181
  folderPath;
@@ -2066,8 +2252,8 @@ class FsFolderKvRepository extends KvRepository {
2066
2252
  }
2067
2253
  }
2068
2254
  // src/kv/PostgresKvRepository.ts
2069
- import { createServiceToken as createServiceToken14 } from "@workglow/util";
2070
- var POSTGRES_KV_REPOSITORY = createServiceToken14("storage.kvRepository.postgres");
2255
+ import { createServiceToken as createServiceToken16 } from "@workglow/util";
2256
+ var POSTGRES_KV_REPOSITORY = createServiceToken16("storage.kvRepository.postgres");
2071
2257
 
2072
2258
  class PostgresKvRepository extends KvViaTabularRepository {
2073
2259
  db;
@@ -2081,8 +2267,8 @@ class PostgresKvRepository extends KvViaTabularRepository {
2081
2267
  }
2082
2268
  }
2083
2269
  // src/kv/SqliteKvRepository.ts
2084
- import { createServiceToken as createServiceToken15 } from "@workglow/util";
2085
- var SQLITE_KV_REPOSITORY = createServiceToken15("storage.kvRepository.sqlite");
2270
+ import { createServiceToken as createServiceToken17 } from "@workglow/util";
2271
+ var SQLITE_KV_REPOSITORY = createServiceToken17("storage.kvRepository.sqlite");
2086
2272
 
2087
2273
  class SqliteKvRepository extends KvViaTabularRepository {
2088
2274
  db;
@@ -2096,8 +2282,8 @@ class SqliteKvRepository extends KvViaTabularRepository {
2096
2282
  }
2097
2283
  }
2098
2284
  // src/kv/SupabaseKvRepository.ts
2099
- import { createServiceToken as createServiceToken16 } from "@workglow/util";
2100
- var SUPABASE_KV_REPOSITORY = createServiceToken16("storage.kvRepository.supabase");
2285
+ import { createServiceToken as createServiceToken18 } from "@workglow/util";
2286
+ var SUPABASE_KV_REPOSITORY = createServiceToken18("storage.kvRepository.supabase");
2101
2287
 
2102
2288
  class SupabaseKvRepository extends KvViaTabularRepository {
2103
2289
  client;
@@ -2111,15 +2297,169 @@ class SupabaseKvRepository extends KvViaTabularRepository {
2111
2297
  }
2112
2298
  }
2113
2299
  // src/queue/PostgresQueueStorage.ts
2114
- import { createServiceToken as createServiceToken17, makeFingerprint as makeFingerprint5, uuid4 as uuid42 } from "@workglow/util";
2115
- var POSTGRES_QUEUE_STORAGE = createServiceToken17("jobqueue.storage.postgres");
2300
+ import { createServiceToken as createServiceToken19, makeFingerprint as makeFingerprint5, uuid4 as uuid42 } from "@workglow/util";
2301
+
2302
+ // src/util/PollingSubscriptionManager.ts
2303
+ class PollingSubscriptionManager {
2304
+ intervals = new Map;
2305
+ lastKnownState = new Map;
2306
+ initialized = false;
2307
+ fetchState;
2308
+ compareItems;
2309
+ payloadFactory;
2310
+ defaultIntervalMs;
2311
+ constructor(fetchState, compareItems, payloadFactory, options) {
2312
+ this.fetchState = fetchState;
2313
+ this.compareItems = compareItems;
2314
+ this.payloadFactory = payloadFactory;
2315
+ this.defaultIntervalMs = options?.defaultIntervalMs ?? 1000;
2316
+ }
2317
+ subscribe(callback, options) {
2318
+ const interval = options?.intervalMs ?? this.defaultIntervalMs;
2319
+ const subscription = {
2320
+ callback,
2321
+ intervalMs: interval
2322
+ };
2323
+ let intervalGroup = this.intervals.get(interval);
2324
+ if (!intervalGroup) {
2325
+ const subscribers = new Set;
2326
+ const intervalId = setInterval(() => this.poll(subscribers), interval);
2327
+ intervalGroup = { intervalId, subscribers };
2328
+ this.intervals.set(interval, intervalGroup);
2329
+ if (!this.initialized) {
2330
+ this.initialized = true;
2331
+ this.initAndPoll(subscribers, subscription);
2332
+ } else {
2333
+ this.pollForNewSubscriber(subscription);
2334
+ }
2335
+ } else {
2336
+ this.pollForNewSubscriber(subscription);
2337
+ }
2338
+ intervalGroup.subscribers.add(subscription);
2339
+ return () => {
2340
+ const group = this.intervals.get(interval);
2341
+ if (group) {
2342
+ group.subscribers.delete(subscription);
2343
+ if (group.subscribers.size === 0) {
2344
+ clearInterval(group.intervalId);
2345
+ this.intervals.delete(interval);
2346
+ }
2347
+ }
2348
+ };
2349
+ }
2350
+ async initAndPoll(subscribers, newSubscription) {
2351
+ try {
2352
+ this.lastKnownState = await this.fetchState();
2353
+ for (const [, item] of this.lastKnownState) {
2354
+ const payload = this.payloadFactory.insert(item);
2355
+ try {
2356
+ newSubscription.callback(payload);
2357
+ } catch {}
2358
+ }
2359
+ } catch {}
2360
+ }
2361
+ pollForNewSubscriber(subscription) {
2362
+ for (const [, item] of this.lastKnownState) {
2363
+ const payload = this.payloadFactory.insert(item);
2364
+ try {
2365
+ subscription.callback(payload);
2366
+ } catch {}
2367
+ }
2368
+ }
2369
+ async poll(subscribers) {
2370
+ if (subscribers.size === 0)
2371
+ return;
2372
+ try {
2373
+ const currentState = await this.fetchState();
2374
+ const changes = [];
2375
+ for (const [key, item] of currentState) {
2376
+ const oldItem = this.lastKnownState.get(key);
2377
+ if (!oldItem) {
2378
+ changes.push(this.payloadFactory.insert(item));
2379
+ } else if (!this.compareItems(oldItem, item)) {
2380
+ changes.push(this.payloadFactory.update(oldItem, item));
2381
+ }
2382
+ }
2383
+ for (const [key, item] of this.lastKnownState) {
2384
+ if (!currentState.has(key)) {
2385
+ changes.push(this.payloadFactory.delete(item));
2386
+ }
2387
+ }
2388
+ this.lastKnownState = currentState;
2389
+ for (const change of changes) {
2390
+ for (const sub of subscribers) {
2391
+ try {
2392
+ sub.callback(change);
2393
+ } catch {}
2394
+ }
2395
+ }
2396
+ } catch {}
2397
+ }
2398
+ get subscriptionCount() {
2399
+ let count = 0;
2400
+ for (const group of this.intervals.values()) {
2401
+ count += group.subscribers.size;
2402
+ }
2403
+ return count;
2404
+ }
2405
+ get hasSubscriptions() {
2406
+ return this.intervals.size > 0;
2407
+ }
2408
+ destroy() {
2409
+ for (const group of this.intervals.values()) {
2410
+ clearInterval(group.intervalId);
2411
+ }
2412
+ this.intervals.clear();
2413
+ this.lastKnownState.clear();
2414
+ this.initialized = false;
2415
+ }
2416
+ }
2417
+
2418
+ // src/queue/PostgresQueueStorage.ts
2419
+ var POSTGRES_QUEUE_STORAGE = createServiceToken19("jobqueue.storage.postgres");
2116
2420
 
2117
2421
  class PostgresQueueStorage {
2118
2422
  db;
2119
2423
  queueName;
2120
- constructor(db, queueName) {
2424
+ prefixes;
2425
+ prefixValues;
2426
+ tableName;
2427
+ pollingManager = null;
2428
+ constructor(db, queueName, options) {
2121
2429
  this.db = db;
2122
2430
  this.queueName = queueName;
2431
+ this.prefixes = options?.prefixes ?? [];
2432
+ this.prefixValues = options?.prefixValues ?? {};
2433
+ if (this.prefixes.length > 0) {
2434
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
2435
+ this.tableName = `job_queue_${prefixNames}`;
2436
+ } else {
2437
+ this.tableName = "job_queue";
2438
+ }
2439
+ }
2440
+ getPrefixColumnType(type) {
2441
+ return type === "uuid" ? "UUID" : "INTEGER";
2442
+ }
2443
+ buildPrefixColumnsSql() {
2444
+ if (this.prefixes.length === 0)
2445
+ return "";
2446
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
2447
+ `) + `,
2448
+ `;
2449
+ }
2450
+ getPrefixColumnNames() {
2451
+ return this.prefixes.map((p) => p.name);
2452
+ }
2453
+ buildPrefixWhereClause(startParam) {
2454
+ if (this.prefixes.length === 0) {
2455
+ return { conditions: "", params: [] };
2456
+ }
2457
+ const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
2458
+ const params = this.prefixes.map((p) => this.prefixValues[p.name]);
2459
+ return { conditions: " AND " + conditions, params };
2460
+ }
2461
+ getPrefixParamValues() {
2462
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
2123
2463
  }
2124
2464
  async setupDatabase() {
2125
2465
  let sql;
@@ -2130,10 +2470,13 @@ class PostgresQueueStorage {
2130
2470
  if (e.code !== "42710")
2131
2471
  throw e;
2132
2472
  }
2473
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
2474
+ const prefixColumnNames = this.getPrefixColumnNames();
2475
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
2133
2476
  sql = `
2134
- CREATE TABLE IF NOT EXISTS job_queue (
2477
+ CREATE TABLE IF NOT EXISTS ${this.tableName} (
2135
2478
  id SERIAL NOT NULL,
2136
- fingerprint text NOT NULL,
2479
+ ${prefixColumnsSql}fingerprint text NOT NULL,
2137
2480
  queue text NOT NULL,
2138
2481
  job_run_id text NOT NULL,
2139
2482
  status job_status NOT NULL default 'PENDING',
@@ -2150,20 +2493,22 @@ class PostgresQueueStorage {
2150
2493
  error_code text,
2151
2494
  progress real DEFAULT 0,
2152
2495
  progress_message text DEFAULT '',
2153
- progress_details jsonb
2496
+ progress_details jsonb,
2497
+ worker_id text
2154
2498
  )`;
2155
2499
  await this.db.query(sql);
2500
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
2156
2501
  sql = `
2157
- CREATE INDEX IF NOT EXISTS job_fetcher_idx
2158
- ON job_queue (id, status, run_after)`;
2502
+ CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx
2503
+ ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`;
2159
2504
  await this.db.query(sql);
2160
2505
  sql = `
2161
- CREATE INDEX IF NOT EXISTS job_queue_fetcher_idx
2162
- ON job_queue (queue, status, run_after)`;
2506
+ CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx
2507
+ ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`;
2163
2508
  await this.db.query(sql);
2164
2509
  sql = `
2165
- CREATE INDEX IF NOT EXISTS jobs_fingerprint_unique_idx
2166
- ON job_queue (queue, fingerprint, status)`;
2510
+ CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx
2511
+ ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`;
2167
2512
  await this.db.query(sql);
2168
2513
  }
2169
2514
  async add(job) {
@@ -2177,9 +2522,14 @@ class PostgresQueueStorage {
2177
2522
  job.progress_details = null;
2178
2523
  job.created_at = now;
2179
2524
  job.run_after = now;
2525
+ const prefixColumnNames = this.getPrefixColumnNames();
2526
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
2527
+ const prefixParamValues = this.getPrefixParamValues();
2528
+ const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(",") + "," : "";
2529
+ const baseParamStart = prefixColumnNames.length + 1;
2180
2530
  const sql = `
2181
- INSERT INTO job_queue(
2182
- queue,
2531
+ INSERT INTO ${this.tableName}(
2532
+ ${prefixColumnsInsert}queue,
2183
2533
  fingerprint,
2184
2534
  input,
2185
2535
  run_after,
@@ -2192,9 +2542,10 @@ class PostgresQueueStorage {
2192
2542
  progress_details
2193
2543
  )
2194
2544
  VALUES
2195
- ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
2545
+ (${prefixParamPlaceholders}$${baseParamStart},$${baseParamStart + 1},$${baseParamStart + 2},$${baseParamStart + 3},$${baseParamStart + 4},$${baseParamStart + 5},$${baseParamStart + 6},$${baseParamStart + 7},$${baseParamStart + 8},$${baseParamStart + 9},$${baseParamStart + 10})
2196
2546
  RETURNING id`;
2197
2547
  const params = [
2548
+ ...prefixParamValues,
2198
2549
  job.queue,
2199
2550
  job.fingerprint,
2200
2551
  JSON.stringify(job.input),
@@ -2214,68 +2565,75 @@ class PostgresQueueStorage {
2214
2565
  return job.id;
2215
2566
  }
2216
2567
  async get(id) {
2568
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2217
2569
  const result = await this.db.query(`SELECT *
2218
- FROM job_queue
2219
- WHERE id = $1 AND queue = $2
2570
+ FROM ${this.tableName}
2571
+ WHERE id = $1 AND queue = $2${prefixConditions}
2220
2572
  FOR UPDATE SKIP LOCKED
2221
- LIMIT 1`, [id, this.queueName]);
2573
+ LIMIT 1`, [id, this.queueName, ...prefixParams]);
2222
2574
  if (!result || result.rows.length === 0)
2223
2575
  return;
2224
2576
  return result.rows[0];
2225
2577
  }
2226
2578
  async peek(status = "PENDING" /* PENDING */, num = 100) {
2227
2579
  num = Number(num) || 100;
2580
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
2228
2581
  const result = await this.db.query(`
2229
2582
  SELECT *
2230
- FROM job_queue
2583
+ FROM ${this.tableName}
2231
2584
  WHERE queue = $1
2232
- AND status = $2
2585
+ AND status = $2${prefixConditions}
2233
2586
  ORDER BY run_after ASC
2234
2587
  LIMIT $3
2235
- FOR UPDATE SKIP LOCKED`, [this.queueName, status, num]);
2588
+ FOR UPDATE SKIP LOCKED`, [this.queueName, status, num, ...prefixParams]);
2236
2589
  if (!result)
2237
2590
  return [];
2238
2591
  return result.rows;
2239
2592
  }
2240
- async next() {
2593
+ async next(workerId) {
2594
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(5);
2241
2595
  const result = await this.db.query(`
2242
- UPDATE job_queue
2243
- SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC'
2596
+ UPDATE ${this.tableName}
2597
+ SET status = $1, last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = $4
2244
2598
  WHERE id = (
2245
2599
  SELECT id
2246
- FROM job_queue
2600
+ FROM ${this.tableName}
2247
2601
  WHERE queue = $2
2248
- AND status = $3
2602
+ AND status = $3${prefixConditions}
2249
2603
  AND run_after <= NOW() AT TIME ZONE 'UTC'
2250
2604
  ORDER BY run_after ASC
2251
2605
  FOR UPDATE SKIP LOCKED
2252
2606
  LIMIT 1
2253
2607
  )
2254
- RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING */]);
2608
+ RETURNING *`, ["PROCESSING" /* PROCESSING */, this.queueName, "PENDING" /* PENDING */, workerId ?? null, ...prefixParams]);
2255
2609
  return result?.rows?.[0] ?? undefined;
2256
2610
  }
2257
2611
  async size(status = "PENDING" /* PENDING */) {
2612
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2258
2613
  const result = await this.db.query(`
2259
2614
  SELECT COUNT(*) as count
2260
- FROM job_queue
2615
+ FROM ${this.tableName}
2261
2616
  WHERE queue = $1
2262
- AND status = $2`, [this.queueName, status]);
2617
+ AND status = $2${prefixConditions}`, [this.queueName, status, ...prefixParams]);
2263
2618
  if (!result)
2264
2619
  return 0;
2265
2620
  return parseInt(result.rows[0].count, 10);
2266
2621
  }
2267
2622
  async complete(jobDetails) {
2623
+ const prefixParams = this.getPrefixParamValues();
2268
2624
  if (jobDetails.status === "DISABLED" /* DISABLED */) {
2269
- await this.db.query(`UPDATE job_queue
2625
+ const { conditions: prefixConditions } = this.buildPrefixWhereClause(4);
2626
+ await this.db.query(`UPDATE ${this.tableName}
2270
2627
  SET
2271
2628
  status = $1,
2272
2629
  progress = 100,
2273
2630
  progress_message = '',
2274
2631
  progress_details = NULL,
2275
2632
  completed_at = NOW() AT TIME ZONE 'UTC'
2276
- WHERE id = $2 AND queue = $3`, [jobDetails.status, jobDetails.id, this.queueName]);
2633
+ WHERE id = $2 AND queue = $3${prefixConditions}`, [jobDetails.status, jobDetails.id, this.queueName, ...prefixParams]);
2277
2634
  } else if (jobDetails.status === "PENDING" /* PENDING */) {
2278
- await this.db.query(`UPDATE job_queue
2635
+ const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
2636
+ await this.db.query(`UPDATE ${this.tableName}
2279
2637
  SET
2280
2638
  error = $1,
2281
2639
  error_code = $2,
@@ -2286,17 +2644,19 @@ class PostgresQueueStorage {
2286
2644
  progress_details = NULL,
2287
2645
  run_attempts = run_attempts + 1,
2288
2646
  last_ran_at = NOW() AT TIME ZONE 'UTC'
2289
- WHERE id = $5 AND queue = $6`, [
2647
+ WHERE id = $5 AND queue = $6${prefixConditions}`, [
2290
2648
  jobDetails.error,
2291
2649
  jobDetails.error_code,
2292
2650
  jobDetails.status,
2293
2651
  jobDetails.run_after,
2294
2652
  jobDetails.id,
2295
- this.queueName
2653
+ this.queueName,
2654
+ ...prefixParams
2296
2655
  ]);
2297
2656
  } else {
2657
+ const { conditions: prefixConditions } = this.buildPrefixWhereClause(7);
2298
2658
  await this.db.query(`
2299
- UPDATE job_queue
2659
+ UPDATE ${this.tableName}
2300
2660
  SET
2301
2661
  output = $1,
2302
2662
  error = $2,
@@ -2308,86 +2668,231 @@ class PostgresQueueStorage {
2308
2668
  run_attempts = run_attempts + 1,
2309
2669
  completed_at = NOW() AT TIME ZONE 'UTC',
2310
2670
  last_ran_at = NOW() AT TIME ZONE 'UTC'
2311
- WHERE id = $5 AND queue = $6`, [
2671
+ WHERE id = $5 AND queue = $6${prefixConditions}`, [
2312
2672
  jobDetails.output ? JSON.stringify(jobDetails.output) : null,
2313
2673
  jobDetails.error ?? null,
2314
2674
  jobDetails.error_code ?? null,
2315
2675
  jobDetails.status,
2316
2676
  jobDetails.id,
2317
- this.queueName
2677
+ this.queueName,
2678
+ ...prefixParams
2318
2679
  ]);
2319
2680
  }
2320
2681
  }
2321
2682
  async deleteAll() {
2683
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
2322
2684
  await this.db.query(`
2323
- DELETE FROM job_queue
2324
- WHERE queue = $1`, [this.queueName]);
2685
+ DELETE FROM ${this.tableName}
2686
+ WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
2325
2687
  }
2326
2688
  async outputForInput(input) {
2327
2689
  const fingerprint = await makeFingerprint5(input);
2690
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2328
2691
  const result = await this.db.query(`
2329
2692
  SELECT output
2330
- FROM job_queue
2331
- WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'`, [fingerprint, this.queueName]);
2332
- if (!result)
2693
+ FROM ${this.tableName}
2694
+ WHERE fingerprint = $1 AND queue = $2 AND status = 'COMPLETED'${prefixConditions}`, [fingerprint, this.queueName, ...prefixParams]);
2695
+ if (!result || result.rows.length === 0)
2333
2696
  return null;
2334
2697
  return result.rows[0].output;
2335
2698
  }
2336
2699
  async abort(jobId) {
2337
- const result = await this.db.query(`
2338
- UPDATE job_queue
2700
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2701
+ await this.db.query(`
2702
+ UPDATE ${this.tableName}
2339
2703
  SET status = 'ABORTING'
2340
- WHERE id = $1 AND queue = $2`, [jobId, this.queueName]);
2704
+ WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
2341
2705
  }
2342
2706
  async getByRunId(job_run_id) {
2707
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2343
2708
  const result = await this.db.query(`
2344
- SELECT * FROM job_queue WHERE job_run_id = $1 AND queue = $2`, [job_run_id, this.queueName]);
2709
+ SELECT * FROM ${this.tableName} WHERE job_run_id = $1 AND queue = $2${prefixConditions}`, [job_run_id, this.queueName, ...prefixParams]);
2345
2710
  if (!result)
2346
2711
  return [];
2347
2712
  return result.rows;
2348
2713
  }
2349
2714
  async saveProgress(jobId, progress, message, details) {
2715
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(6);
2350
2716
  await this.db.query(`
2351
- UPDATE job_queue
2717
+ UPDATE ${this.tableName}
2352
2718
  SET progress = $1,
2353
2719
  progress_message = $2,
2354
2720
  progress_details = $3
2355
- WHERE id = $4 AND queue = $5`, [progress, message, details ? JSON.stringify(details) : null, jobId, this.queueName]);
2356
- }
2357
- async delete(jobId) {
2358
- await this.db.query("DELETE FROM job_queue WHERE id = $1 AND queue = $2", [
2721
+ WHERE id = $4 AND queue = $5${prefixConditions}`, [
2722
+ progress,
2723
+ message,
2724
+ details ? JSON.stringify(details) : null,
2359
2725
  jobId,
2360
- this.queueName
2726
+ this.queueName,
2727
+ ...prefixParams
2361
2728
  ]);
2362
2729
  }
2730
+ async delete(jobId) {
2731
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
2732
+ await this.db.query(`DELETE FROM ${this.tableName} WHERE id = $1 AND queue = $2${prefixConditions}`, [jobId, this.queueName, ...prefixParams]);
2733
+ }
2363
2734
  async deleteJobsByStatusAndAge(status, olderThanMs) {
2364
2735
  const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
2365
- await this.db.query(`DELETE FROM job_queue
2736
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(4);
2737
+ await this.db.query(`DELETE FROM ${this.tableName}
2366
2738
  WHERE queue = $1
2367
2739
  AND status = $2
2368
2740
  AND completed_at IS NOT NULL
2369
- AND completed_at <= $3`, [this.queueName, status, cutoffDate]);
2741
+ AND completed_at <= $3${prefixConditions}`, [this.queueName, status, cutoffDate, ...prefixParams]);
2742
+ }
2743
+ async getAllJobs() {
2744
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
2745
+ const result = await this.db.query(`SELECT * FROM ${this.tableName} WHERE queue = $1${prefixConditions}`, [this.queueName, ...prefixParams]);
2746
+ if (!result)
2747
+ return [];
2748
+ return result.rows;
2749
+ }
2750
+ async getAllJobsWithFilter(prefixFilter) {
2751
+ const filterEntries = Object.entries(prefixFilter);
2752
+ let query = `SELECT * FROM ${this.tableName} WHERE queue = $1`;
2753
+ const params = [this.queueName];
2754
+ filterEntries.forEach(([key, value], index) => {
2755
+ query += ` AND ${key} = $${index + 2}`;
2756
+ params.push(value);
2757
+ });
2758
+ const result = await this.db.query(query, params);
2759
+ if (!result)
2760
+ return [];
2761
+ return result.rows;
2762
+ }
2763
+ isCustomPrefixFilter(prefixFilter) {
2764
+ if (prefixFilter === undefined) {
2765
+ return false;
2766
+ }
2767
+ if (Object.keys(prefixFilter).length === 0) {
2768
+ return true;
2769
+ }
2770
+ const instanceKeys = Object.keys(this.prefixValues);
2771
+ const filterKeys = Object.keys(prefixFilter);
2772
+ if (instanceKeys.length !== filterKeys.length) {
2773
+ return true;
2774
+ }
2775
+ for (const key of instanceKeys) {
2776
+ if (this.prefixValues[key] !== prefixFilter[key]) {
2777
+ return true;
2778
+ }
2779
+ }
2780
+ return false;
2781
+ }
2782
+ getPollingManager() {
2783
+ if (!this.pollingManager) {
2784
+ this.pollingManager = new PollingSubscriptionManager(async () => {
2785
+ const jobs = await this.getAllJobs();
2786
+ return new Map(jobs.map((j) => [j.id, j]));
2787
+ }, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
2788
+ insert: (item) => ({ type: "INSERT", new: item }),
2789
+ update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
2790
+ delete: (item) => ({ type: "DELETE", old: item })
2791
+ });
2792
+ }
2793
+ return this.pollingManager;
2794
+ }
2795
+ subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
2796
+ let lastKnownJobs = new Map;
2797
+ let cancelled = false;
2798
+ const poll = async () => {
2799
+ if (cancelled)
2800
+ return;
2801
+ try {
2802
+ const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
2803
+ if (cancelled)
2804
+ return;
2805
+ const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
2806
+ for (const [id, job] of currentMap) {
2807
+ const old = lastKnownJobs.get(id);
2808
+ if (!old) {
2809
+ callback({ type: "INSERT", new: job });
2810
+ } else if (JSON.stringify(old) !== JSON.stringify(job)) {
2811
+ callback({ type: "UPDATE", old, new: job });
2812
+ }
2813
+ }
2814
+ for (const [id, job] of lastKnownJobs) {
2815
+ if (!currentMap.has(id)) {
2816
+ callback({ type: "DELETE", old: job });
2817
+ }
2818
+ }
2819
+ lastKnownJobs = currentMap;
2820
+ } catch {}
2821
+ };
2822
+ const intervalId = setInterval(poll, intervalMs);
2823
+ poll();
2824
+ return () => {
2825
+ cancelled = true;
2826
+ clearInterval(intervalId);
2827
+ };
2828
+ }
2829
+ subscribeToChanges(callback, options) {
2830
+ const intervalMs = options?.pollingIntervalMs ?? 1000;
2831
+ if (this.isCustomPrefixFilter(options?.prefixFilter)) {
2832
+ return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
2833
+ }
2834
+ const manager = this.getPollingManager();
2835
+ return manager.subscribe(callback, { intervalMs });
2370
2836
  }
2371
2837
  }
2372
2838
  // src/queue/SqliteQueueStorage.ts
2373
- import { createServiceToken as createServiceToken18, makeFingerprint as makeFingerprint6, sleep as sleep3, uuid4 as uuid43 } from "@workglow/util";
2374
- var SQLITE_QUEUE_STORAGE = createServiceToken18("jobqueue.storage.sqlite");
2839
+ import { createServiceToken as createServiceToken20, makeFingerprint as makeFingerprint6, sleep as sleep4, uuid4 as uuid43 } from "@workglow/util";
2840
+ var SQLITE_QUEUE_STORAGE = createServiceToken20("jobqueue.storage.sqlite");
2375
2841
 
2376
2842
  class SqliteQueueStorage {
2377
2843
  db;
2378
2844
  queueName;
2379
2845
  options;
2846
+ prefixes;
2847
+ prefixValues;
2848
+ tableName;
2849
+ pollingManager = null;
2380
2850
  constructor(db, queueName, options) {
2381
2851
  this.db = db;
2382
2852
  this.queueName = queueName;
2383
2853
  this.options = options;
2854
+ this.prefixes = options?.prefixes ?? [];
2855
+ this.prefixValues = options?.prefixValues ?? {};
2856
+ if (this.prefixes.length > 0) {
2857
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
2858
+ this.tableName = `job_queue_${prefixNames}`;
2859
+ } else {
2860
+ this.tableName = "job_queue";
2861
+ }
2862
+ }
2863
+ getPrefixColumnType(type) {
2864
+ return type === "uuid" ? "TEXT" : "INTEGER";
2865
+ }
2866
+ buildPrefixColumnsSql() {
2867
+ if (this.prefixes.length === 0)
2868
+ return "";
2869
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
2870
+ `) + `,
2871
+ `;
2872
+ }
2873
+ getPrefixColumnNames() {
2874
+ return this.prefixes.map((p) => p.name);
2875
+ }
2876
+ buildPrefixWhereClause() {
2877
+ if (this.prefixes.length === 0) {
2878
+ return "";
2879
+ }
2880
+ const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
2881
+ return " AND " + conditions;
2882
+ }
2883
+ getPrefixParamValues() {
2884
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
2384
2885
  }
2385
2886
  async setupDatabase() {
2386
- await sleep3(0);
2887
+ await sleep4(0);
2888
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
2889
+ const prefixColumnNames = this.getPrefixColumnNames();
2890
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
2891
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
2387
2892
  this.db.exec(`
2388
- CREATE TABLE IF NOT EXISTS job_queue (
2893
+ CREATE TABLE IF NOT EXISTS ${this.tableName} (
2389
2894
  id INTEGER PRIMARY KEY,
2390
- fingerprint text NOT NULL,
2895
+ ${prefixColumnsSql}fingerprint text NOT NULL,
2391
2896
  queue text NOT NULL,
2392
2897
  job_run_id text NOT NULL,
2393
2898
  status TEXT NOT NULL default 'PENDING',
@@ -2404,12 +2909,13 @@ class SqliteQueueStorage {
2404
2909
  error_code TEXT,
2405
2910
  progress REAL DEFAULT 0,
2406
2911
  progress_message TEXT DEFAULT '',
2407
- progress_details TEXT NULL
2912
+ progress_details TEXT NULL,
2913
+ worker_id TEXT
2408
2914
  );
2409
2915
 
2410
- CREATE INDEX IF NOT EXISTS job_queue_fetcher_idx ON job_queue (queue, status, run_after);
2411
- CREATE INDEX IF NOT EXISTS job_queue_fingerprint_idx ON job_queue (queue, fingerprint, status);
2412
- CREATE INDEX IF NOT EXISTS job_queue_job_run_id_idx ON job_queue (queue, job_run_id);
2916
+ CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after);
2917
+ CREATE INDEX IF NOT EXISTS job_queue_fingerprint${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status);
2918
+ CREATE INDEX IF NOT EXISTS job_queue_job_run_id${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, job_run_id);
2413
2919
  `);
2414
2920
  }
2415
2921
  async add(job) {
@@ -2423,9 +2929,13 @@ class SqliteQueueStorage {
2423
2929
  job.progress_details = null;
2424
2930
  job.created_at = now;
2425
2931
  job.run_after = now;
2932
+ const prefixColumnNames = this.getPrefixColumnNames();
2933
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
2934
+ const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
2935
+ const prefixParamValues = this.getPrefixParamValues();
2426
2936
  const AddQuery = `
2427
- INSERT INTO job_queue(
2428
- queue,
2937
+ INSERT INTO ${this.tableName}(
2938
+ ${prefixColumnsInsert}queue,
2429
2939
  fingerprint,
2430
2940
  input,
2431
2941
  run_after,
@@ -2437,21 +2947,23 @@ class SqliteQueueStorage {
2437
2947
  progress_details,
2438
2948
  created_at
2439
2949
  )
2440
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
2950
+ VALUES (${prefixPlaceholders}?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
2441
2951
  RETURNING id`;
2442
2952
  const stmt = this.db.prepare(AddQuery);
2443
- const result = stmt.get(job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
2953
+ const result = stmt.get(...prefixParamValues, job.queue, job.fingerprint, JSON.stringify(job.input), job.run_after, job.deadline_at ?? null, job.max_retries, job.job_run_id, job.progress, job.progress_message, job.progress_details ? JSON.stringify(job.progress_details) : null, job.created_at);
2444
2954
  job.id = result?.id;
2445
2955
  return result?.id;
2446
2956
  }
2447
2957
  async get(id) {
2958
+ const prefixConditions = this.buildPrefixWhereClause();
2959
+ const prefixParams = this.getPrefixParamValues();
2448
2960
  const JobQuery = `
2449
2961
  SELECT *
2450
- FROM job_queue
2451
- WHERE id = ? AND queue = ?
2962
+ FROM ${this.tableName}
2963
+ WHERE id = ? AND queue = ?${prefixConditions}
2452
2964
  LIMIT 1`;
2453
2965
  const stmt = this.db.prepare(JobQuery);
2454
- const result = stmt.get(id, this.queueName);
2966
+ const result = stmt.get(String(id), this.queueName, ...prefixParams);
2455
2967
  if (!result)
2456
2968
  return;
2457
2969
  if (result.input)
@@ -2464,15 +2976,17 @@ class SqliteQueueStorage {
2464
2976
  }
2465
2977
  async peek(status = "PENDING" /* PENDING */, num = 100) {
2466
2978
  num = Number(num) || 100;
2979
+ const prefixConditions = this.buildPrefixWhereClause();
2980
+ const prefixParams = this.getPrefixParamValues();
2467
2981
  const FutureJobQuery = `
2468
2982
  SELECT *
2469
- FROM job_queue
2983
+ FROM ${this.tableName}
2470
2984
  WHERE queue = ?
2471
- AND status = ?
2985
+ AND status = ?${prefixConditions}
2472
2986
  ORDER BY run_after ASC
2473
2987
  LIMIT ${num}`;
2474
2988
  const stmt = this.db.prepare(FutureJobQuery);
2475
- const result = stmt.all(this.queueName, status);
2989
+ const result = stmt.all(this.queueName, status, ...prefixParams);
2476
2990
  return (result || []).map((details) => {
2477
2991
  if (details.input)
2478
2992
  details.input = JSON.parse(details.input);
@@ -2484,20 +2998,24 @@ class SqliteQueueStorage {
2484
2998
  });
2485
2999
  }
2486
3000
  async abort(jobId) {
3001
+ const prefixConditions = this.buildPrefixWhereClause();
3002
+ const prefixParams = this.getPrefixParamValues();
2487
3003
  const AbortQuery = `
2488
- UPDATE job_queue
3004
+ UPDATE ${this.tableName}
2489
3005
  SET status = ?
2490
- WHERE id = ? AND queue = ?`;
3006
+ WHERE id = ? AND queue = ?${prefixConditions}`;
2491
3007
  const stmt = this.db.prepare(AbortQuery);
2492
- stmt.run("ABORTING" /* ABORTING */, jobId, this.queueName);
3008
+ stmt.run("ABORTING" /* ABORTING */, String(jobId), this.queueName, ...prefixParams);
2493
3009
  }
2494
3010
  async getByRunId(job_run_id) {
3011
+ const prefixConditions = this.buildPrefixWhereClause();
3012
+ const prefixParams = this.getPrefixParamValues();
2495
3013
  const JobsByRunIdQuery = `
2496
3014
  SELECT *
2497
- FROM job_queue
2498
- WHERE job_run_id = ? AND queue = ?`;
3015
+ FROM ${this.tableName}
3016
+ WHERE job_run_id = ? AND queue = ?${prefixConditions}`;
2499
3017
  const stmt = this.db.prepare(JobsByRunIdQuery);
2500
- const result = stmt.all(job_run_id, this.queueName);
3018
+ const result = stmt.all(job_run_id, this.queueName, ...prefixParams);
2501
3019
  return (result || []).map((details) => {
2502
3020
  if (details.input)
2503
3021
  details.input = JSON.parse(details.input);
@@ -2508,22 +3026,24 @@ class SqliteQueueStorage {
2508
3026
  return details;
2509
3027
  });
2510
3028
  }
2511
- async next() {
3029
+ async next(workerId) {
2512
3030
  const now = new Date().toISOString();
3031
+ const prefixConditions = this.buildPrefixWhereClause();
3032
+ const prefixParams = this.getPrefixParamValues();
2513
3033
  const stmt = this.db.prepare(`
2514
- UPDATE job_queue
2515
- SET status = ?, last_ran_at = ?
3034
+ UPDATE ${this.tableName}
3035
+ SET status = ?, last_ran_at = ?, worker_id = ?
2516
3036
  WHERE id = (
2517
3037
  SELECT id
2518
- FROM job_queue
3038
+ FROM ${this.tableName}
2519
3039
  WHERE queue = ?
2520
- AND status = ?
3040
+ AND status = ?${prefixConditions}
2521
3041
  AND run_after <= ?
2522
3042
  ORDER BY run_after ASC
2523
3043
  LIMIT 1
2524
3044
  )
2525
3045
  RETURNING *`);
2526
- const result = stmt.get("PROCESSING" /* PROCESSING */, now, this.queueName, "PENDING" /* PENDING */, now);
3046
+ const result = stmt.get("PROCESSING" /* PROCESSING */, now, workerId ?? null, this.queueName, "PENDING" /* PENDING */, ...prefixParams, now);
2527
3047
  if (!result)
2528
3048
  return;
2529
3049
  if (result.input)
@@ -2535,33 +3055,37 @@ class SqliteQueueStorage {
2535
3055
  return result;
2536
3056
  }
2537
3057
  async size(status = "PENDING" /* PENDING */) {
3058
+ const prefixConditions = this.buildPrefixWhereClause();
3059
+ const prefixParams = this.getPrefixParamValues();
2538
3060
  const sizeQuery = `
2539
3061
  SELECT COUNT(*) as count
2540
- FROM job_queue
3062
+ FROM ${this.tableName}
2541
3063
  WHERE queue = ?
2542
- AND status = ?`;
3064
+ AND status = ?${prefixConditions}`;
2543
3065
  const stmt = this.db.prepare(sizeQuery);
2544
- const result = stmt.get(this.queueName, status);
3066
+ const result = stmt.get(this.queueName, status, ...prefixParams);
2545
3067
  return result.count;
2546
3068
  }
2547
3069
  async complete(job) {
2548
3070
  const now = new Date().toISOString();
3071
+ const prefixConditions = this.buildPrefixWhereClause();
3072
+ const prefixParams = this.getPrefixParamValues();
2549
3073
  let updateQuery;
2550
3074
  let params;
2551
3075
  if (job.status === "DISABLED" /* DISABLED */) {
2552
3076
  updateQuery = `
2553
- UPDATE job_queue
3077
+ UPDATE ${this.tableName}
2554
3078
  SET
2555
3079
  status = ?,
2556
3080
  progress = 100,
2557
3081
  progress_message = '',
2558
3082
  progress_details = NULL,
2559
3083
  completed_at = ?
2560
- WHERE id = ? AND queue = ?`;
2561
- params = [job.status, now, job.id, this.queueName];
3084
+ WHERE id = ? AND queue = ?${prefixConditions}`;
3085
+ params = [job.status, now, job.id, this.queueName, ...prefixParams];
2562
3086
  } else {
2563
3087
  updateQuery = `
2564
- UPDATE job_queue
3088
+ UPDATE ${this.tableName}
2565
3089
  SET
2566
3090
  output = ?,
2567
3091
  error = ?,
@@ -2573,7 +3097,7 @@ class SqliteQueueStorage {
2573
3097
  last_ran_at = ?,
2574
3098
  completed_at = ?,
2575
3099
  run_attempts = run_attempts + 1
2576
- WHERE id = ? AND queue = ?`;
3100
+ WHERE id = ? AND queue = ?${prefixConditions}`;
2577
3101
  params = [
2578
3102
  job.output ? JSON.stringify(job.output) : null,
2579
3103
  job.error ?? null,
@@ -2582,68 +3106,227 @@ class SqliteQueueStorage {
2582
3106
  now,
2583
3107
  now,
2584
3108
  job.id,
2585
- this.queueName
3109
+ this.queueName,
3110
+ ...prefixParams
2586
3111
  ];
2587
3112
  }
2588
3113
  const stmt = this.db.prepare(updateQuery);
2589
3114
  stmt.run(...params);
2590
3115
  }
2591
3116
  async deleteAll() {
3117
+ const prefixConditions = this.buildPrefixWhereClause();
3118
+ const prefixParams = this.getPrefixParamValues();
2592
3119
  const ClearQuery = `
2593
- DELETE FROM job_queue
2594
- WHERE queue = ?`;
3120
+ DELETE FROM ${this.tableName}
3121
+ WHERE queue = ?${prefixConditions}`;
2595
3122
  const stmt = this.db.prepare(ClearQuery);
2596
- stmt.run(this.queueName);
3123
+ stmt.run(this.queueName, ...prefixParams);
2597
3124
  }
2598
3125
  async outputForInput(input) {
2599
3126
  const fingerprint = await makeFingerprint6(input);
3127
+ const prefixConditions = this.buildPrefixWhereClause();
3128
+ const prefixParams = this.getPrefixParamValues();
2600
3129
  const OutputQuery = `
2601
3130
  SELECT output
2602
- FROM job_queue
2603
- WHERE queue = ? AND fingerprint = ? AND status = ?`;
3131
+ FROM ${this.tableName}
3132
+ WHERE queue = ? AND fingerprint = ? AND status = ?${prefixConditions}`;
2604
3133
  const stmt = this.db.prepare(OutputQuery);
2605
- const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED */);
3134
+ const result = stmt.get(this.queueName, fingerprint, "COMPLETED" /* COMPLETED */, ...prefixParams);
2606
3135
  return result?.output ? JSON.parse(result.output) : null;
2607
3136
  }
2608
3137
  async saveProgress(jobId, progress, message, details) {
3138
+ const prefixConditions = this.buildPrefixWhereClause();
3139
+ const prefixParams = this.getPrefixParamValues();
2609
3140
  const UpdateProgressQuery = `
2610
- UPDATE job_queue
3141
+ UPDATE ${this.tableName}
2611
3142
  SET progress = ?,
2612
3143
  progress_message = ?,
2613
3144
  progress_details = ?
2614
- WHERE id = ? AND queue = ?`;
3145
+ WHERE id = ? AND queue = ?${prefixConditions}`;
2615
3146
  const stmt = this.db.prepare(UpdateProgressQuery);
2616
- stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName);
3147
+ stmt.run(progress, message, JSON.stringify(details), String(jobId), this.queueName, ...prefixParams);
2617
3148
  }
2618
3149
  async delete(jobId) {
3150
+ const prefixConditions = this.buildPrefixWhereClause();
3151
+ const prefixParams = this.getPrefixParamValues();
2619
3152
  const DeleteQuery = `
2620
- DELETE FROM job_queue
2621
- WHERE id = ? AND queue = ?`;
3153
+ DELETE FROM ${this.tableName}
3154
+ WHERE id = ? AND queue = ?${prefixConditions}`;
2622
3155
  const stmt = this.db.prepare(DeleteQuery);
2623
- stmt.run(String(jobId), this.queueName);
3156
+ stmt.run(String(jobId), this.queueName, ...prefixParams);
2624
3157
  }
2625
3158
  async deleteJobsByStatusAndAge(status, olderThanMs) {
2626
3159
  const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
3160
+ const prefixConditions = this.buildPrefixWhereClause();
3161
+ const prefixParams = this.getPrefixParamValues();
2627
3162
  const DeleteQuery = `
2628
- DELETE FROM job_queue
3163
+ DELETE FROM ${this.tableName}
2629
3164
  WHERE queue = ?
2630
3165
  AND status = ?
2631
3166
  AND completed_at IS NOT NULL
2632
- AND completed_at <= ?`;
3167
+ AND completed_at <= ?${prefixConditions}`;
2633
3168
  const stmt = this.db.prepare(DeleteQuery);
2634
- stmt.run(this.queueName, status, cutoffDate);
3169
+ stmt.run(this.queueName, status, cutoffDate, ...prefixParams);
3170
+ }
3171
+ getAllJobs() {
3172
+ const prefixConditions = this.buildPrefixWhereClause();
3173
+ const prefixParams = this.getPrefixParamValues();
3174
+ const AllJobsQuery = `
3175
+ SELECT *
3176
+ FROM ${this.tableName}
3177
+ WHERE queue = ?${prefixConditions}`;
3178
+ const stmt = this.db.prepare(AllJobsQuery);
3179
+ const result = stmt.all(this.queueName, ...prefixParams);
3180
+ return (result || []).map((details) => {
3181
+ if (details.input)
3182
+ details.input = JSON.parse(details.input);
3183
+ if (details.output)
3184
+ details.output = JSON.parse(details.output);
3185
+ if (details.progress_details)
3186
+ details.progress_details = JSON.parse(details.progress_details);
3187
+ return details;
3188
+ });
3189
+ }
3190
+ getAllJobsWithFilter(prefixFilter) {
3191
+ const filterEntries = Object.entries(prefixFilter);
3192
+ let whereClause = "WHERE queue = ?";
3193
+ const params = [this.queueName];
3194
+ for (const [key, value] of filterEntries) {
3195
+ whereClause += ` AND ${key} = ?`;
3196
+ params.push(value);
3197
+ }
3198
+ const AllJobsQuery = `SELECT * FROM ${this.tableName} ${whereClause}`;
3199
+ const stmt = this.db.prepare(AllJobsQuery);
3200
+ const result = stmt.all(...params);
3201
+ return (result || []).map((details) => {
3202
+ if (details.input)
3203
+ details.input = JSON.parse(details.input);
3204
+ if (details.output)
3205
+ details.output = JSON.parse(details.output);
3206
+ if (details.progress_details)
3207
+ details.progress_details = JSON.parse(details.progress_details);
3208
+ return details;
3209
+ });
3210
+ }
3211
+ isCustomPrefixFilter(prefixFilter) {
3212
+ if (prefixFilter === undefined) {
3213
+ return false;
3214
+ }
3215
+ if (Object.keys(prefixFilter).length === 0) {
3216
+ return true;
3217
+ }
3218
+ const instanceKeys = Object.keys(this.prefixValues);
3219
+ const filterKeys = Object.keys(prefixFilter);
3220
+ if (instanceKeys.length !== filterKeys.length) {
3221
+ return true;
3222
+ }
3223
+ for (const key of instanceKeys) {
3224
+ if (this.prefixValues[key] !== prefixFilter[key]) {
3225
+ return true;
3226
+ }
3227
+ }
3228
+ return false;
3229
+ }
3230
+ getPollingManager() {
3231
+ if (!this.pollingManager) {
3232
+ this.pollingManager = new PollingSubscriptionManager(async () => {
3233
+ const jobs = this.getAllJobs();
3234
+ return new Map(jobs.map((j) => [j.id, j]));
3235
+ }, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
3236
+ insert: (item) => ({ type: "INSERT", new: item }),
3237
+ update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
3238
+ delete: (item) => ({ type: "DELETE", old: item })
3239
+ });
3240
+ }
3241
+ return this.pollingManager;
3242
+ }
3243
+ subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
3244
+ let lastKnownJobs = new Map;
3245
+ const poll = () => {
3246
+ try {
3247
+ const currentJobs = this.getAllJobsWithFilter(prefixFilter);
3248
+ const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
3249
+ for (const [id, job] of currentMap) {
3250
+ const old = lastKnownJobs.get(id);
3251
+ if (!old) {
3252
+ callback({ type: "INSERT", new: job });
3253
+ } else if (JSON.stringify(old) !== JSON.stringify(job)) {
3254
+ callback({ type: "UPDATE", old, new: job });
3255
+ }
3256
+ }
3257
+ for (const [id, job] of lastKnownJobs) {
3258
+ if (!currentMap.has(id)) {
3259
+ callback({ type: "DELETE", old: job });
3260
+ }
3261
+ }
3262
+ lastKnownJobs = currentMap;
3263
+ } catch {}
3264
+ };
3265
+ const intervalId = setInterval(poll, intervalMs);
3266
+ poll();
3267
+ return () => {
3268
+ clearInterval(intervalId);
3269
+ };
3270
+ }
3271
+ subscribeToChanges(callback, options) {
3272
+ const intervalMs = options?.pollingIntervalMs ?? 1000;
3273
+ if (this.isCustomPrefixFilter(options?.prefixFilter)) {
3274
+ return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
3275
+ }
3276
+ const manager = this.getPollingManager();
3277
+ return manager.subscribe(callback, { intervalMs });
2635
3278
  }
2636
3279
  }
2637
3280
  // src/queue/SupabaseQueueStorage.ts
2638
- import { createServiceToken as createServiceToken19, makeFingerprint as makeFingerprint7, uuid4 as uuid44 } from "@workglow/util";
2639
- var SUPABASE_QUEUE_STORAGE = createServiceToken19("jobqueue.storage.supabase");
3281
+ import { createServiceToken as createServiceToken21, makeFingerprint as makeFingerprint7, uuid4 as uuid44 } from "@workglow/util";
3282
+ var SUPABASE_QUEUE_STORAGE = createServiceToken21("jobqueue.storage.supabase");
2640
3283
 
2641
3284
  class SupabaseQueueStorage {
2642
3285
  client;
2643
3286
  queueName;
2644
- constructor(client, queueName) {
3287
+ prefixes;
3288
+ prefixValues;
3289
+ tableName;
3290
+ realtimeChannel = null;
3291
+ pollingManager = null;
3292
+ constructor(client, queueName, options) {
2645
3293
  this.client = client;
2646
3294
  this.queueName = queueName;
3295
+ this.prefixes = options?.prefixes ?? [];
3296
+ this.prefixValues = options?.prefixValues ?? {};
3297
+ if (this.prefixes.length > 0) {
3298
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
3299
+ this.tableName = `job_queue_${prefixNames}`;
3300
+ } else {
3301
+ this.tableName = "job_queue";
3302
+ }
3303
+ }
3304
+ getPrefixColumnType(type) {
3305
+ return type === "uuid" ? "UUID" : "INTEGER";
3306
+ }
3307
+ buildPrefixColumnsSql() {
3308
+ if (this.prefixes.length === 0)
3309
+ return "";
3310
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
3311
+ `) + `,
3312
+ `;
3313
+ }
3314
+ getPrefixColumnNames() {
3315
+ return this.prefixes.map((p) => p.name);
3316
+ }
3317
+ applyPrefixFilters(query) {
3318
+ let result = query;
3319
+ for (const prefix of this.prefixes) {
3320
+ result = result.eq(prefix.name, this.prefixValues[prefix.name]);
3321
+ }
3322
+ return result;
3323
+ }
3324
+ getPrefixInsertValues() {
3325
+ const values = {};
3326
+ for (const prefix of this.prefixes) {
3327
+ values[prefix.name] = this.prefixValues[prefix.name];
3328
+ }
3329
+ return values;
2647
3330
  }
2648
3331
  async setupDatabase() {
2649
3332
  const createTypeSql = `CREATE TYPE job_status AS ENUM (${Object.values(JobStatus).map((v) => `'${v}'`).join(",")})`;
@@ -2651,10 +3334,14 @@ class SupabaseQueueStorage {
2651
3334
  if (typeError && typeError.code !== "42710") {
2652
3335
  throw typeError;
2653
3336
  }
3337
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
3338
+ const prefixColumnNames = this.getPrefixColumnNames();
3339
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3340
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
2654
3341
  const createTableSql = `
2655
- CREATE TABLE IF NOT EXISTS job_queue (
3342
+ CREATE TABLE IF NOT EXISTS ${this.tableName} (
2656
3343
  id SERIAL NOT NULL,
2657
- fingerprint text NOT NULL,
3344
+ ${prefixColumnsSql}fingerprint text NOT NULL,
2658
3345
  queue text NOT NULL,
2659
3346
  job_run_id text NOT NULL,
2660
3347
  status job_status NOT NULL default 'PENDING',
@@ -2671,7 +3358,8 @@ class SupabaseQueueStorage {
2671
3358
  error_code text,
2672
3359
  progress real DEFAULT 0,
2673
3360
  progress_message text DEFAULT '',
2674
- progress_details jsonb
3361
+ progress_details jsonb,
3362
+ worker_id text
2675
3363
  )`;
2676
3364
  const { error: tableError } = await this.client.rpc("exec_sql", { query: createTableSql });
2677
3365
  if (tableError) {
@@ -2680,12 +3368,12 @@ class SupabaseQueueStorage {
2680
3368
  }
2681
3369
  }
2682
3370
  const indexes = [
2683
- `CREATE INDEX IF NOT EXISTS job_fetcher_idx ON job_queue (id, status, run_after)`,
2684
- `CREATE INDEX IF NOT EXISTS job_queue_fetcher_idx ON job_queue (queue, status, run_after)`,
2685
- `CREATE INDEX IF NOT EXISTS jobs_fingerprint_unique_idx ON job_queue (queue, fingerprint, status)`
3371
+ `CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`,
3372
+ `CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`,
3373
+ `CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`
2686
3374
  ];
2687
3375
  for (const indexSql of indexes) {
2688
- const { error: indexError } = await this.client.rpc("exec_sql", { query: indexSql });
3376
+ await this.client.rpc("exec_sql", { query: indexSql });
2689
3377
  }
2690
3378
  }
2691
3379
  async add(job) {
@@ -2699,7 +3387,9 @@ class SupabaseQueueStorage {
2699
3387
  job.progress_details = null;
2700
3388
  job.created_at = now;
2701
3389
  job.run_after = now;
2702
- const { data, error } = await this.client.from("job_queue").insert({
3390
+ const prefixInsertValues = this.getPrefixInsertValues();
3391
+ const { data, error } = await this.client.from(this.tableName).insert({
3392
+ ...prefixInsertValues,
2703
3393
  queue: job.queue,
2704
3394
  fingerprint: job.fingerprint,
2705
3395
  input: job.input,
@@ -2720,7 +3410,9 @@ class SupabaseQueueStorage {
2720
3410
  return job.id;
2721
3411
  }
2722
3412
  async get(id) {
2723
- const { data, error } = await this.client.from("job_queue").select("*").eq("id", id).eq("queue", this.queueName).single();
3413
+ let query = this.client.from(this.tableName).select("*").eq("id", id).eq("queue", this.queueName);
3414
+ query = this.applyPrefixFilters(query);
3415
+ const { data, error } = await query.single();
2724
3416
  if (error) {
2725
3417
  if (error.code === "PGRST116")
2726
3418
  return;
@@ -2730,36 +3422,53 @@ class SupabaseQueueStorage {
2730
3422
  }
2731
3423
  async peek(status = "PENDING" /* PENDING */, num = 100) {
2732
3424
  num = Number(num) || 100;
2733
- const { data, error } = await this.client.from("job_queue").select("*").eq("queue", this.queueName).eq("status", status).order("run_after", { ascending: true }).limit(num);
3425
+ let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", status);
3426
+ query = this.applyPrefixFilters(query);
3427
+ const { data, error } = await query.order("run_after", { ascending: true }).limit(num);
2734
3428
  if (error)
2735
3429
  throw error;
2736
3430
  return data ?? [];
2737
3431
  }
2738
- async next() {
2739
- const { data: jobs, error: selectError } = await this.client.from("job_queue").select("*").eq("queue", this.queueName).eq("status", "PENDING" /* PENDING */).lte("run_after", new Date().toISOString()).order("run_after", { ascending: true }).limit(1);
3432
+ async next(workerId) {
3433
+ let selectQuery = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", "PENDING" /* PENDING */).lte("run_after", new Date().toISOString());
3434
+ selectQuery = this.applyPrefixFilters(selectQuery);
3435
+ const { data: jobs, error: selectError } = await selectQuery.order("run_after", { ascending: true }).limit(1);
2740
3436
  if (selectError)
2741
3437
  throw selectError;
2742
3438
  if (!jobs || jobs.length === 0)
2743
3439
  return;
2744
3440
  const job = jobs[0];
2745
- const { data: updatedJob, error: updateError } = await this.client.from("job_queue").update({
3441
+ let updateQuery = this.client.from(this.tableName).update({
2746
3442
  status: "PROCESSING" /* PROCESSING */,
2747
- last_ran_at: new Date().toISOString()
2748
- }).eq("id", job.id).eq("queue", this.queueName).select().single();
3443
+ last_ran_at: new Date().toISOString(),
3444
+ worker_id: workerId ?? null
3445
+ }).eq("id", job.id).eq("queue", this.queueName);
3446
+ updateQuery = this.applyPrefixFilters(updateQuery);
3447
+ const { data: updatedJob, error: updateError } = await updateQuery.select().single();
2749
3448
  if (updateError)
2750
3449
  throw updateError;
2751
3450
  return updatedJob;
2752
3451
  }
2753
3452
  async size(status = "PENDING" /* PENDING */) {
2754
- const { count, error } = await this.client.from("job_queue").select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
3453
+ let query = this.client.from(this.tableName).select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
3454
+ query = this.applyPrefixFilters(query);
3455
+ const { count, error } = await query;
2755
3456
  if (error)
2756
3457
  throw error;
2757
3458
  return count ?? 0;
2758
3459
  }
3460
+ async getAllJobs() {
3461
+ let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
3462
+ query = this.applyPrefixFilters(query);
3463
+ const { data, error } = await query;
3464
+ if (error)
3465
+ throw error;
3466
+ return data ?? [];
3467
+ }
2759
3468
  async complete(jobDetails) {
2760
3469
  const now = new Date().toISOString();
2761
3470
  if (jobDetails.status === "DISABLED" /* DISABLED */) {
2762
- const { error: error2 } = await this.client.from("job_queue").update({
3471
+ let query2 = this.client.from(this.tableName).update({
2763
3472
  status: jobDetails.status,
2764
3473
  progress: 100,
2765
3474
  progress_message: "",
@@ -2767,16 +3476,20 @@ class SupabaseQueueStorage {
2767
3476
  completed_at: now,
2768
3477
  last_ran_at: now
2769
3478
  }).eq("id", jobDetails.id).eq("queue", this.queueName);
3479
+ query2 = this.applyPrefixFilters(query2);
3480
+ const { error: error2 } = await query2;
2770
3481
  if (error2)
2771
3482
  throw error2;
2772
3483
  return;
2773
3484
  }
2774
- const { data: current, error: getError } = await this.client.from("job_queue").select("run_attempts").eq("id", jobDetails.id).eq("queue", this.queueName).single();
3485
+ let getQuery = this.client.from(this.tableName).select("run_attempts").eq("id", jobDetails.id).eq("queue", this.queueName);
3486
+ getQuery = this.applyPrefixFilters(getQuery);
3487
+ const { data: current, error: getError } = await getQuery.single();
2775
3488
  if (getError)
2776
3489
  throw getError;
2777
3490
  const nextAttempts = (current?.run_attempts ?? 0) + 1;
2778
3491
  if (jobDetails.status === "PENDING" /* PENDING */) {
2779
- const { error: error2 } = await this.client.from("job_queue").update({
3492
+ let query2 = this.client.from(this.tableName).update({
2780
3493
  error: jobDetails.error ?? null,
2781
3494
  error_code: jobDetails.error_code ?? null,
2782
3495
  status: jobDetails.status,
@@ -2787,12 +3500,14 @@ class SupabaseQueueStorage {
2787
3500
  run_attempts: nextAttempts,
2788
3501
  last_ran_at: now
2789
3502
  }).eq("id", jobDetails.id).eq("queue", this.queueName);
3503
+ query2 = this.applyPrefixFilters(query2);
3504
+ const { error: error2 } = await query2;
2790
3505
  if (error2)
2791
3506
  throw error2;
2792
3507
  return;
2793
3508
  }
2794
3509
  if (jobDetails.status === "COMPLETED" /* COMPLETED */ || jobDetails.status === "FAILED" /* FAILED */) {
2795
- const { error: error2 } = await this.client.from("job_queue").update({
3510
+ let query2 = this.client.from(this.tableName).update({
2796
3511
  output: jobDetails.output ?? null,
2797
3512
  error: jobDetails.error ?? null,
2798
3513
  error_code: jobDetails.error_code ?? null,
@@ -2804,11 +3519,13 @@ class SupabaseQueueStorage {
2804
3519
  completed_at: now,
2805
3520
  last_ran_at: now
2806
3521
  }).eq("id", jobDetails.id).eq("queue", this.queueName);
3522
+ query2 = this.applyPrefixFilters(query2);
3523
+ const { error: error2 } = await query2;
2807
3524
  if (error2)
2808
3525
  throw error2;
2809
3526
  return;
2810
3527
  }
2811
- const { error } = await this.client.from("job_queue").update({
3528
+ let query = this.client.from(this.tableName).update({
2812
3529
  status: jobDetails.status,
2813
3530
  output: jobDetails.output ?? null,
2814
3531
  error: jobDetails.error ?? null,
@@ -2817,17 +3534,23 @@ class SupabaseQueueStorage {
2817
3534
  run_attempts: nextAttempts,
2818
3535
  last_ran_at: now
2819
3536
  }).eq("id", jobDetails.id).eq("queue", this.queueName);
3537
+ query = this.applyPrefixFilters(query);
3538
+ const { error } = await query;
2820
3539
  if (error)
2821
3540
  throw error;
2822
3541
  }
2823
3542
  async deleteAll() {
2824
- const { error } = await this.client.from("job_queue").delete().eq("queue", this.queueName);
3543
+ let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
3544
+ query = this.applyPrefixFilters(query);
3545
+ const { error } = await query;
2825
3546
  if (error)
2826
3547
  throw error;
2827
3548
  }
2828
3549
  async outputForInput(input) {
2829
3550
  const fingerprint = await makeFingerprint7(input);
2830
- const { data, error } = await this.client.from("job_queue").select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", "COMPLETED" /* COMPLETED */).single();
3551
+ let query = this.client.from(this.tableName).select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", "COMPLETED" /* COMPLETED */);
3552
+ query = this.applyPrefixFilters(query);
3553
+ const { data, error } = await query.single();
2831
3554
  if (error) {
2832
3555
  if (error.code === "PGRST116")
2833
3556
  return null;
@@ -2836,47 +3559,622 @@ class SupabaseQueueStorage {
2836
3559
  return data?.output ?? null;
2837
3560
  }
2838
3561
  async abort(jobId) {
2839
- const { error } = await this.client.from("job_queue").update({ status: "ABORTING" /* ABORTING */ }).eq("id", jobId).eq("queue", this.queueName);
3562
+ let query = this.client.from(this.tableName).update({ status: "ABORTING" /* ABORTING */ }).eq("id", jobId).eq("queue", this.queueName);
3563
+ query = this.applyPrefixFilters(query);
3564
+ const { error } = await query;
2840
3565
  if (error)
2841
3566
  throw error;
2842
3567
  }
2843
3568
  async getByRunId(job_run_id) {
2844
- const { data, error } = await this.client.from("job_queue").select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
3569
+ let query = this.client.from(this.tableName).select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
3570
+ query = this.applyPrefixFilters(query);
3571
+ const { data, error } = await query;
2845
3572
  if (error)
2846
3573
  throw error;
2847
3574
  return data ?? [];
2848
3575
  }
2849
3576
  async saveProgress(jobId, progress, message, details) {
2850
- const { error } = await this.client.from("job_queue").update({
3577
+ let query = this.client.from(this.tableName).update({
2851
3578
  progress,
2852
3579
  progress_message: message,
2853
3580
  progress_details: details
2854
3581
  }).eq("id", jobId).eq("queue", this.queueName);
3582
+ query = this.applyPrefixFilters(query);
3583
+ const { error } = await query;
2855
3584
  if (error)
2856
3585
  throw error;
2857
3586
  }
2858
3587
  async delete(jobId) {
2859
- const { error } = await this.client.from("job_queue").delete().eq("id", jobId).eq("queue", this.queueName);
3588
+ let query = this.client.from(this.tableName).delete().eq("id", jobId).eq("queue", this.queueName);
3589
+ query = this.applyPrefixFilters(query);
3590
+ const { error } = await query;
2860
3591
  if (error)
2861
3592
  throw error;
2862
3593
  }
2863
3594
  async deleteJobsByStatusAndAge(status, olderThanMs) {
2864
3595
  const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
2865
- const { error } = await this.client.from("job_queue").delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
3596
+ let query = this.client.from(this.tableName).delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
3597
+ query = this.applyPrefixFilters(query);
3598
+ const { error } = await query;
2866
3599
  if (error)
2867
3600
  throw error;
2868
3601
  }
2869
- }
2870
- // src/kv/IndexedDbKvRepository.ts
2871
- import { createServiceToken as createServiceToken21 } from "@workglow/util";
2872
-
2873
- // src/tabular/IndexedDbTabularRepository.ts
2874
- import { createServiceToken as createServiceToken20 } from "@workglow/util";
2875
-
2876
- // src/util/IndexedDbTable.ts
2877
- var METADATA_STORE_NAME = "__schema_metadata__";
2878
- async function saveSchemaMetadata(db, tableName, snapshot) {
2879
- return new Promise((resolve, reject) => {
3602
+ matchesPrefixFilter(job, prefixFilter) {
3603
+ if (!job)
3604
+ return false;
3605
+ if (job.queue !== this.queueName) {
3606
+ return false;
3607
+ }
3608
+ if (prefixFilter && Object.keys(prefixFilter).length === 0) {
3609
+ return true;
3610
+ }
3611
+ const filterValues = prefixFilter ?? this.prefixValues;
3612
+ if (Object.keys(filterValues).length === 0) {
3613
+ return true;
3614
+ }
3615
+ for (const [key, value] of Object.entries(filterValues)) {
3616
+ if (job[key] !== value) {
3617
+ return false;
3618
+ }
3619
+ }
3620
+ return true;
3621
+ }
3622
+ isCustomPrefixFilter(prefixFilter) {
3623
+ if (prefixFilter === undefined) {
3624
+ return false;
3625
+ }
3626
+ if (Object.keys(prefixFilter).length === 0) {
3627
+ return true;
3628
+ }
3629
+ const instanceKeys = Object.keys(this.prefixValues);
3630
+ const filterKeys = Object.keys(prefixFilter);
3631
+ if (instanceKeys.length !== filterKeys.length) {
3632
+ return true;
3633
+ }
3634
+ for (const key of instanceKeys) {
3635
+ if (this.prefixValues[key] !== prefixFilter[key]) {
3636
+ return true;
3637
+ }
3638
+ }
3639
+ return false;
3640
+ }
3641
+ async getAllJobsWithFilter(prefixFilter) {
3642
+ let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
3643
+ for (const [key, value] of Object.entries(prefixFilter)) {
3644
+ query = query.eq(key, value);
3645
+ }
3646
+ const { data, error } = await query;
3647
+ if (error)
3648
+ throw error;
3649
+ return data ?? [];
3650
+ }
3651
+ subscribeToChanges(callback, options) {
3652
+ return this.subscribeToChangesWithRealtime(callback, options?.prefixFilter);
3653
+ }
3654
+ subscribeToChangesWithRealtime(callback, prefixFilter) {
3655
+ const channelName = `queue-${this.tableName}-${this.queueName}-${Date.now()}`;
3656
+ this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
3657
+ event: "*",
3658
+ schema: "public",
3659
+ table: this.tableName,
3660
+ filter: `queue=eq.${this.queueName}`
3661
+ }, (payload) => {
3662
+ const newJob = payload.new;
3663
+ const oldJob = payload.old;
3664
+ const newMatches = this.matchesPrefixFilter(newJob, prefixFilter);
3665
+ const oldMatches = this.matchesPrefixFilter(oldJob, prefixFilter);
3666
+ if (!newMatches && !oldMatches) {
3667
+ return;
3668
+ }
3669
+ callback({
3670
+ type: payload.eventType.toUpperCase(),
3671
+ old: oldJob && Object.keys(oldJob).length > 0 ? oldJob : undefined,
3672
+ new: newJob && Object.keys(newJob).length > 0 ? newJob : undefined
3673
+ });
3674
+ }).subscribe();
3675
+ return () => {
3676
+ if (this.realtimeChannel) {
3677
+ this.client.removeChannel(this.realtimeChannel);
3678
+ this.realtimeChannel = null;
3679
+ }
3680
+ };
3681
+ }
3682
+ getPollingManager() {
3683
+ if (!this.pollingManager) {
3684
+ this.pollingManager = new PollingSubscriptionManager(async () => {
3685
+ const jobs = await this.getAllJobs();
3686
+ return new Map(jobs.map((j) => [j.id, j]));
3687
+ }, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
3688
+ insert: (item) => ({ type: "INSERT", new: item }),
3689
+ update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
3690
+ delete: (item) => ({ type: "DELETE", old: item })
3691
+ });
3692
+ }
3693
+ return this.pollingManager;
3694
+ }
3695
+ subscribeWithCustomPrefixFilterPolling(callback, prefixFilter, intervalMs) {
3696
+ let lastKnownJobs = new Map;
3697
+ let cancelled = false;
3698
+ const poll = async () => {
3699
+ if (cancelled)
3700
+ return;
3701
+ try {
3702
+ const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
3703
+ if (cancelled)
3704
+ return;
3705
+ const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
3706
+ for (const [id, job] of currentMap) {
3707
+ const old = lastKnownJobs.get(id);
3708
+ if (!old) {
3709
+ callback({ type: "INSERT", new: job });
3710
+ } else if (JSON.stringify(old) !== JSON.stringify(job)) {
3711
+ callback({ type: "UPDATE", old, new: job });
3712
+ }
3713
+ }
3714
+ for (const [id, job] of lastKnownJobs) {
3715
+ if (!currentMap.has(id)) {
3716
+ callback({ type: "DELETE", old: job });
3717
+ }
3718
+ }
3719
+ lastKnownJobs = currentMap;
3720
+ } catch {}
3721
+ };
3722
+ const intervalId = setInterval(poll, intervalMs);
3723
+ poll();
3724
+ return () => {
3725
+ cancelled = true;
3726
+ clearInterval(intervalId);
3727
+ };
3728
+ }
3729
+ subscribeToChangesWithPolling(callback, options) {
3730
+ const intervalMs = options?.pollingIntervalMs ?? 1000;
3731
+ if (this.isCustomPrefixFilter(options?.prefixFilter)) {
3732
+ return this.subscribeWithCustomPrefixFilterPolling(callback, options.prefixFilter, intervalMs);
3733
+ }
3734
+ const manager = this.getPollingManager();
3735
+ return manager.subscribe(callback, { intervalMs });
3736
+ }
3737
+ }
3738
+ // src/limiter/PostgresRateLimiterStorage.ts
3739
+ import { createServiceToken as createServiceToken22 } from "@workglow/util";
3740
+ var POSTGRES_RATE_LIMITER_STORAGE = createServiceToken22("ratelimiter.storage.postgres");
3741
+
3742
+ class PostgresRateLimiterStorage {
3743
+ db;
3744
+ prefixes;
3745
+ prefixValues;
3746
+ executionTableName;
3747
+ nextAvailableTableName;
3748
+ constructor(db, options) {
3749
+ this.db = db;
3750
+ this.prefixes = options?.prefixes ?? [];
3751
+ this.prefixValues = options?.prefixValues ?? {};
3752
+ if (this.prefixes.length > 0) {
3753
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
3754
+ this.executionTableName = `rate_limit_executions_${prefixNames}`;
3755
+ this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
3756
+ } else {
3757
+ this.executionTableName = "rate_limit_executions";
3758
+ this.nextAvailableTableName = "rate_limit_next_available";
3759
+ }
3760
+ }
3761
+ getPrefixColumnType(type) {
3762
+ return type === "uuid" ? "UUID" : "INTEGER";
3763
+ }
3764
+ buildPrefixColumnsSql() {
3765
+ if (this.prefixes.length === 0)
3766
+ return "";
3767
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
3768
+ `) + `,
3769
+ `;
3770
+ }
3771
+ getPrefixColumnNames() {
3772
+ return this.prefixes.map((p) => p.name);
3773
+ }
3774
+ buildPrefixWhereClause(startParam) {
3775
+ if (this.prefixes.length === 0) {
3776
+ return { conditions: "", params: [] };
3777
+ }
3778
+ const conditions = this.prefixes.map((p, i) => `${p.name} = $${startParam + i}`).join(" AND ");
3779
+ const params = this.prefixes.map((p) => this.prefixValues[p.name]);
3780
+ return { conditions: " AND " + conditions, params };
3781
+ }
3782
+ getPrefixParamValues() {
3783
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
3784
+ }
3785
+ async setupDatabase() {
3786
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
3787
+ const prefixColumnNames = this.getPrefixColumnNames();
3788
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3789
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
3790
+ await this.db.query(`
3791
+ CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
3792
+ id SERIAL PRIMARY KEY,
3793
+ ${prefixColumnsSql}queue_name TEXT NOT NULL,
3794
+ executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
3795
+ )
3796
+ `);
3797
+ await this.db.query(`
3798
+ CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
3799
+ ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
3800
+ `);
3801
+ const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
3802
+ await this.db.query(`
3803
+ CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
3804
+ ${prefixColumnsSql}queue_name TEXT NOT NULL,
3805
+ next_available_at TIMESTAMP WITH TIME ZONE,
3806
+ PRIMARY KEY (${primaryKeyColumns})
3807
+ )
3808
+ `);
3809
+ }
3810
+ async recordExecution(queueName) {
3811
+ const prefixColumnNames = this.getPrefixColumnNames();
3812
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3813
+ const prefixParamValues = this.getPrefixParamValues();
3814
+ const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
3815
+ const queueParamNum = prefixColumnNames.length + 1;
3816
+ await this.db.query(`
3817
+ INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
3818
+ VALUES (${prefixParamPlaceholders}$${queueParamNum})
3819
+ `, [...prefixParamValues, queueName]);
3820
+ }
3821
+ async getExecutionCount(queueName, windowStartTime) {
3822
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
3823
+ const result = await this.db.query(`
3824
+ SELECT COUNT(*) AS count
3825
+ FROM ${this.executionTableName}
3826
+ WHERE queue_name = $1 AND executed_at > $2${prefixConditions}
3827
+ `, [queueName, windowStartTime, ...prefixParams]);
3828
+ return parseInt(result.rows[0]?.count ?? "0", 10);
3829
+ }
3830
+ async getOldestExecutionAtOffset(queueName, offset) {
3831
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
3832
+ const result = await this.db.query(`
3833
+ SELECT executed_at
3834
+ FROM ${this.executionTableName}
3835
+ WHERE queue_name = $1${prefixConditions}
3836
+ ORDER BY executed_at ASC
3837
+ LIMIT 1 OFFSET $2
3838
+ `, [queueName, offset, ...prefixParams]);
3839
+ const executedAt = result.rows[0]?.executed_at;
3840
+ if (!executedAt)
3841
+ return;
3842
+ return new Date(executedAt).toISOString();
3843
+ }
3844
+ async getNextAvailableTime(queueName) {
3845
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
3846
+ const result = await this.db.query(`
3847
+ SELECT next_available_at
3848
+ FROM ${this.nextAvailableTableName}
3849
+ WHERE queue_name = $1${prefixConditions}
3850
+ `, [queueName, ...prefixParams]);
3851
+ const nextAvailableAt = result.rows[0]?.next_available_at;
3852
+ if (!nextAvailableAt)
3853
+ return;
3854
+ return new Date(nextAvailableAt).toISOString();
3855
+ }
3856
+ async setNextAvailableTime(queueName, nextAvailableAt) {
3857
+ const prefixColumnNames = this.getPrefixColumnNames();
3858
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3859
+ const prefixParamValues = this.getPrefixParamValues();
3860
+ const prefixParamPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
3861
+ const baseParamStart = prefixColumnNames.length + 1;
3862
+ const conflictColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
3863
+ await this.db.query(`
3864
+ INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
3865
+ VALUES (${prefixParamPlaceholders}$${baseParamStart}, $${baseParamStart + 1})
3866
+ ON CONFLICT (${conflictColumns})
3867
+ DO UPDATE SET next_available_at = EXCLUDED.next_available_at
3868
+ `, [...prefixParamValues, queueName, nextAvailableAt]);
3869
+ }
3870
+ async clear(queueName) {
3871
+ const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(2);
3872
+ await this.db.query(`DELETE FROM ${this.executionTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
3873
+ await this.db.query(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = $1${prefixConditions}`, [queueName, ...prefixParams]);
3874
+ }
3875
+ }
3876
+ // src/limiter/SqliteRateLimiterStorage.ts
3877
+ import { createServiceToken as createServiceToken23, sleep as sleep5, toSQLiteTimestamp } from "@workglow/util";
3878
+ var SQLITE_RATE_LIMITER_STORAGE = createServiceToken23("ratelimiter.storage.sqlite");
3879
+
3880
+ class SqliteRateLimiterStorage {
3881
+ db;
3882
+ prefixes;
3883
+ prefixValues;
3884
+ executionTableName;
3885
+ nextAvailableTableName;
3886
+ constructor(db, options) {
3887
+ this.db = db;
3888
+ this.prefixes = options?.prefixes ?? [];
3889
+ this.prefixValues = options?.prefixValues ?? {};
3890
+ if (this.prefixes.length > 0) {
3891
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
3892
+ this.executionTableName = `rate_limit_executions_${prefixNames}`;
3893
+ this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
3894
+ } else {
3895
+ this.executionTableName = "rate_limit_executions";
3896
+ this.nextAvailableTableName = "rate_limit_next_available";
3897
+ }
3898
+ }
3899
+ getPrefixColumnType(type) {
3900
+ return type === "uuid" ? "TEXT" : "INTEGER";
3901
+ }
3902
+ buildPrefixColumnsSql() {
3903
+ if (this.prefixes.length === 0)
3904
+ return "";
3905
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
3906
+ `) + `,
3907
+ `;
3908
+ }
3909
+ getPrefixColumnNames() {
3910
+ return this.prefixes.map((p) => p.name);
3911
+ }
3912
+ buildPrefixWhereClause() {
3913
+ if (this.prefixes.length === 0) {
3914
+ return "";
3915
+ }
3916
+ const conditions = this.prefixes.map((p) => `${p.name} = ?`).join(" AND ");
3917
+ return " AND " + conditions;
3918
+ }
3919
+ getPrefixParamValues() {
3920
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
3921
+ }
3922
+ async setupDatabase() {
3923
+ await sleep5(0);
3924
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
3925
+ const prefixColumnNames = this.getPrefixColumnNames();
3926
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3927
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
3928
+ this.db.exec(`
3929
+ CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
3930
+ id INTEGER PRIMARY KEY,
3931
+ ${prefixColumnsSql}queue_name TEXT NOT NULL,
3932
+ executed_at TEXT DEFAULT CURRENT_TIMESTAMP
3933
+ );
3934
+
3935
+ CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
3936
+ ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at);
3937
+ `);
3938
+ this.db.exec(`
3939
+ CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
3940
+ ${prefixColumnsSql}queue_name TEXT PRIMARY KEY,
3941
+ next_available_at TEXT
3942
+ );
3943
+ `);
3944
+ }
3945
+ async recordExecution(queueName) {
3946
+ const prefixColumnNames = this.getPrefixColumnNames();
3947
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3948
+ const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
3949
+ const prefixParamValues = this.getPrefixParamValues();
3950
+ const stmt = this.db.prepare(`
3951
+ INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
3952
+ VALUES (${prefixPlaceholders}?)
3953
+ `);
3954
+ stmt.run(...prefixParamValues, queueName);
3955
+ }
3956
+ async getExecutionCount(queueName, windowStartTime) {
3957
+ const prefixConditions = this.buildPrefixWhereClause();
3958
+ const prefixParams = this.getPrefixParamValues();
3959
+ const thresholdTime = toSQLiteTimestamp(new Date(windowStartTime));
3960
+ const stmt = this.db.prepare(`
3961
+ SELECT COUNT(*) AS count
3962
+ FROM ${this.executionTableName}
3963
+ WHERE queue_name = ? AND executed_at > ?${prefixConditions}
3964
+ `);
3965
+ const result = stmt.get(queueName, thresholdTime, ...prefixParams);
3966
+ return result?.count ?? 0;
3967
+ }
3968
+ async getOldestExecutionAtOffset(queueName, offset) {
3969
+ const prefixConditions = this.buildPrefixWhereClause();
3970
+ const prefixParams = this.getPrefixParamValues();
3971
+ const stmt = this.db.prepare(`
3972
+ SELECT executed_at
3973
+ FROM ${this.executionTableName}
3974
+ WHERE queue_name = ?${prefixConditions}
3975
+ ORDER BY executed_at ASC
3976
+ LIMIT 1 OFFSET ?
3977
+ `);
3978
+ const result = stmt.get(queueName, ...prefixParams, offset);
3979
+ if (!result)
3980
+ return;
3981
+ return result.executed_at + "Z";
3982
+ }
3983
+ async getNextAvailableTime(queueName) {
3984
+ const prefixConditions = this.buildPrefixWhereClause();
3985
+ const prefixParams = this.getPrefixParamValues();
3986
+ const stmt = this.db.prepare(`
3987
+ SELECT next_available_at
3988
+ FROM ${this.nextAvailableTableName}
3989
+ WHERE queue_name = ?${prefixConditions}
3990
+ `);
3991
+ const result = stmt.get(queueName, ...prefixParams);
3992
+ if (!result?.next_available_at)
3993
+ return;
3994
+ return result.next_available_at + "Z";
3995
+ }
3996
+ async setNextAvailableTime(queueName, nextAvailableAt) {
3997
+ const prefixColumnNames = this.getPrefixColumnNames();
3998
+ const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
3999
+ const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
4000
+ const prefixParamValues = this.getPrefixParamValues();
4001
+ const stmt = this.db.prepare(`
4002
+ INSERT INTO ${this.nextAvailableTableName} (${prefixColumnsInsert}queue_name, next_available_at)
4003
+ VALUES (${prefixPlaceholders}?, ?)
4004
+ ON CONFLICT(queue_name) DO UPDATE SET next_available_at = excluded.next_available_at
4005
+ `);
4006
+ stmt.run(...prefixParamValues, queueName, nextAvailableAt);
4007
+ }
4008
+ async clear(queueName) {
4009
+ const prefixConditions = this.buildPrefixWhereClause();
4010
+ const prefixParams = this.getPrefixParamValues();
4011
+ this.db.prepare(`DELETE FROM ${this.executionTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
4012
+ this.db.prepare(`DELETE FROM ${this.nextAvailableTableName} WHERE queue_name = ?${prefixConditions}`).run(queueName, ...prefixParams);
4013
+ }
4014
+ }
4015
+ // src/limiter/SupabaseRateLimiterStorage.ts
4016
+ import { createServiceToken as createServiceToken24 } from "@workglow/util";
4017
+ var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken24("ratelimiter.storage.supabase");
4018
+
4019
+ class SupabaseRateLimiterStorage {
4020
+ client;
4021
+ prefixes;
4022
+ prefixValues;
4023
+ executionTableName;
4024
+ nextAvailableTableName;
4025
+ constructor(client, options) {
4026
+ this.client = client;
4027
+ this.prefixes = options?.prefixes ?? [];
4028
+ this.prefixValues = options?.prefixValues ?? {};
4029
+ if (this.prefixes.length > 0) {
4030
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
4031
+ this.executionTableName = `rate_limit_executions_${prefixNames}`;
4032
+ this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
4033
+ } else {
4034
+ this.executionTableName = "rate_limit_executions";
4035
+ this.nextAvailableTableName = "rate_limit_next_available";
4036
+ }
4037
+ }
4038
+ getPrefixColumnType(type) {
4039
+ return type === "uuid" ? "UUID" : "INTEGER";
4040
+ }
4041
+ buildPrefixColumnsSql() {
4042
+ if (this.prefixes.length === 0)
4043
+ return "";
4044
+ return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
4045
+ `) + `,
4046
+ `;
4047
+ }
4048
+ getPrefixColumnNames() {
4049
+ return this.prefixes.map((p) => p.name);
4050
+ }
4051
+ applyPrefixFilters(query) {
4052
+ let result = query;
4053
+ for (const prefix of this.prefixes) {
4054
+ result = result.eq(prefix.name, this.prefixValues[prefix.name]);
4055
+ }
4056
+ return result;
4057
+ }
4058
+ getPrefixInsertValues() {
4059
+ const values = {};
4060
+ for (const prefix of this.prefixes) {
4061
+ values[prefix.name] = this.prefixValues[prefix.name];
4062
+ }
4063
+ return values;
4064
+ }
4065
+ async setupDatabase() {
4066
+ const prefixColumnsSql = this.buildPrefixColumnsSql();
4067
+ const prefixColumnNames = this.getPrefixColumnNames();
4068
+ const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
4069
+ const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
4070
+ const createExecTableSql = `
4071
+ CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
4072
+ id SERIAL PRIMARY KEY,
4073
+ ${prefixColumnsSql}queue_name TEXT NOT NULL,
4074
+ executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
4075
+ )
4076
+ `;
4077
+ const { error: execTableError } = await this.client.rpc("exec_sql", {
4078
+ query: createExecTableSql
4079
+ });
4080
+ if (execTableError && execTableError.code !== "42P07") {
4081
+ throw execTableError;
4082
+ }
4083
+ const createExecIndexSql = `
4084
+ CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
4085
+ ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
4086
+ `;
4087
+ await this.client.rpc("exec_sql", { query: createExecIndexSql });
4088
+ const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
4089
+ const createNextTableSql = `
4090
+ CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
4091
+ ${prefixColumnsSql}queue_name TEXT NOT NULL,
4092
+ next_available_at TIMESTAMP WITH TIME ZONE,
4093
+ PRIMARY KEY (${primaryKeyColumns})
4094
+ )
4095
+ `;
4096
+ const { error: nextTableError } = await this.client.rpc("exec_sql", {
4097
+ query: createNextTableSql
4098
+ });
4099
+ if (nextTableError && nextTableError.code !== "42P07") {
4100
+ throw nextTableError;
4101
+ }
4102
+ }
4103
+ async recordExecution(queueName) {
4104
+ const prefixInsertValues = this.getPrefixInsertValues();
4105
+ const { error } = await this.client.from(this.executionTableName).insert({
4106
+ ...prefixInsertValues,
4107
+ queue_name: queueName
4108
+ });
4109
+ if (error)
4110
+ throw error;
4111
+ }
4112
+ async getExecutionCount(queueName, windowStartTime) {
4113
+ let query = this.client.from(this.executionTableName).select("*", { count: "exact", head: true }).eq("queue_name", queueName).gt("executed_at", windowStartTime);
4114
+ query = this.applyPrefixFilters(query);
4115
+ const { count, error } = await query;
4116
+ if (error)
4117
+ throw error;
4118
+ return count ?? 0;
4119
+ }
4120
+ async getOldestExecutionAtOffset(queueName, offset) {
4121
+ let query = this.client.from(this.executionTableName).select("executed_at").eq("queue_name", queueName);
4122
+ query = this.applyPrefixFilters(query);
4123
+ const { data, error } = await query.order("executed_at", { ascending: true }).range(offset, offset);
4124
+ if (error)
4125
+ throw error;
4126
+ if (!data || data.length === 0)
4127
+ return;
4128
+ return new Date(data[0].executed_at).toISOString();
4129
+ }
4130
+ async getNextAvailableTime(queueName) {
4131
+ let query = this.client.from(this.nextAvailableTableName).select("next_available_at").eq("queue_name", queueName);
4132
+ query = this.applyPrefixFilters(query);
4133
+ const { data, error } = await query.single();
4134
+ if (error) {
4135
+ if (error.code === "PGRST116")
4136
+ return;
4137
+ throw error;
4138
+ }
4139
+ if (!data?.next_available_at)
4140
+ return;
4141
+ return new Date(data.next_available_at).toISOString();
4142
+ }
4143
+ async setNextAvailableTime(queueName, nextAvailableAt) {
4144
+ const prefixInsertValues = this.getPrefixInsertValues();
4145
+ const { error } = await this.client.from(this.nextAvailableTableName).upsert({
4146
+ ...prefixInsertValues,
4147
+ queue_name: queueName,
4148
+ next_available_at: nextAvailableAt
4149
+ }, {
4150
+ onConflict: this.prefixes.length > 0 ? `${this.getPrefixColumnNames().join(",")},queue_name` : "queue_name"
4151
+ });
4152
+ if (error)
4153
+ throw error;
4154
+ }
4155
+ async clear(queueName) {
4156
+ let execQuery = this.client.from(this.executionTableName).delete().eq("queue_name", queueName);
4157
+ execQuery = this.applyPrefixFilters(execQuery);
4158
+ const { error: execError } = await execQuery;
4159
+ if (execError)
4160
+ throw execError;
4161
+ let nextQuery = this.client.from(this.nextAvailableTableName).delete().eq("queue_name", queueName);
4162
+ nextQuery = this.applyPrefixFilters(nextQuery);
4163
+ const { error: nextError } = await nextQuery;
4164
+ if (nextError)
4165
+ throw nextError;
4166
+ }
4167
+ }
4168
+ // src/kv/IndexedDbKvRepository.ts
4169
+ import { createServiceToken as createServiceToken26 } from "@workglow/util";
4170
+
4171
+ // src/tabular/IndexedDbTabularRepository.ts
4172
+ import { createServiceToken as createServiceToken25 } from "@workglow/util";
4173
+
4174
+ // src/util/IndexedDbTable.ts
4175
+ var METADATA_STORE_NAME = "__schema_metadata__";
4176
+ async function saveSchemaMetadata(db, tableName, snapshot) {
4177
+ return new Promise((resolve, reject) => {
2880
4178
  try {
2881
4179
  const transaction = db.transaction(METADATA_STORE_NAME, "readwrite");
2882
4180
  const store = transaction.objectStore(METADATA_STORE_NAME);
@@ -3211,7 +4509,7 @@ async function dropIndexedDbTable(tableName) {
3211
4509
  }
3212
4510
 
3213
4511
  // src/tabular/IndexedDbTabularRepository.ts
3214
- var IDB_TABULAR_REPOSITORY = createServiceToken20("storage.tabularRepository.indexedDb");
4512
+ var IDB_TABULAR_REPOSITORY = createServiceToken25("storage.tabularRepository.indexedDb");
3215
4513
 
3216
4514
  class IndexedDbTabularRepository extends TabularRepository {
3217
4515
  table;
@@ -3547,7 +4845,7 @@ class IndexedDbTabularRepository extends TabularRepository {
3547
4845
  }
3548
4846
 
3549
4847
  // src/kv/IndexedDbKvRepository.ts
3550
- var IDB_KV_REPOSITORY = createServiceToken21("storage.kvRepository.indexedDb");
4848
+ var IDB_KV_REPOSITORY = createServiceToken26("storage.kvRepository.indexedDb");
3551
4849
 
3552
4850
  class IndexedDbKvRepository extends KvViaTabularRepository {
3553
4851
  dbName;
@@ -3559,18 +4857,42 @@ class IndexedDbKvRepository extends KvViaTabularRepository {
3559
4857
  }
3560
4858
  }
3561
4859
  // src/queue/IndexedDbQueueStorage.ts
3562
- import { createServiceToken as createServiceToken22, makeFingerprint as makeFingerprint8, uuid4 as uuid45 } from "@workglow/util";
3563
- var INDEXED_DB_QUEUE_STORAGE = createServiceToken22("jobqueue.storage.indexedDb");
4860
+ import { createServiceToken as createServiceToken27, makeFingerprint as makeFingerprint8, uuid4 as uuid45 } from "@workglow/util";
4861
+ var INDEXED_DB_QUEUE_STORAGE = createServiceToken27("jobqueue.storage.indexedDb");
3564
4862
 
3565
4863
  class IndexedDbQueueStorage {
3566
4864
  queueName;
3567
4865
  db;
3568
4866
  tableName;
3569
4867
  migrationOptions;
3570
- constructor(queueName, migrationOptions = {}) {
4868
+ prefixes;
4869
+ prefixValues;
4870
+ pollingManager = null;
4871
+ constructor(queueName, options = {}) {
3571
4872
  this.queueName = queueName;
3572
- this.tableName = `jobs_${queueName}`;
3573
- this.migrationOptions = migrationOptions;
4873
+ this.migrationOptions = options;
4874
+ this.prefixes = options.prefixes ?? [];
4875
+ this.prefixValues = options.prefixValues ?? {};
4876
+ if (this.prefixes.length > 0) {
4877
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
4878
+ this.tableName = `jobs_${prefixNames}`;
4879
+ } else {
4880
+ this.tableName = "jobs";
4881
+ }
4882
+ }
4883
+ getPrefixColumnNames() {
4884
+ return this.prefixes.map((p) => p.name);
4885
+ }
4886
+ matchesPrefixes(job) {
4887
+ for (const [key, value] of Object.entries(this.prefixValues)) {
4888
+ if (job[key] !== value) {
4889
+ return false;
4890
+ }
4891
+ }
4892
+ return true;
4893
+ }
4894
+ getPrefixKeyValues() {
4895
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
3574
4896
  }
3575
4897
  async getDb() {
3576
4898
  if (this.db)
@@ -3579,25 +4901,29 @@ class IndexedDbQueueStorage {
3579
4901
  return this.db;
3580
4902
  }
3581
4903
  async setupDatabase() {
4904
+ const prefixColumnNames = this.getPrefixColumnNames();
4905
+ const buildKeyPath = (basePath) => {
4906
+ return [...prefixColumnNames, ...basePath];
4907
+ };
3582
4908
  const expectedIndexes = [
3583
4909
  {
3584
- name: "status",
3585
- keyPath: `status`,
4910
+ name: "queue_status",
4911
+ keyPath: buildKeyPath(["queue", "status"]),
3586
4912
  options: { unique: false }
3587
4913
  },
3588
4914
  {
3589
- name: "status_run_after",
3590
- keyPath: ["status", "run_after"],
4915
+ name: "queue_status_run_after",
4916
+ keyPath: buildKeyPath(["queue", "status", "run_after"]),
3591
4917
  options: { unique: false }
3592
4918
  },
3593
4919
  {
3594
- name: "job_run_id",
3595
- keyPath: `job_run_id`,
4920
+ name: "queue_job_run_id",
4921
+ keyPath: buildKeyPath(["queue", "job_run_id"]),
3596
4922
  options: { unique: false }
3597
4923
  },
3598
4924
  {
3599
- name: "fingerprint_status",
3600
- keyPath: ["fingerprint", "status"],
4925
+ name: "queue_fingerprint_status",
4926
+ keyPath: buildKeyPath(["queue", "fingerprint", "status"]),
3601
4927
  options: { unique: false }
3602
4928
  }
3603
4929
  ];
@@ -3606,21 +4932,25 @@ class IndexedDbQueueStorage {
3606
4932
  async add(job) {
3607
4933
  const db = await this.getDb();
3608
4934
  const now = new Date().toISOString();
3609
- job.id = job.id ?? uuid45();
3610
- job.job_run_id = job.job_run_id ?? uuid45();
3611
- job.queue = this.queueName;
3612
- job.fingerprint = await makeFingerprint8(job.input);
3613
- job.status = "PENDING" /* PENDING */;
3614
- job.progress = 0;
3615
- job.progress_message = "";
3616
- job.progress_details = null;
3617
- job.created_at = now;
3618
- job.run_after = now;
4935
+ const jobWithPrefixes = job;
4936
+ jobWithPrefixes.id = jobWithPrefixes.id ?? uuid45();
4937
+ jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid45();
4938
+ jobWithPrefixes.queue = this.queueName;
4939
+ jobWithPrefixes.fingerprint = await makeFingerprint8(jobWithPrefixes.input);
4940
+ jobWithPrefixes.status = "PENDING" /* PENDING */;
4941
+ jobWithPrefixes.progress = 0;
4942
+ jobWithPrefixes.progress_message = "";
4943
+ jobWithPrefixes.progress_details = null;
4944
+ jobWithPrefixes.created_at = now;
4945
+ jobWithPrefixes.run_after = now;
4946
+ for (const [key, value] of Object.entries(this.prefixValues)) {
4947
+ jobWithPrefixes[key] = value;
4948
+ }
3619
4949
  const tx = db.transaction(this.tableName, "readwrite");
3620
4950
  const store = tx.objectStore(this.tableName);
3621
4951
  return new Promise((resolve, reject) => {
3622
- const request = store.add(job);
3623
- tx.oncomplete = () => resolve(job.id);
4952
+ const request = store.add(jobWithPrefixes);
4953
+ tx.oncomplete = () => resolve(jobWithPrefixes.id);
3624
4954
  tx.onerror = () => reject(tx.error);
3625
4955
  request.onerror = () => reject(request.error);
3626
4956
  });
@@ -3631,7 +4961,14 @@ class IndexedDbQueueStorage {
3631
4961
  const store = tx.objectStore(this.tableName);
3632
4962
  const request = store.get(id);
3633
4963
  return new Promise((resolve, reject) => {
3634
- request.onsuccess = () => resolve(request.result);
4964
+ request.onsuccess = () => {
4965
+ const job = request.result;
4966
+ if (job && job.queue === this.queueName && this.matchesPrefixes(job)) {
4967
+ resolve(job);
4968
+ } else {
4969
+ resolve(undefined);
4970
+ }
4971
+ };
3635
4972
  request.onerror = () => reject(request.error);
3636
4973
  tx.onerror = () => reject(tx.error);
3637
4974
  });
@@ -3640,10 +4977,11 @@ class IndexedDbQueueStorage {
3640
4977
  const db = await this.getDb();
3641
4978
  const tx = db.transaction(this.tableName, "readonly");
3642
4979
  const store = tx.objectStore(this.tableName);
3643
- const index = store.index("status_run_after");
4980
+ const index = store.index("queue_status_run_after");
4981
+ const prefixKeyValues = this.getPrefixKeyValues();
3644
4982
  return new Promise((resolve, reject) => {
3645
4983
  const ret = new Map;
3646
- const keyRange = IDBKeyRange.bound([status, ""], [status, "￿"]);
4984
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, status, ""], [...prefixKeyValues, this.queueName, status, "￿"]);
3647
4985
  const cursorRequest = index.openCursor(keyRange);
3648
4986
  const handleCursor = (e) => {
3649
4987
  const cursor = e.target.result;
@@ -3651,7 +4989,10 @@ class IndexedDbQueueStorage {
3651
4989
  resolve(Array.from(ret.values()));
3652
4990
  return;
3653
4991
  }
3654
- ret.set(cursor.value.id, cursor.value);
4992
+ const job = cursor.value;
4993
+ if (this.matchesPrefixes(job)) {
4994
+ ret.set(cursor.value.id, cursor.value);
4995
+ }
3655
4996
  cursor.continue();
3656
4997
  };
3657
4998
  cursorRequest.onsuccess = handleCursor;
@@ -3659,14 +5000,15 @@ class IndexedDbQueueStorage {
3659
5000
  tx.onerror = () => reject(tx.error);
3660
5001
  });
3661
5002
  }
3662
- async next() {
5003
+ async next(workerId) {
3663
5004
  const db = await this.getDb();
3664
5005
  const tx = db.transaction(this.tableName, "readwrite");
3665
5006
  const store = tx.objectStore(this.tableName);
3666
- const index = store.index("status_run_after");
5007
+ const index = store.index("queue_status_run_after");
3667
5008
  const now = new Date().toISOString();
5009
+ const prefixKeyValues = this.getPrefixKeyValues();
3668
5010
  return new Promise((resolve, reject) => {
3669
- const cursorRequest = index.openCursor(IDBKeyRange.bound(["PENDING" /* PENDING */, ""], ["PENDING" /* PENDING */, now], false, true));
5011
+ const cursorRequest = index.openCursor(IDBKeyRange.bound([...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, ""], [...prefixKeyValues, this.queueName, "PENDING" /* PENDING */, now], false, true));
3670
5012
  let jobToReturn;
3671
5013
  cursorRequest.onsuccess = (e) => {
3672
5014
  const cursor = e.target.result;
@@ -3679,12 +5021,13 @@ class IndexedDbQueueStorage {
3679
5021
  return;
3680
5022
  }
3681
5023
  const job = cursor.value;
3682
- if (job.status !== "PENDING" /* PENDING */) {
5024
+ if (job.queue !== this.queueName || job.status !== "PENDING" /* PENDING */ || !this.matchesPrefixes(job)) {
3683
5025
  cursor.continue();
3684
5026
  return;
3685
5027
  }
3686
5028
  job.status = "PROCESSING" /* PROCESSING */;
3687
5029
  job.last_ran_at = now;
5030
+ job.worker_id = workerId ?? null;
3688
5031
  try {
3689
5032
  const updateRequest = store.put(job);
3690
5033
  updateRequest.onsuccess = () => {
@@ -3708,11 +5051,13 @@ class IndexedDbQueueStorage {
3708
5051
  }
3709
5052
  async size(status = "PENDING" /* PENDING */) {
3710
5053
  const db = await this.getDb();
5054
+ const prefixKeyValues = this.getPrefixKeyValues();
3711
5055
  return new Promise((resolve, reject) => {
3712
5056
  const tx = db.transaction(this.tableName, "readonly");
3713
5057
  const store = tx.objectStore(this.tableName);
3714
- const index = store.index("status");
3715
- const request = index.count(status);
5058
+ const index = store.index("queue_status");
5059
+ const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
5060
+ const request = index.count(keyRange);
3716
5061
  request.onsuccess = () => resolve(request.result);
3717
5062
  request.onerror = () => reject(request.error);
3718
5063
  tx.onerror = () => reject(tx.error);
@@ -3726,9 +5071,18 @@ class IndexedDbQueueStorage {
3726
5071
  const getReq = store.get(job.id);
3727
5072
  getReq.onsuccess = () => {
3728
5073
  const existing = getReq.result;
3729
- const currentAttempts = existing?.run_attempts ?? 0;
5074
+ if (!existing || existing.queue !== this.queueName || !this.matchesPrefixes(existing)) {
5075
+ reject(new Error(`Job ${job.id} not found or does not belong to queue ${this.queueName}`));
5076
+ return;
5077
+ }
5078
+ const currentAttempts = existing.run_attempts ?? 0;
3730
5079
  job.run_attempts = currentAttempts + 1;
3731
- const putReq = store.put(job);
5080
+ job.queue = this.queueName;
5081
+ const jobWithPrefixes = job;
5082
+ for (const [key, value] of Object.entries(this.prefixValues)) {
5083
+ jobWithPrefixes[key] = value;
5084
+ }
5085
+ const putReq = store.put(jobWithPrefixes);
3732
5086
  putReq.onsuccess = () => {};
3733
5087
  putReq.onerror = () => reject(putReq.error);
3734
5088
  };
@@ -3748,10 +5102,15 @@ class IndexedDbQueueStorage {
3748
5102
  const db = await this.getDb();
3749
5103
  const tx = db.transaction(this.tableName, "readonly");
3750
5104
  const store = tx.objectStore(this.tableName);
3751
- const index = store.index("job_run_id");
3752
- const request = index.getAll(job_run_id);
5105
+ const index = store.index("queue_job_run_id");
5106
+ const prefixKeyValues = this.getPrefixKeyValues();
5107
+ const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, job_run_id]);
5108
+ const request = index.getAll(keyRange);
3753
5109
  return new Promise((resolve, reject) => {
3754
- request.onsuccess = () => resolve(request.result);
5110
+ request.onsuccess = () => {
5111
+ const results = (request.result || []).filter((job) => this.matchesPrefixes(job));
5112
+ resolve(results);
5113
+ };
3755
5114
  request.onerror = () => reject(request.error);
3756
5115
  tx.onerror = () => reject(tx.error);
3757
5116
  });
@@ -3760,11 +5119,31 @@ class IndexedDbQueueStorage {
3760
5119
  const db = await this.getDb();
3761
5120
  const tx = db.transaction(this.tableName, "readwrite");
3762
5121
  const store = tx.objectStore(this.tableName);
3763
- const request = store.clear();
5122
+ const index = store.index("queue_status");
5123
+ const prefixKeyValues = this.getPrefixKeyValues();
3764
5124
  return new Promise((resolve, reject) => {
3765
- request.onsuccess = () => resolve();
3766
- request.onerror = () => reject(request.error);
5125
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, "￿"]);
5126
+ const request = index.openCursor(keyRange);
5127
+ request.onsuccess = (event) => {
5128
+ const cursor = event.target.result;
5129
+ if (cursor) {
5130
+ const job = cursor.value;
5131
+ if (job.queue === this.queueName && this.matchesPrefixes(job)) {
5132
+ const deleteRequest = cursor.delete();
5133
+ deleteRequest.onsuccess = () => {
5134
+ cursor.continue();
5135
+ };
5136
+ deleteRequest.onerror = () => {
5137
+ cursor.continue();
5138
+ };
5139
+ } else {
5140
+ cursor.continue();
5141
+ }
5142
+ }
5143
+ };
5144
+ tx.oncomplete = () => resolve();
3767
5145
  tx.onerror = () => reject(tx.error);
5146
+ request.onerror = () => reject(request.error);
3768
5147
  });
3769
5148
  }
3770
5149
  async outputForInput(input) {
@@ -3772,10 +5151,23 @@ class IndexedDbQueueStorage {
3772
5151
  const db = await this.getDb();
3773
5152
  const tx = db.transaction(this.tableName, "readonly");
3774
5153
  const store = tx.objectStore(this.tableName);
3775
- const index = store.index("fingerprint_status");
3776
- const request = index.get([fingerprint, "COMPLETED" /* COMPLETED */]);
5154
+ const index = store.index("queue_fingerprint_status");
5155
+ const prefixKeyValues = this.getPrefixKeyValues();
5156
+ const request = index.get([
5157
+ ...prefixKeyValues,
5158
+ this.queueName,
5159
+ fingerprint,
5160
+ "COMPLETED" /* COMPLETED */
5161
+ ]);
3777
5162
  return new Promise((resolve, reject) => {
3778
- request.onsuccess = () => resolve(request.result?.output ?? null);
5163
+ request.onsuccess = () => {
5164
+ const job = request.result;
5165
+ if (job && this.matchesPrefixes(job)) {
5166
+ resolve(job.output ?? null);
5167
+ } else {
5168
+ resolve(null);
5169
+ }
5170
+ };
3779
5171
  request.onerror = () => reject(request.error);
3780
5172
  tx.onerror = () => reject(tx.error);
3781
5173
  });
@@ -3790,6 +5182,9 @@ class IndexedDbQueueStorage {
3790
5182
  await this.complete(job);
3791
5183
  }
3792
5184
  async delete(id) {
5185
+ const job = await this.get(id);
5186
+ if (!job)
5187
+ return;
3793
5188
  const db = await this.getDb();
3794
5189
  const tx = db.transaction(this.tableName, "readwrite");
3795
5190
  const store = tx.objectStore(this.tableName);
@@ -3804,15 +5199,17 @@ class IndexedDbQueueStorage {
3804
5199
  const db = await this.getDb();
3805
5200
  const tx = db.transaction(this.tableName, "readwrite");
3806
5201
  const store = tx.objectStore(this.tableName);
3807
- const index = store.index("status");
5202
+ const index = store.index("queue_status");
3808
5203
  const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
5204
+ const prefixKeyValues = this.getPrefixKeyValues();
5205
+ const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
3809
5206
  return new Promise((resolve, reject) => {
3810
- const request = index.openCursor();
5207
+ const request = index.openCursor(keyRange);
3811
5208
  request.onsuccess = (event) => {
3812
5209
  const cursor = event.target.result;
3813
5210
  if (cursor) {
3814
5211
  const job = cursor.value;
3815
- if (job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
5212
+ if (job.queue === this.queueName && this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
3816
5213
  cursor.delete();
3817
5214
  }
3818
5215
  cursor.continue();
@@ -3823,6 +5220,363 @@ class IndexedDbQueueStorage {
3823
5220
  request.onerror = () => reject(request.error);
3824
5221
  });
3825
5222
  }
5223
+ async getAllJobs() {
5224
+ const db = await this.getDb();
5225
+ const tx = db.transaction(this.tableName, "readonly");
5226
+ const store = tx.objectStore(this.tableName);
5227
+ const index = store.index("queue_status");
5228
+ const prefixKeyValues = this.getPrefixKeyValues();
5229
+ return new Promise((resolve, reject) => {
5230
+ const jobs = [];
5231
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, "￿"]);
5232
+ const request = index.openCursor(keyRange);
5233
+ request.onsuccess = (event) => {
5234
+ const cursor = event.target.result;
5235
+ if (cursor) {
5236
+ const job = cursor.value;
5237
+ if (job.queue === this.queueName && this.matchesPrefixes(job)) {
5238
+ jobs.push(job);
5239
+ }
5240
+ cursor.continue();
5241
+ }
5242
+ };
5243
+ tx.oncomplete = () => resolve(jobs);
5244
+ tx.onerror = () => reject(tx.error);
5245
+ request.onerror = () => reject(request.error);
5246
+ });
5247
+ }
5248
+ async getAllJobsWithFilter(prefixFilter) {
5249
+ const db = await this.getDb();
5250
+ const tx = db.transaction(this.tableName, "readonly");
5251
+ const store = tx.objectStore(this.tableName);
5252
+ return new Promise((resolve, reject) => {
5253
+ const jobs = [];
5254
+ const request = store.openCursor();
5255
+ request.onsuccess = (event) => {
5256
+ const cursor = event.target.result;
5257
+ if (cursor) {
5258
+ const job = cursor.value;
5259
+ if (job.queue !== this.queueName) {
5260
+ cursor.continue();
5261
+ return;
5262
+ }
5263
+ if (Object.keys(prefixFilter).length === 0) {
5264
+ jobs.push(job);
5265
+ } else {
5266
+ let matches = true;
5267
+ for (const [key, value] of Object.entries(prefixFilter)) {
5268
+ if (job[key] !== value) {
5269
+ matches = false;
5270
+ break;
5271
+ }
5272
+ }
5273
+ if (matches) {
5274
+ jobs.push(job);
5275
+ }
5276
+ }
5277
+ cursor.continue();
5278
+ }
5279
+ };
5280
+ tx.oncomplete = () => resolve(jobs);
5281
+ tx.onerror = () => reject(tx.error);
5282
+ request.onerror = () => reject(request.error);
5283
+ });
5284
+ }
5285
+ isCustomPrefixFilter(prefixFilter) {
5286
+ if (prefixFilter === undefined) {
5287
+ return false;
5288
+ }
5289
+ if (Object.keys(prefixFilter).length === 0) {
5290
+ return true;
5291
+ }
5292
+ const instanceKeys = Object.keys(this.prefixValues);
5293
+ const filterKeys = Object.keys(prefixFilter);
5294
+ if (instanceKeys.length !== filterKeys.length) {
5295
+ return true;
5296
+ }
5297
+ for (const key of instanceKeys) {
5298
+ if (this.prefixValues[key] !== prefixFilter[key]) {
5299
+ return true;
5300
+ }
5301
+ }
5302
+ return false;
5303
+ }
5304
+ getPollingManager() {
5305
+ if (!this.pollingManager) {
5306
+ this.pollingManager = new PollingSubscriptionManager(async () => {
5307
+ const jobs = await this.getAllJobs();
5308
+ return new Map(jobs.map((j) => [j.id, j]));
5309
+ }, (a, b) => JSON.stringify(a) === JSON.stringify(b), {
5310
+ insert: (item) => ({ type: "INSERT", new: item }),
5311
+ update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
5312
+ delete: (item) => ({ type: "DELETE", old: item })
5313
+ });
5314
+ }
5315
+ return this.pollingManager;
5316
+ }
5317
+ subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
5318
+ let lastKnownJobs = new Map;
5319
+ let cancelled = false;
5320
+ const poll = async () => {
5321
+ if (cancelled)
5322
+ return;
5323
+ try {
5324
+ const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
5325
+ if (cancelled)
5326
+ return;
5327
+ const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
5328
+ for (const [id, job] of currentMap) {
5329
+ const old = lastKnownJobs.get(id);
5330
+ if (!old) {
5331
+ callback({ type: "INSERT", new: job });
5332
+ } else if (JSON.stringify(old) !== JSON.stringify(job)) {
5333
+ callback({ type: "UPDATE", old, new: job });
5334
+ }
5335
+ }
5336
+ for (const [id, job] of lastKnownJobs) {
5337
+ if (!currentMap.has(id)) {
5338
+ callback({ type: "DELETE", old: job });
5339
+ }
5340
+ }
5341
+ lastKnownJobs = currentMap;
5342
+ } catch {}
5343
+ };
5344
+ const intervalId = setInterval(poll, intervalMs);
5345
+ poll();
5346
+ return () => {
5347
+ cancelled = true;
5348
+ clearInterval(intervalId);
5349
+ };
5350
+ }
5351
+ subscribeToChanges(callback, options) {
5352
+ const intervalMs = options?.pollingIntervalMs ?? 1000;
5353
+ if (this.isCustomPrefixFilter(options?.prefixFilter)) {
5354
+ return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
5355
+ }
5356
+ const manager = this.getPollingManager();
5357
+ return manager.subscribe(callback, { intervalMs });
5358
+ }
5359
+ }
5360
+ // src/limiter/IndexedDbRateLimiterStorage.ts
5361
+ import { createServiceToken as createServiceToken28 } from "@workglow/util";
5362
+ var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken28("ratelimiter.storage.indexedDb");
5363
+
5364
+ class IndexedDbRateLimiterStorage {
5365
+ executionDb;
5366
+ nextAvailableDb;
5367
+ executionTableName;
5368
+ nextAvailableTableName;
5369
+ migrationOptions;
5370
+ prefixes;
5371
+ prefixValues;
5372
+ constructor(options = {}) {
5373
+ this.migrationOptions = options;
5374
+ this.prefixes = options.prefixes ?? [];
5375
+ this.prefixValues = options.prefixValues ?? {};
5376
+ if (this.prefixes.length > 0) {
5377
+ const prefixNames = this.prefixes.map((p) => p.name).join("_");
5378
+ this.executionTableName = `rate_limit_executions_${prefixNames}`;
5379
+ this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
5380
+ } else {
5381
+ this.executionTableName = "rate_limit_executions";
5382
+ this.nextAvailableTableName = "rate_limit_next_available";
5383
+ }
5384
+ }
5385
+ getPrefixColumnNames() {
5386
+ return this.prefixes.map((p) => p.name);
5387
+ }
5388
+ matchesPrefixes(record) {
5389
+ for (const [key, value] of Object.entries(this.prefixValues)) {
5390
+ if (record[key] !== value) {
5391
+ return false;
5392
+ }
5393
+ }
5394
+ return true;
5395
+ }
5396
+ getPrefixKeyValues() {
5397
+ return this.prefixes.map((p) => this.prefixValues[p.name]);
5398
+ }
5399
+ async getExecutionDb() {
5400
+ if (this.executionDb)
5401
+ return this.executionDb;
5402
+ await this.setupDatabase();
5403
+ return this.executionDb;
5404
+ }
5405
+ async getNextAvailableDb() {
5406
+ if (this.nextAvailableDb)
5407
+ return this.nextAvailableDb;
5408
+ await this.setupDatabase();
5409
+ return this.nextAvailableDb;
5410
+ }
5411
+ async setupDatabase() {
5412
+ const prefixColumnNames = this.getPrefixColumnNames();
5413
+ const buildKeyPath = (basePath) => {
5414
+ return [...prefixColumnNames, ...basePath];
5415
+ };
5416
+ const executionIndexes = [
5417
+ {
5418
+ name: "queue_executed_at",
5419
+ keyPath: buildKeyPath(["queue_name", "executed_at"]),
5420
+ options: { unique: false }
5421
+ }
5422
+ ];
5423
+ this.executionDb = await ensureIndexedDbTable(this.executionTableName, "id", executionIndexes, this.migrationOptions);
5424
+ const nextAvailableIndexes = [
5425
+ {
5426
+ name: "queue_name",
5427
+ keyPath: buildKeyPath(["queue_name"]),
5428
+ options: { unique: true }
5429
+ }
5430
+ ];
5431
+ this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
5432
+ }
5433
+ async recordExecution(queueName) {
5434
+ const db = await this.getExecutionDb();
5435
+ const tx = db.transaction(this.executionTableName, "readwrite");
5436
+ const store = tx.objectStore(this.executionTableName);
5437
+ const record = {
5438
+ id: crypto.randomUUID(),
5439
+ queue_name: queueName,
5440
+ executed_at: new Date().toISOString()
5441
+ };
5442
+ for (const [key, value] of Object.entries(this.prefixValues)) {
5443
+ record[key] = value;
5444
+ }
5445
+ return new Promise((resolve, reject) => {
5446
+ const request = store.add(record);
5447
+ tx.oncomplete = () => resolve();
5448
+ tx.onerror = () => reject(tx.error);
5449
+ request.onerror = () => reject(request.error);
5450
+ });
5451
+ }
5452
+ async getExecutionCount(queueName, windowStartTime) {
5453
+ const db = await this.getExecutionDb();
5454
+ const tx = db.transaction(this.executionTableName, "readonly");
5455
+ const store = tx.objectStore(this.executionTableName);
5456
+ const index = store.index("queue_executed_at");
5457
+ const prefixKeyValues = this.getPrefixKeyValues();
5458
+ return new Promise((resolve, reject) => {
5459
+ let count = 0;
5460
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartTime], [...prefixKeyValues, queueName, "￿"], true, false);
5461
+ const request = index.openCursor(keyRange);
5462
+ request.onsuccess = (event) => {
5463
+ const cursor = event.target.result;
5464
+ if (cursor) {
5465
+ const record = cursor.value;
5466
+ if (this.matchesPrefixes(record)) {
5467
+ count++;
5468
+ }
5469
+ cursor.continue();
5470
+ }
5471
+ };
5472
+ tx.oncomplete = () => resolve(count);
5473
+ tx.onerror = () => reject(tx.error);
5474
+ request.onerror = () => reject(request.error);
5475
+ });
5476
+ }
5477
+ async getOldestExecutionAtOffset(queueName, offset) {
5478
+ const db = await this.getExecutionDb();
5479
+ const tx = db.transaction(this.executionTableName, "readonly");
5480
+ const store = tx.objectStore(this.executionTableName);
5481
+ const index = store.index("queue_executed_at");
5482
+ const prefixKeyValues = this.getPrefixKeyValues();
5483
+ return new Promise((resolve, reject) => {
5484
+ const executions = [];
5485
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, "￿"]);
5486
+ const request = index.openCursor(keyRange);
5487
+ request.onsuccess = (event) => {
5488
+ const cursor = event.target.result;
5489
+ if (cursor) {
5490
+ const record = cursor.value;
5491
+ if (this.matchesPrefixes(record)) {
5492
+ executions.push(record.executed_at);
5493
+ }
5494
+ cursor.continue();
5495
+ }
5496
+ };
5497
+ tx.oncomplete = () => {
5498
+ executions.sort();
5499
+ resolve(executions[offset]);
5500
+ };
5501
+ tx.onerror = () => reject(tx.error);
5502
+ request.onerror = () => reject(request.error);
5503
+ });
5504
+ }
5505
+ async getNextAvailableTime(queueName) {
5506
+ const db = await this.getNextAvailableDb();
5507
+ const tx = db.transaction(this.nextAvailableTableName, "readonly");
5508
+ const store = tx.objectStore(this.nextAvailableTableName);
5509
+ const prefixKeyValues = this.getPrefixKeyValues();
5510
+ const key = [...prefixKeyValues, queueName].join("_");
5511
+ return new Promise((resolve, reject) => {
5512
+ const request = store.get(key);
5513
+ request.onsuccess = () => {
5514
+ const record = request.result;
5515
+ if (record && this.matchesPrefixes(record)) {
5516
+ resolve(record.next_available_at);
5517
+ } else {
5518
+ resolve(undefined);
5519
+ }
5520
+ };
5521
+ request.onerror = () => reject(request.error);
5522
+ tx.onerror = () => reject(tx.error);
5523
+ });
5524
+ }
5525
+ async setNextAvailableTime(queueName, nextAvailableAt) {
5526
+ const db = await this.getNextAvailableDb();
5527
+ const tx = db.transaction(this.nextAvailableTableName, "readwrite");
5528
+ const store = tx.objectStore(this.nextAvailableTableName);
5529
+ const prefixKeyValues = this.getPrefixKeyValues();
5530
+ const key = [...prefixKeyValues, queueName].join("_");
5531
+ const record = {
5532
+ queue_name: queueName,
5533
+ next_available_at: nextAvailableAt
5534
+ };
5535
+ for (const [k, value] of Object.entries(this.prefixValues)) {
5536
+ record[k] = value;
5537
+ }
5538
+ record[this.getPrefixColumnNames().concat(["queue_name"]).join("_")] = key;
5539
+ return new Promise((resolve, reject) => {
5540
+ const request = store.put(record);
5541
+ tx.oncomplete = () => resolve();
5542
+ tx.onerror = () => reject(tx.error);
5543
+ request.onerror = () => reject(request.error);
5544
+ });
5545
+ }
5546
+ async clear(queueName) {
5547
+ const execDb = await this.getExecutionDb();
5548
+ const execTx = execDb.transaction(this.executionTableName, "readwrite");
5549
+ const execStore = execTx.objectStore(this.executionTableName);
5550
+ const execIndex = execStore.index("queue_executed_at");
5551
+ const prefixKeyValues = this.getPrefixKeyValues();
5552
+ await new Promise((resolve, reject) => {
5553
+ const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, "￿"]);
5554
+ const request = execIndex.openCursor(keyRange);
5555
+ request.onsuccess = (event) => {
5556
+ const cursor = event.target.result;
5557
+ if (cursor) {
5558
+ const record = cursor.value;
5559
+ if (this.matchesPrefixes(record)) {
5560
+ cursor.delete();
5561
+ }
5562
+ cursor.continue();
5563
+ }
5564
+ };
5565
+ execTx.oncomplete = () => resolve();
5566
+ execTx.onerror = () => reject(execTx.error);
5567
+ request.onerror = () => reject(request.error);
5568
+ });
5569
+ const nextDb = await this.getNextAvailableDb();
5570
+ const nextTx = nextDb.transaction(this.nextAvailableTableName, "readwrite");
5571
+ const nextStore = nextTx.objectStore(this.nextAvailableTableName);
5572
+ const key = [...prefixKeyValues, queueName].join("_");
5573
+ await new Promise((resolve, reject) => {
5574
+ const request = nextStore.delete(key);
5575
+ nextTx.oncomplete = () => resolve();
5576
+ nextTx.onerror = () => reject(nextTx.error);
5577
+ request.onerror = () => reject(request.error);
5578
+ });
5579
+ }
3826
5580
  }
3827
5581
  export {
3828
5582
  ensureIndexedDbTable,
@@ -3830,22 +5584,29 @@ export {
3830
5584
  TabularRepository,
3831
5585
  TABULAR_REPOSITORY,
3832
5586
  SupabaseTabularRepository,
5587
+ SupabaseRateLimiterStorage,
3833
5588
  SupabaseQueueStorage,
3834
5589
  SupabaseKvRepository,
3835
5590
  SqliteTabularRepository,
5591
+ SqliteRateLimiterStorage,
3836
5592
  SqliteQueueStorage,
3837
5593
  SqliteKvRepository,
3838
5594
  SUPABASE_TABULAR_REPOSITORY,
5595
+ SUPABASE_RATE_LIMITER_STORAGE,
3839
5596
  SUPABASE_QUEUE_STORAGE,
3840
5597
  SUPABASE_KV_REPOSITORY,
3841
5598
  SQLITE_TABULAR_REPOSITORY,
5599
+ SQLITE_RATE_LIMITER_STORAGE,
3842
5600
  SQLITE_QUEUE_STORAGE,
3843
5601
  SQLITE_KV_REPOSITORY,
5602
+ RATE_LIMITER_STORAGE,
3844
5603
  QUEUE_STORAGE,
3845
5604
  PostgresTabularRepository,
5605
+ PostgresRateLimiterStorage,
3846
5606
  PostgresQueueStorage,
3847
5607
  PostgresKvRepository,
3848
5608
  POSTGRES_TABULAR_REPOSITORY,
5609
+ POSTGRES_RATE_LIMITER_STORAGE,
3849
5610
  POSTGRES_QUEUE_STORAGE,
3850
5611
  POSTGRES_KV_REPOSITORY,
3851
5612
  MEMORY_TABULAR_REPOSITORY,
@@ -3855,12 +5616,16 @@ export {
3855
5616
  KV_REPOSITORY,
3856
5617
  JobStatus,
3857
5618
  IndexedDbTabularRepository,
5619
+ IndexedDbRateLimiterStorage,
3858
5620
  IndexedDbQueueStorage,
3859
5621
  IndexedDbKvRepository,
3860
5622
  InMemoryTabularRepository,
5623
+ InMemoryRateLimiterStorage,
3861
5624
  InMemoryQueueStorage,
3862
5625
  InMemoryKvRepository,
5626
+ IN_MEMORY_RATE_LIMITER_STORAGE,
3863
5627
  IN_MEMORY_QUEUE_STORAGE,
5628
+ INDEXED_DB_RATE_LIMITER_STORAGE,
3864
5629
  INDEXED_DB_QUEUE_STORAGE,
3865
5630
  IDB_TABULAR_REPOSITORY,
3866
5631
  IDB_KV_REPOSITORY,
@@ -3876,4 +5641,4 @@ export {
3876
5641
  CACHED_TABULAR_REPOSITORY
3877
5642
  };
3878
5643
 
3879
- //# debugId=C9E5F9D56E81505C64756E2164756E21
5644
+ //# debugId=A8A981E6F7A25E7F64756E2164756E21