@workglow/storage 0.2.27 → 0.2.29

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/dist/browser.d.ts +0 -10
  2. package/dist/browser.d.ts.map +1 -1
  3. package/dist/browser.js +406 -4207
  4. package/dist/browser.js.map +8 -24
  5. package/dist/bun.js +341 -7235
  6. package/dist/bun.js.map +8 -35
  7. package/dist/common-server.d.ts +0 -21
  8. package/dist/common-server.d.ts.map +1 -1
  9. package/dist/common.d.ts +1 -6
  10. package/dist/common.d.ts.map +1 -1
  11. package/dist/node.js +341 -7236
  12. package/dist/node.js.map +8 -35
  13. package/package.json +5 -71
  14. package/dist/kv/IndexedDbKvStorage.d.ts +0 -27
  15. package/dist/kv/IndexedDbKvStorage.d.ts.map +0 -1
  16. package/dist/kv/PostgresKvStorage.d.ts +0 -28
  17. package/dist/kv/PostgresKvStorage.d.ts.map +0 -1
  18. package/dist/kv/SqliteKvStorage.d.ts +0 -28
  19. package/dist/kv/SqliteKvStorage.d.ts.map +0 -1
  20. package/dist/kv/SupabaseKvStorage.d.ts +0 -33
  21. package/dist/kv/SupabaseKvStorage.d.ts.map +0 -1
  22. package/dist/postgres/browser.d.ts +0 -32
  23. package/dist/postgres/browser.d.ts.map +0 -1
  24. package/dist/postgres/browser.js +0 -150
  25. package/dist/postgres/browser.js.map +0 -11
  26. package/dist/postgres/node-bun.d.ts +0 -26
  27. package/dist/postgres/node-bun.d.ts.map +0 -1
  28. package/dist/postgres/node-bun.js +0 -41
  29. package/dist/postgres/node-bun.js.map +0 -10
  30. package/dist/postgres/pglite-pool.d.ts +0 -21
  31. package/dist/postgres/pglite-pool.d.ts.map +0 -1
  32. package/dist/queue/IQueueStorage.d.ts +0 -229
  33. package/dist/queue/IQueueStorage.d.ts.map +0 -1
  34. package/dist/queue/InMemoryQueueStorage.d.ts +0 -149
  35. package/dist/queue/InMemoryQueueStorage.d.ts.map +0 -1
  36. package/dist/queue/IndexedDbQueueStorage.d.ts +0 -166
  37. package/dist/queue/IndexedDbQueueStorage.d.ts.map +0 -1
  38. package/dist/queue/PostgresQueueStorage.d.ts +0 -154
  39. package/dist/queue/PostgresQueueStorage.d.ts.map +0 -1
  40. package/dist/queue/SqliteQueueStorage.d.ts +0 -149
  41. package/dist/queue/SqliteQueueStorage.d.ts.map +0 -1
  42. package/dist/queue/SupabaseQueueStorage.d.ts +0 -195
  43. package/dist/queue/SupabaseQueueStorage.d.ts.map +0 -1
  44. package/dist/queue/TelemetryQueueStorage.d.ts +0 -33
  45. package/dist/queue/TelemetryQueueStorage.d.ts.map +0 -1
  46. package/dist/queue-limiter/IRateLimiterStorage.d.ts +0 -127
  47. package/dist/queue-limiter/IRateLimiterStorage.d.ts.map +0 -1
  48. package/dist/queue-limiter/InMemoryRateLimiterStorage.d.ts +0 -43
  49. package/dist/queue-limiter/InMemoryRateLimiterStorage.d.ts.map +0 -1
  50. package/dist/queue-limiter/IndexedDbRateLimiterStorage.d.ts +0 -79
  51. package/dist/queue-limiter/IndexedDbRateLimiterStorage.d.ts.map +0 -1
  52. package/dist/queue-limiter/PostgresRateLimiterStorage.d.ts +0 -57
  53. package/dist/queue-limiter/PostgresRateLimiterStorage.d.ts.map +0 -1
  54. package/dist/queue-limiter/SqliteRateLimiterStorage.d.ts +0 -62
  55. package/dist/queue-limiter/SqliteRateLimiterStorage.d.ts.map +0 -1
  56. package/dist/queue-limiter/SupabaseRateLimiterStorage.d.ts +0 -54
  57. package/dist/queue-limiter/SupabaseRateLimiterStorage.d.ts.map +0 -1
  58. package/dist/sqlite/browser.d.ts +0 -37
  59. package/dist/sqlite/browser.d.ts.map +0 -1
  60. package/dist/sqlite/browser.js +0 -125
  61. package/dist/sqlite/browser.js.map +0 -10
  62. package/dist/sqlite/bun.d.ts +0 -32
  63. package/dist/sqlite/bun.d.ts.map +0 -1
  64. package/dist/sqlite/bun.js +0 -84
  65. package/dist/sqlite/bun.js.map +0 -10
  66. package/dist/sqlite/canonical-api.d.ts +0 -34
  67. package/dist/sqlite/canonical-api.d.ts.map +0 -1
  68. package/dist/sqlite/node.d.ts +0 -34
  69. package/dist/sqlite/node.d.ts.map +0 -1
  70. package/dist/sqlite/node.js +0 -65
  71. package/dist/sqlite/node.js.map +0 -10
  72. package/dist/tabular/IndexedDbTabularStorage.d.ts +0 -199
  73. package/dist/tabular/IndexedDbTabularStorage.d.ts.map +0 -1
  74. package/dist/tabular/PostgresTabularStorage.d.ts +0 -196
  75. package/dist/tabular/PostgresTabularStorage.d.ts.map +0 -1
  76. package/dist/tabular/SqliteTabularStorage.d.ts +0 -167
  77. package/dist/tabular/SqliteTabularStorage.d.ts.map +0 -1
  78. package/dist/tabular/SupabaseTabularStorage.d.ts +0 -174
  79. package/dist/tabular/SupabaseTabularStorage.d.ts.map +0 -1
  80. package/dist/util/IndexedDbTable.d.ts +0 -40
  81. package/dist/util/IndexedDbTable.d.ts.map +0 -1
  82. package/dist/util/traced.d.ts +0 -10
  83. package/dist/util/traced.d.ts.map +0 -1
  84. package/dist/vector/IndexedDbVectorStorage.d.ts +0 -53
  85. package/dist/vector/IndexedDbVectorStorage.d.ts.map +0 -1
  86. package/dist/vector/PostgresVectorStorage.d.ts +0 -39
  87. package/dist/vector/PostgresVectorStorage.d.ts.map +0 -1
  88. package/dist/vector/SqliteAiVectorStorage.d.ts +0 -100
  89. package/dist/vector/SqliteAiVectorStorage.d.ts.map +0 -1
  90. package/dist/vector/SqliteVectorStorage.d.ts +0 -49
  91. package/dist/vector/SqliteVectorStorage.d.ts.map +0 -1
  92. package/src/queue/README.md +0 -41
package/dist/browser.js CHANGED
@@ -414,6 +414,205 @@ class BaseTabularStorage {
414
414
  this.destroy();
415
415
  }
416
416
  }
417
+ // src/tabular/BaseSqlTabularStorage.ts
418
+ class BaseSqlTabularStorage extends BaseTabularStorage {
419
+ table;
420
+ _pkColsCache = new Map;
421
+ _valColsCache = new Map;
422
+ _pkColListCache = new Map;
423
+ _valColListCache = new Map;
424
+ constructor(table = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
425
+ super(schema, primaryKeyNames, indexes, clientProvidedKeys);
426
+ this.table = table;
427
+ this.validateTableAndSchema();
428
+ }
429
+ constructPrimaryKeyColumns($delimiter = "") {
430
+ let cached = this._pkColsCache.get($delimiter);
431
+ if (cached === undefined) {
432
+ cached = Object.entries(this.primaryKeySchema.properties).map(([key, typeDef]) => {
433
+ const sqlType = this.mapTypeToSQL(typeDef);
434
+ return `${$delimiter}${key}${$delimiter} ${sqlType} NOT NULL`;
435
+ }).join(", ");
436
+ this._pkColsCache.set($delimiter, cached);
437
+ }
438
+ return cached;
439
+ }
440
+ constructValueColumns($delimiter = "") {
441
+ let cached = this._valColsCache.get($delimiter);
442
+ if (cached === undefined) {
443
+ const requiredSet = new Set(this.valueSchema.required ?? []);
444
+ const cols = Object.entries(this.valueSchema.properties).map(([key, typeDef]) => {
445
+ const sqlType = this.mapTypeToSQL(typeDef);
446
+ const isRequired = requiredSet.has(key);
447
+ const nullable = !isRequired || this.isNullable(typeDef);
448
+ return `${$delimiter}${key}${$delimiter} ${sqlType}${nullable ? " NULL" : " NOT NULL"}`;
449
+ }).join(", ");
450
+ cached = cols.length > 0 ? `, ${cols}` : "";
451
+ this._valColsCache.set($delimiter, cached);
452
+ }
453
+ return cached;
454
+ }
455
+ isNullable(typeDef) {
456
+ if (typeof typeDef === "boolean")
457
+ return typeDef;
458
+ if (typeDef.type === "null") {
459
+ return true;
460
+ }
461
+ if (Array.isArray(typeDef.type)) {
462
+ return typeDef.type.includes("null");
463
+ }
464
+ if (typeDef.anyOf && Array.isArray(typeDef.anyOf)) {
465
+ return typeDef.anyOf.some((type) => type.type === "null");
466
+ }
467
+ if (typeDef.oneOf && Array.isArray(typeDef.oneOf)) {
468
+ return typeDef.oneOf.some((type) => type.type === "null");
469
+ }
470
+ return false;
471
+ }
472
+ primaryKeyColumnList($delimiter = "") {
473
+ let cached = this._pkColListCache.get($delimiter);
474
+ if (cached === undefined) {
475
+ cached = $delimiter + this.primaryKeyColumns().join(`${$delimiter}, ${$delimiter}`) + $delimiter;
476
+ this._pkColListCache.set($delimiter, cached);
477
+ }
478
+ return cached;
479
+ }
480
+ valueColumnList($delimiter = "") {
481
+ let cached = this._valColListCache.get($delimiter);
482
+ if (cached === undefined) {
483
+ cached = $delimiter + this.valueColumns().join(`${$delimiter}, ${$delimiter}`) + $delimiter;
484
+ this._valColListCache.set($delimiter, cached);
485
+ }
486
+ return cached;
487
+ }
488
+ getNonNullType(typeDef) {
489
+ if (typeof typeDef === "boolean")
490
+ return typeDef;
491
+ if (typeDef.anyOf && Array.isArray(typeDef.anyOf)) {
492
+ const nonNullType = typeDef.anyOf.find((t) => t.type !== "null");
493
+ if (nonNullType) {
494
+ return nonNullType;
495
+ }
496
+ }
497
+ if (typeDef.oneOf && Array.isArray(typeDef.oneOf)) {
498
+ const nonNullType = typeDef.oneOf.find((t) => t.type !== "null");
499
+ if (nonNullType) {
500
+ return nonNullType;
501
+ }
502
+ }
503
+ return typeDef;
504
+ }
505
+ getValueAsOrderedArray(value) {
506
+ const orderedParams = [];
507
+ const valueAsRecord = value;
508
+ const requiredSet = new Set(this.valueSchema.required ?? []);
509
+ for (const key in this.valueSchema.properties) {
510
+ if (Object.prototype.hasOwnProperty.call(valueAsRecord, key)) {
511
+ const val = valueAsRecord[key];
512
+ if (val === undefined && !requiredSet.has(key)) {
513
+ orderedParams.push(null);
514
+ } else {
515
+ orderedParams.push(this.jsToSqlValue(key, val));
516
+ }
517
+ } else {
518
+ if (requiredSet.has(key)) {
519
+ throw new Error(`Missing required value field: ${key}`);
520
+ }
521
+ orderedParams.push(null);
522
+ }
523
+ }
524
+ return orderedParams;
525
+ }
526
+ getPrimaryKeyAsOrderedArray(key) {
527
+ const orderedParams = [];
528
+ const keyObj = key;
529
+ for (const k of Object.keys(this.primaryKeySchema.properties)) {
530
+ if (k in keyObj) {
531
+ const value = keyObj[k];
532
+ if (value === null) {
533
+ throw new Error(`Primary key field ${k} cannot be null`);
534
+ }
535
+ orderedParams.push(this.jsToSqlValue(k, value));
536
+ } else {
537
+ throw new Error(`Missing required primary key field: ${k}`);
538
+ }
539
+ }
540
+ return orderedParams;
541
+ }
542
+ jsToSqlValue(column, value) {
543
+ const typeDef = this.schema.properties[column];
544
+ if (!typeDef) {
545
+ return value;
546
+ }
547
+ if (value === null && this.isNullable(typeDef)) {
548
+ return null;
549
+ }
550
+ const actualType = this.getNonNullType(typeDef);
551
+ if (typeof actualType === "boolean") {
552
+ return value;
553
+ }
554
+ if (actualType.contentEncoding === "blob") {
555
+ if (value instanceof Uint8Array) {
556
+ return value;
557
+ }
558
+ if (typeof Buffer !== "undefined" && value instanceof Buffer) {
559
+ return new Uint8Array(value);
560
+ }
561
+ if (Array.isArray(value)) {
562
+ return new Uint8Array(value);
563
+ }
564
+ return value;
565
+ } else if (value instanceof Date) {
566
+ return value.toISOString();
567
+ } else {
568
+ return value;
569
+ }
570
+ }
571
+ sqlToJsValue(column, value) {
572
+ const typeDef = this.schema.properties[column];
573
+ if (!typeDef) {
574
+ return value;
575
+ }
576
+ if (value === null && this.isNullable(typeDef)) {
577
+ return null;
578
+ }
579
+ const actualType = this.getNonNullType(typeDef);
580
+ if (typeof actualType === "boolean") {
581
+ return value;
582
+ }
583
+ if (actualType.contentEncoding === "blob") {
584
+ if (typeof Buffer !== "undefined" && value instanceof Buffer) {
585
+ return new Uint8Array(value);
586
+ }
587
+ if (value instanceof Uint8Array) {
588
+ return value;
589
+ }
590
+ return value;
591
+ } else {
592
+ return value;
593
+ }
594
+ }
595
+ validateTableAndSchema() {
596
+ if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(this.table)) {
597
+ throw new Error("Table name must start with a letter and contain only letters, digits, and underscores, got: " + this.table);
598
+ }
599
+ const validateSchemaKeys = (schema) => {
600
+ for (const key in schema.properties) {
601
+ if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(key)) {
602
+ throw new Error("Schema keys must start with a letter and contain only letters, digits, and underscores, got: " + key);
603
+ }
604
+ }
605
+ };
606
+ validateSchemaKeys(this.primaryKeySchema);
607
+ validateSchemaKeys(this.valueSchema);
608
+ const primaryKeys = new Set(Object.keys(this.primaryKeySchema.properties));
609
+ const valueKeys = Object.keys(this.valueSchema.properties);
610
+ const duplicates = valueKeys.filter((key) => primaryKeys.has(key));
611
+ if (duplicates.length > 0) {
612
+ throw new Error(`Duplicate keys found in schemas: ${duplicates.join(", ")}`);
613
+ }
614
+ }
615
+ }
417
616
  // src/tabular/CachedTabularStorage.ts
418
617
  import { createServiceToken as createServiceToken3, getLogger } from "@workglow/util";
419
618
 
@@ -1272,28 +1471,9 @@ registerInputCompactor("storage:tabular", (value, _format, registry) => {
1272
1471
  }
1273
1472
  return;
1274
1473
  });
1275
- // src/util/traced.ts
1276
- import { getTelemetryProvider, SpanStatusCode } from "@workglow/util";
1277
- async function traced(spanName, storageName, fn) {
1278
- const telemetry = getTelemetryProvider();
1279
- if (!telemetry.isEnabled)
1280
- return fn();
1281
- const span = telemetry.startSpan(spanName, {
1282
- attributes: { "workglow.storage.name": storageName }
1283
- });
1284
- try {
1285
- const result = await fn();
1286
- span.setStatus(SpanStatusCode.OK);
1287
- return result;
1288
- } catch (err) {
1289
- span.setStatus(SpanStatusCode.ERROR, err instanceof Error ? err.message : String(err));
1290
- throw err;
1291
- } finally {
1292
- span.end();
1293
- }
1294
- }
1295
-
1296
1474
  // src/tabular/TelemetryTabularStorage.ts
1475
+ import { traced } from "@workglow/util";
1476
+
1297
1477
  class TelemetryTabularStorage {
1298
1478
  storageName;
1299
1479
  inner;
@@ -1500,6 +1680,8 @@ class InMemoryKvStorage extends KvViaTabularStorage {
1500
1680
  }
1501
1681
  }
1502
1682
  // src/kv/TelemetryKvStorage.ts
1683
+ import { traced as traced2 } from "@workglow/util";
1684
+
1503
1685
  class TelemetryKvStorage {
1504
1686
  storageName;
1505
1687
  inner;
@@ -1508,25 +1690,25 @@ class TelemetryKvStorage {
1508
1690
  this.inner = inner;
1509
1691
  }
1510
1692
  put(key, value) {
1511
- return traced("workglow.storage.kv.put", this.storageName, () => this.inner.put(key, value));
1693
+ return traced2("workglow.storage.kv.put", this.storageName, () => this.inner.put(key, value));
1512
1694
  }
1513
1695
  putBulk(items) {
1514
- return traced("workglow.storage.kv.putBulk", this.storageName, () => this.inner.putBulk(items));
1696
+ return traced2("workglow.storage.kv.putBulk", this.storageName, () => this.inner.putBulk(items));
1515
1697
  }
1516
1698
  get(key) {
1517
- return traced("workglow.storage.kv.get", this.storageName, () => this.inner.get(key));
1699
+ return traced2("workglow.storage.kv.get", this.storageName, () => this.inner.get(key));
1518
1700
  }
1519
1701
  delete(key) {
1520
- return traced("workglow.storage.kv.delete", this.storageName, () => this.inner.delete(key));
1702
+ return traced2("workglow.storage.kv.delete", this.storageName, () => this.inner.delete(key));
1521
1703
  }
1522
1704
  getAll() {
1523
- return traced("workglow.storage.kv.getAll", this.storageName, () => this.inner.getAll());
1705
+ return traced2("workglow.storage.kv.getAll", this.storageName, () => this.inner.getAll());
1524
1706
  }
1525
1707
  deleteAll() {
1526
- return traced("workglow.storage.kv.deleteAll", this.storageName, () => this.inner.deleteAll());
1708
+ return traced2("workglow.storage.kv.deleteAll", this.storageName, () => this.inner.deleteAll());
1527
1709
  }
1528
1710
  size() {
1529
- return traced("workglow.storage.kv.size", this.storageName, () => this.inner.size());
1711
+ return traced2("workglow.storage.kv.size", this.storageName, () => this.inner.size());
1530
1712
  }
1531
1713
  getObjectAsIdString(object) {
1532
1714
  return this.inner.getObjectAsIdString(object);
@@ -1547,456 +1729,47 @@ class TelemetryKvStorage {
1547
1729
  return this.inner.waitOn(name);
1548
1730
  }
1549
1731
  }
1550
- // src/queue/InMemoryQueueStorage.ts
1551
- import {
1552
- createServiceToken as createServiceToken9,
1553
- EventEmitter as EventEmitter3,
1554
- getLogger as getLogger2,
1555
- makeFingerprint as makeFingerprint4,
1556
- sleep,
1557
- uuid4 as uuid42
1558
- } from "@workglow/util";
1559
-
1560
- // src/queue/IQueueStorage.ts
1561
- import { createServiceToken as createServiceToken8 } from "@workglow/util";
1562
- var QUEUE_STORAGE = createServiceToken8("jobqueue.storage");
1563
- var JobStatus = {
1564
- PENDING: "PENDING",
1565
- PROCESSING: "PROCESSING",
1566
- COMPLETED: "COMPLETED",
1567
- ABORTING: "ABORTING",
1568
- FAILED: "FAILED",
1569
- DISABLED: "DISABLED"
1570
- };
1571
-
1572
- // src/queue/InMemoryQueueStorage.ts
1573
- var IN_MEMORY_QUEUE_STORAGE = createServiceToken9("jobqueue.storage.inMemory");
1574
-
1575
- class InMemoryQueueStorage {
1576
- queueName;
1577
- scope = "process";
1578
- prefixValues;
1579
- events = new EventEmitter3;
1580
- constructor(queueName, options) {
1581
- this.queueName = queueName;
1582
- this.jobQueue = [];
1583
- this.prefixValues = options?.prefixValues ?? {};
1584
- }
1585
- jobQueue;
1586
- matchesPrefixes(job) {
1587
- for (const [key, value] of Object.entries(this.prefixValues)) {
1588
- if (job[key] !== value) {
1589
- return false;
1590
- }
1732
+ // src/util/HybridSubscriptionManager.ts
1733
+ class HybridSubscriptionManager {
1734
+ subscribers = new Set;
1735
+ lastKnownState = new Map;
1736
+ initialized = false;
1737
+ channel = null;
1738
+ backupPollingIntervalId = null;
1739
+ fetchState;
1740
+ compareItems;
1741
+ payloadFactory;
1742
+ options;
1743
+ hasBroadcastChannel;
1744
+ constructor(channelName, fetchState, compareItems, payloadFactory, options) {
1745
+ this.fetchState = fetchState;
1746
+ this.compareItems = compareItems;
1747
+ this.payloadFactory = payloadFactory;
1748
+ this.options = {
1749
+ defaultIntervalMs: options?.defaultIntervalMs ?? 1000,
1750
+ backupPollingIntervalMs: options?.backupPollingIntervalMs ?? 5000,
1751
+ useBroadcastChannel: options?.useBroadcastChannel ?? true,
1752
+ broadcastChannelName: options?.broadcastChannelName ?? channelName
1753
+ };
1754
+ this.hasBroadcastChannel = this.options.useBroadcastChannel && typeof BroadcastChannel !== "undefined";
1755
+ if (this.hasBroadcastChannel) {
1756
+ this.initializeBroadcastChannel();
1591
1757
  }
1592
- return true;
1593
1758
  }
1594
- pendingQueue() {
1595
- const now = new Date().toISOString();
1596
- return this.jobQueue.filter((job) => this.matchesPrefixes(job)).filter((job) => job.status === JobStatus.PENDING).filter((job) => !job.run_after || job.run_after <= now).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || ""));
1597
- }
1598
- async add(job) {
1599
- await sleep(0);
1600
- const now = new Date().toISOString();
1601
- const jobWithPrefixes = job;
1602
- jobWithPrefixes.id = jobWithPrefixes.id ?? uuid42();
1603
- jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid42();
1604
- jobWithPrefixes.queue = this.queueName;
1605
- jobWithPrefixes.fingerprint = await makeFingerprint4(jobWithPrefixes.input);
1606
- jobWithPrefixes.status = JobStatus.PENDING;
1607
- jobWithPrefixes.progress = 0;
1608
- jobWithPrefixes.progress_message = "";
1609
- jobWithPrefixes.progress_details = null;
1610
- jobWithPrefixes.created_at = now;
1611
- jobWithPrefixes.run_after = now;
1612
- for (const [key, value] of Object.entries(this.prefixValues)) {
1613
- jobWithPrefixes[key] = value;
1614
- }
1615
- this.jobQueue.push(jobWithPrefixes);
1616
- this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
1617
- return jobWithPrefixes.id;
1618
- }
1619
- async get(id) {
1620
- await sleep(0);
1621
- const job = this.jobQueue.find((j) => j.id === id);
1622
- if (job && this.matchesPrefixes(job)) {
1623
- return job;
1759
+ initializeBroadcastChannel() {
1760
+ try {
1761
+ this.channel = new BroadcastChannel(this.options.broadcastChannelName);
1762
+ this.channel.onmessage = (event) => {
1763
+ this.handleBroadcastMessage(event.data);
1764
+ };
1765
+ } catch (error) {
1766
+ console.error("Failed to initialize BroadcastChannel:", error);
1767
+ this.channel = null;
1624
1768
  }
1625
- return;
1626
1769
  }
1627
- async peek(status = JobStatus.PENDING, num = 100) {
1628
- await sleep(0);
1629
- num = Number(num) || 100;
1630
- return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
1631
- }
1632
- async next(workerId) {
1633
- await sleep(0);
1634
- const top = this.pendingQueue();
1635
- const job = top[0];
1636
- if (job) {
1637
- const oldJob = { ...job };
1638
- job.status = JobStatus.PROCESSING;
1639
- job.last_ran_at = new Date().toISOString();
1640
- job.worker_id = workerId;
1641
- this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1642
- return job;
1643
- }
1644
- }
1645
- async size(status = JobStatus.PENDING) {
1646
- await sleep(0);
1647
- return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
1648
- }
1649
- async saveProgress(id, progress, message, details) {
1650
- await sleep(0);
1651
- const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
1652
- if (!job) {
1653
- const jobWithAnyPrefix = this.jobQueue.find((j) => j.id === id);
1654
- getLogger2().warn("Job not found for progress update", {
1655
- id,
1656
- reason: jobWithAnyPrefix ? "prefix_mismatch" : "missing",
1657
- existingStatus: jobWithAnyPrefix?.status,
1658
- queueName: this.queueName,
1659
- prefixValues: this.prefixValues
1660
- });
1661
- return;
1662
- }
1663
- if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED) {
1664
- getLogger2().warn("Job already completed or failed for progress update", {
1665
- id,
1666
- status: job.status,
1667
- completedAt: job.completed_at,
1668
- error: job.error
1669
- });
1670
- return;
1671
- }
1672
- const oldJob = { ...job };
1673
- job.progress = progress;
1674
- job.progress_message = message;
1675
- job.progress_details = details;
1676
- this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1677
- }
1678
- async complete(job) {
1679
- await sleep(0);
1680
- const jobWithPrefixes = job;
1681
- const index = this.jobQueue.findIndex((j) => j.id === job.id && this.matchesPrefixes(j));
1682
- if (index !== -1) {
1683
- const existing = this.jobQueue[index];
1684
- const currentAttempts = existing?.run_attempts ?? 0;
1685
- jobWithPrefixes.run_attempts = currentAttempts + 1;
1686
- for (const [key, value] of Object.entries(this.prefixValues)) {
1687
- jobWithPrefixes[key] = value;
1688
- }
1689
- this.jobQueue[index] = jobWithPrefixes;
1690
- this.events.emit("change", { type: "UPDATE", old: existing, new: jobWithPrefixes });
1691
- }
1692
- }
1693
- async release(id) {
1694
- await sleep(0);
1695
- const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
1696
- if (job) {
1697
- const oldJob = { ...job };
1698
- job.status = JobStatus.PENDING;
1699
- job.worker_id = null;
1700
- job.progress = 0;
1701
- job.progress_message = "";
1702
- job.progress_details = null;
1703
- this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1704
- }
1705
- }
1706
- async abort(id) {
1707
- await sleep(0);
1708
- const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
1709
- if (job) {
1710
- const oldJob = { ...job };
1711
- job.status = JobStatus.ABORTING;
1712
- this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
1713
- }
1714
- }
1715
- async getByRunId(runId) {
1716
- await sleep(0);
1717
- return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
1718
- }
1719
- async deleteAll() {
1720
- await sleep(0);
1721
- const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
1722
- this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
1723
- for (const job of deletedJobs) {
1724
- this.events.emit("change", { type: "DELETE", old: job });
1725
- }
1726
- }
1727
- async outputForInput(input) {
1728
- await sleep(0);
1729
- const fingerprint = await makeFingerprint4(input);
1730
- return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === JobStatus.COMPLETED)?.output ?? null;
1731
- }
1732
- async delete(id) {
1733
- await sleep(0);
1734
- const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
1735
- this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
1736
- if (deletedJob) {
1737
- this.events.emit("change", { type: "DELETE", old: deletedJob });
1738
- }
1739
- }
1740
- async deleteJobsByStatusAndAge(status, olderThanMs) {
1741
- await sleep(0);
1742
- const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
1743
- const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
1744
- this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
1745
- for (const job of deletedJobs) {
1746
- this.events.emit("change", { type: "DELETE", old: job });
1747
- }
1748
- }
1749
- async setupDatabase() {}
1750
- matchesPrefixFilter(job, prefixFilter) {
1751
- if (prefixFilter && Object.keys(prefixFilter).length === 0) {
1752
- return true;
1753
- }
1754
- const filterValues = prefixFilter ?? this.prefixValues;
1755
- if (Object.keys(filterValues).length === 0) {
1756
- return true;
1757
- }
1758
- const jobWithPrefixes = job;
1759
- for (const [key, value] of Object.entries(filterValues)) {
1760
- if (jobWithPrefixes[key] !== value) {
1761
- return false;
1762
- }
1763
- }
1764
- return true;
1765
- }
1766
- subscribeToChanges(callback, options) {
1767
- const prefixFilter = options?.prefixFilter;
1768
- const filteredCallback = (change) => {
1769
- const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
1770
- const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
1771
- if (!newMatches && !oldMatches) {
1772
- return;
1773
- }
1774
- callback(change);
1775
- };
1776
- return this.events.subscribe("change", filteredCallback);
1777
- }
1778
- }
1779
- // src/queue/TelemetryQueueStorage.ts
1780
- class TelemetryQueueStorage {
1781
- storageName;
1782
- inner;
1783
- constructor(storageName, inner) {
1784
- this.storageName = storageName;
1785
- this.inner = inner;
1786
- }
1787
- get scope() {
1788
- return this.inner.scope;
1789
- }
1790
- add(job) {
1791
- return traced("workglow.storage.queue.add", this.storageName, () => this.inner.add(job));
1792
- }
1793
- get(id) {
1794
- return traced("workglow.storage.queue.get", this.storageName, () => this.inner.get(id));
1795
- }
1796
- next(workerId) {
1797
- return traced("workglow.storage.queue.next", this.storageName, () => this.inner.next(workerId));
1798
- }
1799
- peek(status, num) {
1800
- return traced("workglow.storage.queue.peek", this.storageName, () => this.inner.peek(status, num));
1801
- }
1802
- size(status) {
1803
- return traced("workglow.storage.queue.size", this.storageName, () => this.inner.size(status));
1804
- }
1805
- complete(job) {
1806
- return traced("workglow.storage.queue.complete", this.storageName, () => this.inner.complete(job));
1807
- }
1808
- release(id) {
1809
- return traced("workglow.storage.queue.release", this.storageName, () => this.inner.release(id));
1810
- }
1811
- deleteAll() {
1812
- return traced("workglow.storage.queue.deleteAll", this.storageName, () => this.inner.deleteAll());
1813
- }
1814
- outputForInput(input) {
1815
- return traced("workglow.storage.queue.outputForInput", this.storageName, () => this.inner.outputForInput(input));
1816
- }
1817
- abort(id) {
1818
- return traced("workglow.storage.queue.abort", this.storageName, () => this.inner.abort(id));
1819
- }
1820
- getByRunId(runId) {
1821
- return traced("workglow.storage.queue.getByRunId", this.storageName, () => this.inner.getByRunId(runId));
1822
- }
1823
- saveProgress(id, progress, message, details) {
1824
- return traced("workglow.storage.queue.saveProgress", this.storageName, () => this.inner.saveProgress(id, progress, message, details));
1825
- }
1826
- delete(id) {
1827
- return traced("workglow.storage.queue.delete", this.storageName, () => this.inner.delete(id));
1828
- }
1829
- deleteJobsByStatusAndAge(status, olderThanMs) {
1830
- return traced("workglow.storage.queue.deleteJobsByStatusAndAge", this.storageName, () => this.inner.deleteJobsByStatusAndAge(status, olderThanMs));
1831
- }
1832
- setupDatabase() {
1833
- return this.inner.setupDatabase();
1834
- }
1835
- subscribeToChanges(callback, options) {
1836
- return this.inner.subscribeToChanges(callback, options);
1837
- }
1838
- }
1839
- // src/queue-limiter/InMemoryRateLimiterStorage.ts
1840
- import { createServiceToken as createServiceToken10, sleep as sleep2, uuid4 as uuid43 } from "@workglow/util";
1841
- var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken10("ratelimiter.storage.inMemory");
1842
-
1843
- class InMemoryRateLimiterStorage {
1844
- scope = "process";
1845
- prefixValues;
1846
- executions = new Map;
1847
- nextAvailableTimes = new Map;
1848
- reserveChains = new Map;
1849
- constructor(options) {
1850
- this.prefixValues = options?.prefixValues ?? {};
1851
- }
1852
- makeKey(queueName) {
1853
- const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
1854
- return prefixPart ? `${prefixPart}|${queueName}` : queueName;
1855
- }
1856
- async setupDatabase() {}
1857
- async withKeyLock(key, fn) {
1858
- const previous = this.reserveChains.get(key) ?? Promise.resolve();
1859
- let release;
1860
- const next = new Promise((resolve) => {
1861
- release = resolve;
1862
- });
1863
- this.reserveChains.set(key, next);
1864
- try {
1865
- await previous;
1866
- return await fn();
1867
- } finally {
1868
- release(undefined);
1869
- if (this.reserveChains.get(key) === next) {
1870
- this.reserveChains.delete(key);
1871
- }
1872
- }
1873
- }
1874
- async tryReserveExecution(queueName, maxExecutions, windowMs) {
1875
- const key = this.makeKey(queueName);
1876
- return this.withKeyLock(key, () => {
1877
- const now = Date.now();
1878
- const windowStart = new Date(now - windowMs);
1879
- const executions = this.executions.get(key) ?? [];
1880
- const live = executions.filter((e) => e.executedAt > windowStart);
1881
- if (live.length >= maxExecutions) {
1882
- if (live.length !== executions.length) {
1883
- this.executions.set(key, live);
1884
- }
1885
- return null;
1886
- }
1887
- const next = this.nextAvailableTimes.get(key);
1888
- if (next && next.getTime() > now) {
1889
- return null;
1890
- }
1891
- const id = uuid43();
1892
- live.push({ id, queueName, executedAt: new Date(now) });
1893
- this.executions.set(key, live);
1894
- return id;
1895
- });
1896
- }
1897
- async releaseExecution(queueName, token) {
1898
- if (token === null || token === undefined)
1899
- return;
1900
- const key = this.makeKey(queueName);
1901
- await this.withKeyLock(key, () => {
1902
- const executions = this.executions.get(key);
1903
- if (!executions || executions.length === 0)
1904
- return;
1905
- const idx = executions.findIndex((e) => e.id === token);
1906
- if (idx === -1)
1907
- return;
1908
- executions.splice(idx, 1);
1909
- this.executions.set(key, executions);
1910
- });
1911
- }
1912
- async recordExecution(queueName) {
1913
- await sleep2(0);
1914
- const key = this.makeKey(queueName);
1915
- const executions = this.executions.get(key) ?? [];
1916
- executions.push({
1917
- id: uuid43(),
1918
- queueName,
1919
- executedAt: new Date
1920
- });
1921
- this.executions.set(key, executions);
1922
- }
1923
- async getExecutionCount(queueName, windowStartTime) {
1924
- await sleep2(0);
1925
- const key = this.makeKey(queueName);
1926
- const executions = this.executions.get(key) ?? [];
1927
- const windowStart = new Date(windowStartTime);
1928
- return executions.filter((e) => e.executedAt > windowStart).length;
1929
- }
1930
- async getOldestExecutionAtOffset(queueName, offset) {
1931
- await sleep2(0);
1932
- const key = this.makeKey(queueName);
1933
- const executions = this.executions.get(key) ?? [];
1934
- const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
1935
- const execution = sorted[offset];
1936
- return execution?.executedAt.toISOString();
1937
- }
1938
- async getNextAvailableTime(queueName) {
1939
- await sleep2(0);
1940
- const key = this.makeKey(queueName);
1941
- const time = this.nextAvailableTimes.get(key);
1942
- return time?.toISOString();
1943
- }
1944
- async setNextAvailableTime(queueName, nextAvailableAt) {
1945
- await sleep2(0);
1946
- const key = this.makeKey(queueName);
1947
- this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
1948
- }
1949
- async clear(queueName) {
1950
- await sleep2(0);
1951
- const key = this.makeKey(queueName);
1952
- this.executions.delete(key);
1953
- this.nextAvailableTimes.delete(key);
1954
- }
1955
- }
1956
- // src/queue-limiter/IRateLimiterStorage.ts
1957
- import { createServiceToken as createServiceToken11 } from "@workglow/util";
1958
- var RATE_LIMITER_STORAGE = createServiceToken11("ratelimiter.storage");
1959
- // src/util/HybridSubscriptionManager.ts
1960
- class HybridSubscriptionManager {
1961
- subscribers = new Set;
1962
- lastKnownState = new Map;
1963
- initialized = false;
1964
- channel = null;
1965
- backupPollingIntervalId = null;
1966
- fetchState;
1967
- compareItems;
1968
- payloadFactory;
1969
- options;
1970
- hasBroadcastChannel;
1971
- constructor(channelName, fetchState, compareItems, payloadFactory, options) {
1972
- this.fetchState = fetchState;
1973
- this.compareItems = compareItems;
1974
- this.payloadFactory = payloadFactory;
1975
- this.options = {
1976
- defaultIntervalMs: options?.defaultIntervalMs ?? 1000,
1977
- backupPollingIntervalMs: options?.backupPollingIntervalMs ?? 5000,
1978
- useBroadcastChannel: options?.useBroadcastChannel ?? true,
1979
- broadcastChannelName: options?.broadcastChannelName ?? channelName
1980
- };
1981
- this.hasBroadcastChannel = this.options.useBroadcastChannel && typeof BroadcastChannel !== "undefined";
1982
- if (this.hasBroadcastChannel) {
1983
- this.initializeBroadcastChannel();
1984
- }
1985
- }
1986
- initializeBroadcastChannel() {
1987
- try {
1988
- this.channel = new BroadcastChannel(this.options.broadcastChannelName);
1989
- this.channel.onmessage = (event) => {
1990
- this.handleBroadcastMessage(event.data);
1991
- };
1992
- } catch (error) {
1993
- console.error("Failed to initialize BroadcastChannel:", error);
1994
- this.channel = null;
1995
- }
1996
- }
1997
- async handleBroadcastMessage(message) {
1998
- if (message.type === "CHANGE") {
1999
- await this.pollAndNotify();
1770
+ async handleBroadcastMessage(message) {
1771
+ if (message.type === "CHANGE") {
1772
+ await this.pollAndNotify();
2000
1773
  }
2001
1774
  }
2002
1775
  notifyLocalChange() {
@@ -2353,6 +2126,8 @@ class InMemoryVectorStorage extends InMemoryTabularStorage {
2353
2126
  }
2354
2127
  }
2355
2128
  // src/vector/TelemetryVectorStorage.ts
2129
+ import { traced as traced3 } from "@workglow/util";
2130
+
2356
2131
  class TelemetryVectorStorage extends TelemetryTabularStorage {
2357
2132
  vectorInner;
2358
2133
  constructor(storageName, inner) {
@@ -2363,13 +2138,13 @@ class TelemetryVectorStorage extends TelemetryTabularStorage {
2363
2138
  return this.vectorInner.getVectorDimensions();
2364
2139
  }
2365
2140
  similaritySearch(query, options) {
2366
- return traced("workglow.storage.vector.similaritySearch", this.storageName, () => this.vectorInner.similaritySearch(query, options));
2141
+ return traced3("workglow.storage.vector.similaritySearch", this.storageName, () => this.vectorInner.similaritySearch(query, options));
2367
2142
  }
2368
2143
  hybridSearch(query, options) {
2369
2144
  if (!this.vectorInner.hybridSearch) {
2370
2145
  throw new Error("hybridSearch is not supported by the underlying storage implementation");
2371
2146
  }
2372
- return traced("workglow.storage.vector.hybridSearch", this.storageName, () => this.vectorInner.hybridSearch(query, options));
2147
+ return traced3("workglow.storage.vector.hybridSearch", this.storageName, () => this.vectorInner.hybridSearch(query, options));
2373
2148
  }
2374
2149
  }
2375
2150
  // src/credentials/EncryptedKvCredentialStore.ts
@@ -2495,3750 +2270,201 @@ class LazyEncryptedCredentialStore {
2495
2270
  return this.inner.deleteAll();
2496
2271
  }
2497
2272
  }
2498
- // src/tabular/IndexedDbTabularStorage.ts
2499
- import { createServiceToken as createServiceToken12, deepEqual as deepEqual2, makeFingerprint as makeFingerprint5, uuid4 as uuid44 } from "@workglow/util";
2273
+ // src/tabular/SharedInMemoryTabularStorage.ts
2274
+ import { createServiceToken as createServiceToken8 } from "@workglow/util";
2275
+ var SHARED_IN_MEMORY_TABULAR_REPOSITORY = createServiceToken8("storage.tabularRepository.sharedInMemory");
2276
+ var SYNC_TIMEOUT = 1000;
2277
+ var MAX_PENDING_MESSAGES = 1000;
2500
2278
 
2501
- // src/util/IndexedDbTable.ts
2502
- import { deepEqual } from "@workglow/util";
2503
- var METADATA_STORE_NAME = "__schema_metadata__";
2504
- async function saveSchemaMetadata(db, tableName, snapshot) {
2505
- return new Promise((resolve, reject) => {
2279
+ class SharedInMemoryTabularStorage extends BaseTabularStorage {
2280
+ channel = null;
2281
+ channelName;
2282
+ inMemoryRepo;
2283
+ isInitialized = false;
2284
+ syncInProgress = false;
2285
+ pendingMessages = [];
2286
+ constructor(channelName = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
2287
+ super(schema, primaryKeyNames, indexes, clientProvidedKeys);
2288
+ this.channelName = channelName;
2289
+ this.inMemoryRepo = new InMemoryTabularStorage(schema, primaryKeyNames, indexes, clientProvidedKeys);
2290
+ this.setupEventForwarding();
2291
+ this.initializeBroadcastChannel();
2292
+ }
2293
+ isBroadcastChannelAvailable() {
2294
+ return typeof BroadcastChannel !== "undefined";
2295
+ }
2296
+ initializeBroadcastChannel() {
2297
+ if (!this.isBroadcastChannelAvailable()) {
2298
+ console.warn("BroadcastChannel is not available. Tab synchronization will not work.");
2299
+ return;
2300
+ }
2506
2301
  try {
2507
- const transaction = db.transaction(METADATA_STORE_NAME, "readwrite");
2508
- const store = transaction.objectStore(METADATA_STORE_NAME);
2509
- const request = store.put({ ...snapshot, tableName }, tableName);
2510
- request.onsuccess = () => resolve();
2511
- request.onerror = () => reject(request.error);
2512
- transaction.onerror = () => reject(transaction.error);
2513
- } catch (err) {
2514
- resolve();
2515
- }
2516
- });
2517
- }
2518
- async function openIndexedDbTable(tableName, version, upgradeNeededCallback) {
2519
- return new Promise((resolve, reject) => {
2520
- const openRequest = indexedDB.open(tableName, version);
2521
- openRequest.onsuccess = (event) => {
2522
- const db = event.target.result;
2523
- db.onversionchange = () => {
2524
- db.close();
2302
+ this.channel = new BroadcastChannel(this.channelName);
2303
+ this.channel.onmessage = (event) => {
2304
+ this.handleBroadcastMessage(event.data);
2525
2305
  };
2526
- resolve(db);
2527
- };
2528
- openRequest.onupgradeneeded = (event) => {
2529
- if (upgradeNeededCallback) {
2530
- upgradeNeededCallback(event);
2531
- }
2532
- };
2533
- openRequest.onerror = () => {
2534
- const error = openRequest.error;
2535
- if (error && error.name === "VersionError") {
2536
- reject(new Error(`Database ${tableName} exists at a higher version. Cannot open at version ${version || "current"}.`));
2537
- } else {
2538
- reject(error);
2539
- }
2540
- };
2541
- openRequest.onblocked = () => {
2542
- reject(new Error(`Database ${tableName} is blocked. Close all other tabs using this database.`));
2543
- };
2544
- });
2545
- }
2546
- async function deleteIndexedDbTable(tableName) {
2547
- return new Promise((resolve, reject) => {
2548
- const deleteRequest = indexedDB.deleteDatabase(tableName);
2549
- deleteRequest.onsuccess = () => resolve();
2550
- deleteRequest.onerror = () => reject(deleteRequest.error);
2551
- deleteRequest.onblocked = () => {
2552
- reject(new Error(`Cannot delete database ${tableName}. Close all other tabs using this database.`));
2553
- };
2554
- });
2555
- }
2556
- function compareSchemas(store, expectedPrimaryKey, expectedIndexes) {
2557
- const diff = {
2558
- indexesToAdd: [],
2559
- indexesToRemove: [],
2560
- indexesToModify: [],
2561
- primaryKeyChanged: false,
2562
- needsObjectStoreRecreation: false
2563
- };
2564
- const actualKeyPath = store.keyPath;
2565
- const normalizedExpected = Array.isArray(expectedPrimaryKey) ? expectedPrimaryKey : expectedPrimaryKey;
2566
- const normalizedActual = Array.isArray(actualKeyPath) ? actualKeyPath : actualKeyPath;
2567
- if (!deepEqual(normalizedExpected, normalizedActual)) {
2568
- diff.primaryKeyChanged = true;
2569
- diff.needsObjectStoreRecreation = true;
2570
- return diff;
2571
- }
2572
- const existingIndexes = new Map;
2573
- for (let i = 0;i < store.indexNames.length; i++) {
2574
- const indexName = store.indexNames[i];
2575
- existingIndexes.set(indexName, store.index(indexName));
2576
- }
2577
- for (const expectedIdx of expectedIndexes) {
2578
- const existingIdx = existingIndexes.get(expectedIdx.name);
2579
- if (!existingIdx) {
2580
- diff.indexesToAdd.push(expectedIdx);
2581
- } else {
2582
- const expectedKeyPath = Array.isArray(expectedIdx.keyPath) ? expectedIdx.keyPath : [expectedIdx.keyPath];
2583
- const actualKeyPath2 = Array.isArray(existingIdx.keyPath) ? existingIdx.keyPath : [existingIdx.keyPath];
2584
- const keyPathChanged = !deepEqual(expectedKeyPath, actualKeyPath2);
2585
- const uniqueChanged = existingIdx.unique !== (expectedIdx.options?.unique ?? false);
2586
- const multiEntryChanged = existingIdx.multiEntry !== (expectedIdx.options?.multiEntry ?? false);
2587
- if (keyPathChanged || uniqueChanged || multiEntryChanged) {
2588
- diff.indexesToModify.push(expectedIdx);
2589
- }
2590
- existingIndexes.delete(expectedIdx.name);
2306
+ this.syncFromOtherTabs();
2307
+ } catch (error) {
2308
+ console.error("Failed to initialize BroadcastChannel:", error);
2591
2309
  }
2592
2310
  }
2593
- diff.indexesToRemove = Array.from(existingIndexes.keys());
2594
- return diff;
2595
- }
2596
- async function readAllData(store) {
2597
- return new Promise((resolve, reject) => {
2598
- const request = store.getAll();
2599
- request.onsuccess = () => resolve(request.result || []);
2600
- request.onerror = () => reject(request.error);
2601
- });
2602
- }
2603
- async function performIncrementalMigration(db, tableName, diff, options = {}) {
2604
- const currentVersion = db.version;
2605
- const newVersion = currentVersion + 1;
2606
- db.close();
2607
- options.onMigrationProgress?.(`Migrating ${tableName} from version ${currentVersion} to ${newVersion}...`, 0);
2608
- return openIndexedDbTable(tableName, newVersion, (event) => {
2609
- const transaction = event.target.transaction;
2610
- const store = transaction.objectStore(tableName);
2611
- for (const indexName of diff.indexesToRemove) {
2612
- options.onMigrationProgress?.(`Removing index: ${indexName}`, 0.2);
2613
- store.deleteIndex(indexName);
2614
- }
2615
- for (const indexDef of diff.indexesToModify) {
2616
- options.onMigrationProgress?.(`Updating index: ${indexDef.name}`, 0.4);
2617
- if (store.indexNames.contains(indexDef.name)) {
2618
- store.deleteIndex(indexDef.name);
2311
+ setupEventForwarding() {
2312
+ this.inMemoryRepo.on("put", (entity) => {
2313
+ this.events.emit("put", entity);
2314
+ });
2315
+ this.inMemoryRepo.on("get", (key, entity) => {
2316
+ this.events.emit("get", key, entity);
2317
+ });
2318
+ this.inMemoryRepo.on("query", (key, entities) => {
2319
+ this.events.emit("query", key, entities);
2320
+ });
2321
+ this.inMemoryRepo.on("delete", (key) => {
2322
+ this.events.emit("delete", key);
2323
+ });
2324
+ this.inMemoryRepo.on("clearall", () => {
2325
+ this.events.emit("clearall");
2326
+ });
2327
+ }
2328
+ async handleBroadcastMessage(message) {
2329
+ if (this.syncInProgress && message.type !== "SYNC_RESPONSE") {
2330
+ if (this.pendingMessages.length < MAX_PENDING_MESSAGES) {
2331
+ this.pendingMessages.push(message);
2619
2332
  }
2620
- store.createIndex(indexDef.name, indexDef.keyPath, indexDef.options);
2621
- }
2622
- for (const indexDef of diff.indexesToAdd) {
2623
- options.onMigrationProgress?.(`Adding index: ${indexDef.name}`, 0.6);
2624
- store.createIndex(indexDef.name, indexDef.keyPath, indexDef.options);
2333
+ return;
2625
2334
  }
2626
- options.onMigrationProgress?.(`Migration complete`, 1);
2627
- });
2628
- }
2629
- async function performDestructiveMigration(db, tableName, primaryKey, expectedIndexes, options = {}, autoIncrement = false) {
2630
- if (!options.allowDestructiveMigration) {
2631
- throw new Error(`Destructive migration required for ${tableName} but not allowed. ` + `Primary key has changed. Set allowDestructiveMigration=true to proceed with data loss, ` + `or provide a dataTransformer to migrate data.`);
2632
- }
2633
- const currentVersion = db.version;
2634
- const newVersion = currentVersion + 1;
2635
- options.onMigrationProgress?.(`Performing destructive migration of ${tableName}. Reading existing data...`, 0);
2636
- let existingData = [];
2637
- try {
2638
- const transaction = db.transaction(tableName, "readonly");
2639
- const store = transaction.objectStore(tableName);
2640
- existingData = await readAllData(store);
2641
- options.onMigrationProgress?.(`Read ${existingData.length} records`, 0.3);
2642
- } catch (err) {
2643
- options.onMigrationWarning?.(`Failed to read existing data during migration: ${err}`, err);
2644
- }
2645
- db.close();
2646
- if (options.dataTransformer && existingData.length > 0) {
2647
- options.onMigrationProgress?.(`Transforming ${existingData.length} records...`, 0.4);
2648
- try {
2649
- const transformed = [];
2650
- for (let i = 0;i < existingData.length; i++) {
2651
- const record = existingData[i];
2652
- const transformedRecord = await options.dataTransformer(record);
2653
- if (transformedRecord !== undefined && transformedRecord !== null) {
2654
- transformed.push(transformedRecord);
2655
- }
2656
- if (i % 100 === 0) {
2657
- options.onMigrationProgress?.(`Transformed ${i}/${existingData.length} records`, 0.4 + i / existingData.length * 0.3);
2335
+ switch (message.type) {
2336
+ case "SYNC_REQUEST":
2337
+ const all = await this.inMemoryRepo.getAll();
2338
+ if (this.channel && all) {
2339
+ this.channel.postMessage({
2340
+ type: "SYNC_RESPONSE",
2341
+ data: all
2342
+ });
2658
2343
  }
2659
- }
2660
- existingData = transformed;
2661
- options.onMigrationProgress?.(`Transformation complete: ${existingData.length} records`, 0.7);
2662
- } catch (err) {
2663
- options.onMigrationWarning?.(`Data transformation failed: ${err}. Some data may be lost.`, err);
2664
- existingData = [];
2665
- }
2666
- }
2667
- options.onMigrationProgress?.(`Recreating object store...`, 0.75);
2668
- const newDb = await openIndexedDbTable(tableName, newVersion, (event) => {
2669
- const db2 = event.target.result;
2670
- if (db2.objectStoreNames.contains(tableName)) {
2671
- db2.deleteObjectStore(tableName);
2672
- }
2673
- const store = db2.createObjectStore(tableName, { keyPath: primaryKey, autoIncrement });
2674
- for (const idx of expectedIndexes) {
2675
- store.createIndex(idx.name, idx.keyPath, idx.options);
2676
- }
2677
- if (existingData.length > 0) {
2678
- options.onMigrationProgress?.(`Restoring ${existingData.length} records...`, 0.8);
2679
- for (const record of existingData) {
2680
- try {
2681
- store.put(record);
2682
- } catch (err) {
2683
- options.onMigrationWarning?.(`Failed to restore record: ${err}`, err);
2344
+ break;
2345
+ case "SYNC_RESPONSE":
2346
+ if (message.data && Array.isArray(message.data)) {
2347
+ await this.copyDataFromArray(message.data);
2684
2348
  }
2685
- }
2686
- }
2687
- });
2688
- options.onMigrationProgress?.(`Destructive migration complete`, 1);
2689
- return newDb;
2690
- }
2691
- async function createNewDatabase(tableName, primaryKey, expectedIndexes, options = {}, autoIncrement = false) {
2692
- options.onMigrationProgress?.(`Creating new database: ${tableName}`, 0);
2693
- try {
2694
- await deleteIndexedDbTable(tableName);
2695
- await new Promise((resolve) => setTimeout(resolve, 50));
2696
- } catch (err) {}
2697
- const version = 1;
2698
- const db = await openIndexedDbTable(tableName, version, (event) => {
2699
- const db2 = event.target.result;
2700
- if (!db2.objectStoreNames.contains(METADATA_STORE_NAME)) {
2701
- db2.createObjectStore(METADATA_STORE_NAME, { keyPath: "tableName" });
2702
- }
2703
- const store = db2.createObjectStore(tableName, { keyPath: primaryKey, autoIncrement });
2704
- for (const idx of expectedIndexes) {
2705
- store.createIndex(idx.name, idx.keyPath, idx.options);
2706
- }
2707
- });
2708
- const snapshot = {
2709
- version: db.version,
2710
- primaryKey,
2711
- indexes: expectedIndexes,
2712
- recordCount: 0,
2713
- timestamp: Date.now()
2714
- };
2715
- await saveSchemaMetadata(db, tableName, snapshot);
2716
- options.onMigrationProgress?.(`Database created successfully`, 1);
2717
- return db;
2718
- }
2719
- async function ensureIndexedDbTable(tableName, primaryKey, expectedIndexes = [], options = {}, autoIncrement = false) {
2720
- try {
2721
- let db;
2722
- let wasJustCreated = false;
2723
- try {
2724
- db = await openIndexedDbTable(tableName);
2725
- if (db.version === 1 && !db.objectStoreNames.contains(tableName)) {
2726
- wasJustCreated = true;
2727
- db.close();
2728
- }
2729
- } catch (err) {
2730
- options.onMigrationProgress?.(`Database ${tableName} does not exist or has version conflict, creating...`, 0);
2731
- return await createNewDatabase(tableName, primaryKey, expectedIndexes, options, autoIncrement);
2349
+ this.syncInProgress = false;
2350
+ await this.drainPendingMessages();
2351
+ break;
2352
+ case "PUT":
2353
+ await this.inMemoryRepo.put(message.entity);
2354
+ break;
2355
+ case "PUT_BULK":
2356
+ await this.inMemoryRepo.putBulk(message.entities);
2357
+ break;
2358
+ case "DELETE":
2359
+ await this.inMemoryRepo.delete(message.key);
2360
+ break;
2361
+ case "DELETE_ALL":
2362
+ await this.inMemoryRepo.deleteAll();
2363
+ break;
2364
+ case "DELETE_SEARCH":
2365
+ await this.inMemoryRepo.deleteSearch(message.criteria);
2366
+ break;
2732
2367
  }
2733
- if (wasJustCreated) {
2734
- options.onMigrationProgress?.(`Creating new database: ${tableName}`, 0);
2735
- try {
2736
- await deleteIndexedDbTable(tableName);
2737
- await new Promise((resolve) => setTimeout(resolve, 50));
2738
- } catch (err) {}
2739
- db = await openIndexedDbTable(tableName, 1, (event) => {
2740
- const db2 = event.target.result;
2741
- if (!db2.objectStoreNames.contains(METADATA_STORE_NAME)) {
2742
- db2.createObjectStore(METADATA_STORE_NAME, { keyPath: "tableName" });
2743
- }
2744
- const store2 = db2.createObjectStore(tableName, { keyPath: primaryKey, autoIncrement });
2745
- for (const idx of expectedIndexes) {
2746
- store2.createIndex(idx.name, idx.keyPath, idx.options);
2747
- }
2748
- });
2749
- const snapshot2 = {
2750
- version: db.version,
2751
- primaryKey,
2752
- indexes: expectedIndexes,
2753
- recordCount: 0,
2754
- timestamp: Date.now()
2755
- };
2756
- await saveSchemaMetadata(db, tableName, snapshot2);
2757
- options.onMigrationProgress?.(`Database created successfully`, 1);
2758
- return db;
2759
- }
2760
- if (!db.objectStoreNames.contains(METADATA_STORE_NAME)) {
2761
- const currentVersion = db.version;
2762
- db.close();
2763
- db = await openIndexedDbTable(tableName, currentVersion + 1, (event) => {
2764
- const db2 = event.target.result;
2765
- if (!db2.objectStoreNames.contains(METADATA_STORE_NAME)) {
2766
- db2.createObjectStore(METADATA_STORE_NAME, { keyPath: "tableName" });
2368
+ }
2369
+ async drainPendingMessages() {
2370
+ while (!this.syncInProgress && this.pendingMessages.length > 0) {
2371
+ const messages = this.pendingMessages;
2372
+ this.pendingMessages = [];
2373
+ for (const message of messages) {
2374
+ await this.handleBroadcastMessage(message);
2375
+ if (this.syncInProgress) {
2376
+ break;
2767
2377
  }
2768
- });
2378
+ }
2769
2379
  }
2770
- if (!db.objectStoreNames.contains(tableName)) {
2771
- options.onMigrationProgress?.(`Object store ${tableName} does not exist, creating...`, 0);
2772
- db.close();
2773
- return await createNewDatabase(tableName, primaryKey, expectedIndexes, options, autoIncrement);
2774
- }
2775
- const transaction = db.transaction(tableName, "readonly");
2776
- const store = transaction.objectStore(tableName);
2777
- const diff = compareSchemas(store, primaryKey, expectedIndexes);
2778
- await new Promise((resolve) => {
2779
- transaction.oncomplete = () => resolve();
2780
- transaction.onerror = () => resolve();
2781
- });
2782
- const needsMigration = diff.indexesToAdd.length > 0 || diff.indexesToRemove.length > 0 || diff.indexesToModify.length > 0 || diff.needsObjectStoreRecreation;
2783
- if (!needsMigration) {
2784
- options.onMigrationProgress?.(`Schema for ${tableName} is up to date`, 1);
2785
- const snapshot2 = {
2786
- version: db.version,
2787
- primaryKey,
2788
- indexes: expectedIndexes,
2789
- timestamp: Date.now()
2790
- };
2791
- await saveSchemaMetadata(db, tableName, snapshot2);
2792
- return db;
2793
- }
2794
- if (diff.needsObjectStoreRecreation) {
2795
- options.onMigrationProgress?.(`Schema change requires object store recreation for ${tableName}`, 0);
2796
- db = await performDestructiveMigration(db, tableName, primaryKey, expectedIndexes, options, autoIncrement);
2797
- } else {
2798
- options.onMigrationProgress?.(`Performing incremental migration for ${tableName}`, 0);
2799
- db = await performIncrementalMigration(db, tableName, diff, options);
2800
- }
2801
- const snapshot = {
2802
- version: db.version,
2803
- primaryKey,
2804
- indexes: expectedIndexes,
2805
- timestamp: Date.now()
2806
- };
2807
- await saveSchemaMetadata(db, tableName, snapshot);
2808
- return db;
2809
- } catch (err) {
2810
- options.onMigrationWarning?.(`Migration failed for ${tableName}: ${err}`, err);
2811
- throw err;
2812
- }
2813
- }
2814
- async function dropIndexedDbTable(tableName) {
2815
- return deleteIndexedDbTable(tableName);
2816
- }
2817
-
2818
- // src/tabular/IndexedDbTabularStorage.ts
2819
- var IDB_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.indexedDb");
2820
- function compareEntitiesForChange(a, b) {
2821
- const au = a?.updated_at;
2822
- const bu = b?.updated_at;
2823
- if (typeof au === "string" && typeof bu === "string") {
2824
- return au === bu;
2825
- }
2826
- return deepEqual2(a, b);
2827
- }
2828
-
2829
- class IndexedDbTabularStorage extends BaseTabularStorage {
2830
- table;
2831
- db;
2832
- setupPromise = null;
2833
- migrationOptions;
2834
- hybridManager = null;
2835
- hybridOptions;
2836
- cursorSafeIndexes;
2837
- constructor(table = "tabular_store", schema, primaryKeyNames, indexes = [], migrationOptions = {}, clientProvidedKeys = "if-missing") {
2838
- super(schema, primaryKeyNames, indexes, clientProvidedKeys);
2839
- this.table = table;
2840
- this.migrationOptions = migrationOptions;
2841
- this.hybridOptions = {
2842
- useBroadcastChannel: migrationOptions.useBroadcastChannel ?? true,
2843
- backupPollingIntervalMs: migrationOptions.backupPollingIntervalMs ?? 5000
2844
- };
2845
- }
2846
- async getDb() {
2847
- if (this.db)
2848
- return this.db;
2849
- await this.setupDatabase();
2850
- return this.db;
2851
2380
  }
2852
- async setupDatabase() {
2853
- if (this.db)
2854
- return;
2855
- if (this.setupPromise) {
2856
- await this.setupPromise;
2381
+ syncFromOtherTabs() {
2382
+ if (!this.channel)
2857
2383
  return;
2858
- }
2859
- this.setupPromise = this.performSetup();
2860
- try {
2861
- this.db = await this.setupPromise;
2862
- } finally {
2863
- this.setupPromise = null;
2864
- }
2865
- }
2866
- async performSetup() {
2867
- const pkColumns = super.primaryKeyColumns();
2868
- const expectedIndexes = [];
2869
- for (const spec of this.indexes) {
2870
- const columns = spec;
2871
- if (columns.length <= pkColumns.length) {
2872
- const isPkPrefix = columns.every((col, idx) => col === pkColumns[idx]);
2873
- if (isPkPrefix)
2874
- continue;
2384
+ this.syncInProgress = true;
2385
+ this.channel.postMessage({ type: "SYNC_REQUEST" });
2386
+ setTimeout(() => {
2387
+ if (this.syncInProgress) {
2388
+ this.syncInProgress = false;
2389
+ this.drainPendingMessages().catch((error) => {
2390
+ console.error("Failed to drain pending messages after sync timeout", error);
2391
+ });
2875
2392
  }
2876
- const columnNames = columns.map((col) => String(col));
2877
- const indexName = columnNames.join("_");
2878
- expectedIndexes.push({
2879
- name: indexName,
2880
- keyPath: columnNames.length === 1 ? columnNames[0] : columnNames,
2881
- options: { unique: false }
2882
- });
2883
- }
2884
- const primaryKey = pkColumns.length === 1 ? pkColumns[0] : pkColumns;
2885
- const useAutoIncrement = this.hasAutoGeneratedKey() && this.autoGeneratedKeyStrategy === "autoincrement" && pkColumns.length === 1;
2886
- return await ensureIndexedDbTable(this.table, primaryKey, expectedIndexes, this.migrationOptions, useAutoIncrement);
2393
+ }, SYNC_TIMEOUT);
2887
2394
  }
2888
- generateKeyValue(columnName, strategy) {
2889
- if (strategy === "uuid") {
2890
- return uuid44();
2891
- }
2892
- throw new Error(`IndexedDB autoincrement keys are generated by the database, not client-side. Column: ${columnName}`);
2893
- }
2894
- async put(record) {
2895
- const db = await this.getDb();
2896
- let recordToStore = record;
2897
- if (this.hasAutoGeneratedKey() && this.autoGeneratedKeyName) {
2898
- const keyName = String(this.autoGeneratedKeyName);
2899
- const clientProvidedValue = record[keyName];
2900
- const hasClientValue = clientProvidedValue !== undefined && clientProvidedValue !== null;
2901
- if (this.autoGeneratedKeyStrategy === "uuid") {
2902
- let shouldGenerate = false;
2903
- if (this.clientProvidedKeys === "never") {
2904
- shouldGenerate = true;
2905
- } else if (this.clientProvidedKeys === "always") {
2906
- if (!hasClientValue) {
2907
- throw new Error(`Auto-generated key "${keyName}" is required when clientProvidedKeys is "always"`);
2908
- }
2909
- shouldGenerate = false;
2910
- } else {
2911
- shouldGenerate = !hasClientValue;
2912
- }
2913
- if (shouldGenerate) {
2914
- const generatedValue = this.generateKeyValue(keyName, "uuid");
2915
- recordToStore = { ...record, [keyName]: generatedValue };
2916
- }
2917
- } else if (this.autoGeneratedKeyStrategy === "autoincrement") {
2918
- if (this.clientProvidedKeys === "always" && !hasClientValue) {
2919
- throw new Error(`Auto-generated key "${keyName}" is required when clientProvidedKeys is "always"`);
2920
- }
2921
- if (this.clientProvidedKeys === "never") {
2922
- const { [keyName]: _, ...rest } = record;
2923
- recordToStore = rest;
2924
- }
2925
- }
2395
+ async copyDataFromArray(entities) {
2396
+ if (entities.length === 0)
2397
+ return;
2398
+ await this.inMemoryRepo.deleteAll();
2399
+ await this.inMemoryRepo.putBulk(entities);
2400
+ }
2401
+ broadcast(message) {
2402
+ if (this.channel) {
2403
+ this.channel.postMessage(message);
2926
2404
  }
2927
- return new Promise((resolve, reject) => {
2928
- const transaction = db.transaction(this.table, "readwrite");
2929
- const store = transaction.objectStore(this.table);
2930
- const request = store.put(recordToStore);
2931
- request.onerror = () => {
2932
- reject(request.error);
2933
- };
2934
- request.onsuccess = () => {
2935
- if (this.hasAutoGeneratedKey() && this.autoGeneratedKeyName && this.autoGeneratedKeyStrategy === "autoincrement") {
2936
- const keyName = String(this.autoGeneratedKeyName);
2937
- if (recordToStore[keyName] === undefined) {
2938
- recordToStore = { ...recordToStore, [keyName]: request.result };
2939
- }
2940
- }
2941
- this.events.emit("put", recordToStore);
2942
- resolve(recordToStore);
2943
- };
2944
- transaction.oncomplete = () => {
2945
- this.hybridManager?.notifyLocalChange();
2946
- };
2947
- });
2948
2405
  }
2949
- async putBulk(records) {
2950
- return await Promise.all(records.map((record) => this.put(record)));
2406
+ async setupDatabase() {
2407
+ if (this.isInitialized)
2408
+ return;
2409
+ this.isInitialized = true;
2410
+ await this.syncFromOtherTabs();
2951
2411
  }
2952
- getPrimaryKeyAsOrderedArray(key) {
2953
- return super.getPrimaryKeyAsOrderedArray(key).map((value) => typeof value === "bigint" ? value.toString() : value);
2412
+ async put(value) {
2413
+ const result = await this.inMemoryRepo.put(value);
2414
+ this.broadcast({ type: "PUT", entity: result });
2415
+ return result;
2954
2416
  }
2955
- getIndexedKey(key) {
2956
- const keys = super.getPrimaryKeyAsOrderedArray(key).map((value) => typeof value === "bigint" ? value.toString() : value);
2957
- return keys.length === 1 ? keys[0] : keys;
2417
+ async putBulk(values) {
2418
+ const result = await this.inMemoryRepo.putBulk(values);
2419
+ this.broadcast({ type: "PUT_BULK", entities: result });
2420
+ return result;
2958
2421
  }
2959
2422
  async get(key) {
2960
- const db = await this.getDb();
2961
- return new Promise((resolve, reject) => {
2962
- const transaction = db.transaction(this.table, "readonly");
2963
- const store = transaction.objectStore(this.table);
2964
- const request = store.get(this.getIndexedKey(key));
2965
- request.onerror = () => reject(request.error);
2966
- request.onsuccess = () => {
2967
- if (!request.result) {
2968
- this.events.emit("get", key, undefined);
2969
- resolve(undefined);
2970
- return;
2971
- }
2972
- this.events.emit("get", key, request.result);
2973
- resolve(request.result);
2974
- };
2975
- });
2976
- }
2977
- async getAll(options) {
2978
- this.validateGetAllOptions(options);
2979
- const db = await this.getDb();
2980
- const transaction = db.transaction(this.table, "readonly");
2981
- const store = transaction.objectStore(this.table);
2982
- const request = store.getAll();
2983
- return new Promise((resolve, reject) => {
2984
- request.onerror = () => reject(request.error);
2985
- request.onsuccess = () => {
2986
- let values = request.result;
2987
- if (values.length === 0) {
2988
- resolve(undefined);
2989
- return;
2990
- }
2991
- if (options?.orderBy && options.orderBy.length > 0) {
2992
- values.sort((a, b) => {
2993
- for (const { column, direction } of options.orderBy) {
2994
- const aVal = a[column];
2995
- const bVal = b[column];
2996
- if (aVal == null && bVal == null)
2997
- continue;
2998
- if (aVal == null)
2999
- return direction === "ASC" ? -1 : 1;
3000
- if (bVal == null)
3001
- return direction === "ASC" ? 1 : -1;
3002
- if (aVal < bVal)
3003
- return direction === "ASC" ? -1 : 1;
3004
- if (aVal > bVal)
3005
- return direction === "ASC" ? 1 : -1;
3006
- }
3007
- return 0;
3008
- });
3009
- }
3010
- if (options?.offset !== undefined) {
3011
- values = values.slice(options.offset);
3012
- }
3013
- if (options?.limit !== undefined) {
3014
- values = values.slice(0, options.limit);
3015
- }
3016
- resolve(values.length > 0 ? values : undefined);
3017
- };
3018
- });
2423
+ return await this.inMemoryRepo.get(key);
3019
2424
  }
3020
- async delete(key) {
3021
- const db = await this.getDb();
3022
- return new Promise((resolve, reject) => {
3023
- const transaction = db.transaction(this.table, "readwrite");
3024
- const store = transaction.objectStore(this.table);
3025
- const request = store.delete(this.getIndexedKey(key));
3026
- request.onerror = () => reject(request.error);
3027
- request.onsuccess = () => {
3028
- this.events.emit("delete", key);
3029
- resolve();
3030
- };
3031
- transaction.oncomplete = () => {
3032
- this.hybridManager?.notifyLocalChange();
3033
- };
3034
- });
2425
+ async delete(value) {
2426
+ await this.inMemoryRepo.delete(value);
2427
+ const { key } = this.separateKeyValueFromCombined(value);
2428
+ this.broadcast({ type: "DELETE", key });
3035
2429
  }
3036
2430
  async deleteAll() {
3037
- const db = await this.getDb();
3038
- return new Promise((resolve, reject) => {
3039
- const transaction = db.transaction(this.table, "readwrite");
3040
- const store = transaction.objectStore(this.table);
3041
- const request = store.clear();
3042
- request.onerror = () => reject(request.error);
3043
- request.onsuccess = () => {
3044
- this.events.emit("clearall");
3045
- resolve();
3046
- };
3047
- transaction.oncomplete = () => {
3048
- this.hybridManager?.notifyLocalChange();
3049
- };
3050
- });
3051
- }
3052
- async size() {
3053
- const db = await this.getDb();
3054
- return new Promise((resolve, reject) => {
3055
- const transaction = db.transaction(this.table, "readonly");
3056
- const store = transaction.objectStore(this.table);
3057
- const request = store.count();
3058
- request.onerror = () => reject(request.error);
3059
- request.onsuccess = () => resolve(request.result);
3060
- });
3061
- }
3062
- getCursorSafeIndexes() {
3063
- if (this.cursorSafeIndexes)
3064
- return this.cursorSafeIndexes;
3065
- const required = new Set(this.schema.required ?? []);
3066
- this.cursorSafeIndexes = this.indexes.filter((columns) => columns.every((column) => required.has(String(column))));
3067
- return this.cursorSafeIndexes;
2431
+ await this.inMemoryRepo.deleteAll();
2432
+ this.broadcast({ type: "DELETE_ALL" });
3068
2433
  }
3069
- createIndexedRange(store, criteria) {
3070
- const criteriaColumns = Object.keys(criteria);
3071
- if (criteriaColumns.length === 0)
3072
- return;
3073
- let best;
3074
- for (const indexColumns of this.getCursorSafeIndexes()) {
3075
- const prefixValues = [];
3076
- for (const column of indexColumns) {
3077
- const value = this.getEqualityCriterionValue(criteria, column);
3078
- if (value === undefined)
3079
- break;
3080
- prefixValues.push(value);
3081
- }
3082
- if (prefixValues.length === 0)
3083
- continue;
3084
- const indexedPrefix = indexColumns.slice(0, prefixValues.length);
3085
- const coversCriteria = criteriaColumns.every((column) => indexedPrefix.includes(column));
3086
- const better = !best || coversCriteria && !best.coversCriteria || coversCriteria === best.coversCriteria && prefixValues.length > best.prefixValues.length;
3087
- if (better) {
3088
- best = {
3089
- indexName: indexColumns.map((column) => String(column)).join("_"),
3090
- prefixValues,
3091
- fullMatch: prefixValues.length === indexColumns.length,
3092
- coversCriteria
3093
- };
3094
- }
3095
- }
3096
- if (!best)
3097
- return;
3098
- const range = best.fullMatch ? IDBKeyRange.only(best.prefixValues.length === 1 ? best.prefixValues[0] : best.prefixValues) : IDBKeyRange.bound(best.prefixValues, [...best.prefixValues, []]);
3099
- return {
3100
- source: store.index(best.indexName),
3101
- range,
3102
- coversCriteria: best.coversCriteria
3103
- };
2434
+ async getAll(options) {
2435
+ return await this.inMemoryRepo.getAll(options);
3104
2436
  }
3105
- async count(criteria) {
3106
- if (!criteria || Object.keys(criteria).length === 0) {
3107
- return await this.size();
3108
- }
3109
- this.validateQueryParams(criteria);
3110
- const db = await this.getDb();
3111
- return new Promise((resolve, reject) => {
3112
- const transaction = db.transaction(this.table, "readonly");
3113
- const store = transaction.objectStore(this.table);
3114
- const plan = this.createIndexedRange(store, criteria);
3115
- if (plan?.coversCriteria) {
3116
- const request2 = plan.source.count(plan.range);
3117
- request2.onerror = () => reject(request2.error);
3118
- request2.onsuccess = () => resolve(request2.result);
3119
- return;
3120
- }
3121
- const source = plan?.source ?? store;
3122
- const range = plan?.range;
3123
- let count = 0;
3124
- const request = source.openCursor(range);
3125
- request.onerror = () => reject(request.error);
3126
- request.onsuccess = () => {
3127
- const cursor = request.result;
3128
- if (!cursor) {
3129
- resolve(count);
3130
- return;
3131
- }
3132
- if (this.matchesCriteria(cursor.value, criteria)) {
3133
- count += 1;
3134
- }
3135
- cursor.continue();
3136
- };
3137
- });
2437
+ async size() {
2438
+ return await this.inMemoryRepo.size();
3138
2439
  }
3139
2440
  async getBulk(offset, limit) {
3140
- if (offset < 0) {
3141
- throw new RangeError(`offset must be non-negative, got ${offset}`);
3142
- }
3143
- if (limit <= 0) {
3144
- return;
3145
- }
3146
- const db = await this.getDb();
3147
- return new Promise((resolve, reject) => {
3148
- const transaction = db.transaction(this.table, "readonly");
3149
- const store = transaction.objectStore(this.table);
3150
- const request = store.openCursor();
3151
- const entities = [];
3152
- let skipped = false;
3153
- request.onerror = () => reject(request.error);
3154
- request.onsuccess = () => {
3155
- const cursor = request.result;
3156
- if (cursor) {
3157
- if (!skipped && offset > 0) {
3158
- skipped = true;
3159
- cursor.advance(offset);
3160
- return;
3161
- }
3162
- entities.push(cursor.value);
3163
- if (entities.length === limit) {
3164
- resolve(entities);
3165
- return;
3166
- }
3167
- cursor.continue();
3168
- } else {
3169
- resolve(entities.length > 0 ? entities : undefined);
3170
- }
3171
- };
3172
- });
3173
- }
3174
- matchesCriteria(record, criteria) {
3175
- for (const column of Object.keys(criteria)) {
3176
- const criterion = criteria[column];
3177
- const recordValue = record[column];
3178
- let operator = "=";
3179
- let value;
3180
- if (isSearchCondition(criterion)) {
3181
- operator = criterion.operator;
3182
- value = criterion.value;
3183
- } else {
3184
- value = criterion;
3185
- }
3186
- if (operator !== "=" && (recordValue === null || recordValue === undefined)) {
3187
- return false;
3188
- }
3189
- switch (operator) {
3190
- case "=":
3191
- if (recordValue !== value)
3192
- return false;
3193
- break;
3194
- case "<":
3195
- if (!(recordValue < value))
3196
- return false;
3197
- break;
3198
- case "<=":
3199
- if (!(recordValue <= value))
3200
- return false;
3201
- break;
3202
- case ">":
3203
- if (!(recordValue > value))
3204
- return false;
3205
- break;
3206
- case ">=":
3207
- if (!(recordValue >= value))
3208
- return false;
3209
- break;
3210
- default:
3211
- return false;
3212
- }
3213
- }
3214
- return true;
3215
- }
3216
- async deleteSearch(criteria) {
3217
- const criteriaKeys = Object.keys(criteria);
3218
- if (criteriaKeys.length === 0) {
3219
- return;
3220
- }
3221
- const db = await this.getDb();
3222
- return new Promise(async (resolve, reject) => {
3223
- try {
3224
- const transaction = db.transaction(this.table, "readwrite");
3225
- const store = transaction.objectStore(this.table);
3226
- transaction.oncomplete = () => {
3227
- this.events.emit("delete", criteriaKeys[0]);
3228
- this.hybridManager?.notifyLocalChange();
3229
- resolve();
3230
- };
3231
- transaction.onerror = () => {
3232
- reject(transaction.error);
3233
- };
3234
- const getAllRequest = store.getAll();
3235
- getAllRequest.onsuccess = () => {
3236
- const allRecords = getAllRequest.result;
3237
- const recordsToDelete = allRecords.filter((record) => this.matchesCriteria(record, criteria));
3238
- if (recordsToDelete.length === 0) {
3239
- return;
3240
- }
3241
- for (const record of recordsToDelete) {
3242
- const primaryKey = this.primaryKeyColumns().reduce((key, col) => {
3243
- key[col] = record[col];
3244
- return key;
3245
- }, {});
3246
- const request = store.delete(this.getIndexedKey(primaryKey));
3247
- request.onerror = () => {
3248
- console.error("Error deleting record:", request.error);
3249
- };
3250
- }
3251
- };
3252
- getAllRequest.onerror = () => {
3253
- reject(getAllRequest.error);
3254
- };
3255
- } catch (error) {
3256
- reject(error);
3257
- }
3258
- });
3259
- }
3260
- getEqualityCriterionValue(criteria, column) {
3261
- const criterion = criteria[column];
3262
- if (criterion === undefined)
3263
- return;
3264
- if (isSearchCondition(criterion)) {
3265
- return criterion.operator === "=" ? criterion.value : undefined;
3266
- }
3267
- return criterion;
3268
- }
3269
- compareByOrder(a, b, options) {
3270
- if (!options?.orderBy)
3271
- return 0;
3272
- for (const { column, direction } of options.orderBy) {
3273
- const aVal = a[column];
3274
- const bVal = b[column];
3275
- if (aVal == null && bVal == null)
3276
- continue;
3277
- if (aVal == null)
3278
- return direction === "ASC" ? -1 : 1;
3279
- if (bVal == null)
3280
- return direction === "ASC" ? 1 : -1;
3281
- if (aVal < bVal)
3282
- return direction === "ASC" ? -1 : 1;
3283
- if (aVal > bVal)
3284
- return direction === "ASC" ? 1 : -1;
3285
- }
3286
- return 0;
3287
- }
3288
- createIndexedQuery(store, criteria, options) {
3289
- const orderBy = options?.orderBy ?? [];
3290
- let best;
3291
- for (const indexColumns of this.getCursorSafeIndexes()) {
3292
- const prefixValues = [];
3293
- for (const column of indexColumns) {
3294
- const value = this.getEqualityCriterionValue(criteria, column);
3295
- if (value === undefined)
3296
- break;
3297
- prefixValues.push(value);
3298
- }
3299
- if (prefixValues.length === 0)
3300
- continue;
3301
- const remainingColumns = indexColumns.slice(prefixValues.length);
3302
- let redundantOrderPrefixLength = 0;
3303
- while (redundantOrderPrefixLength < orderBy.length && redundantOrderPrefixLength < prefixValues.length && orderBy[redundantOrderPrefixLength]?.column === indexColumns[redundantOrderPrefixLength]) {
3304
- redundantOrderPrefixLength++;
3305
- }
3306
- const normalizedOrderBy = orderBy.slice(redundantOrderPrefixLength);
3307
- const satisfiesOrder = normalizedOrderBy.length === 0 || normalizedOrderBy.length <= remainingColumns.length && normalizedOrderBy.every((order, index) => order.column === remainingColumns[index]) && orderBy.every((order) => order.direction === orderBy[0]?.direction);
3308
- if (!satisfiesOrder && best)
3309
- continue;
3310
- if (!best || satisfiesOrder && !best.satisfiesOrder || prefixValues.length > best.prefixValues.length) {
3311
- best = {
3312
- indexName: indexColumns.map((column) => String(column)).join("_"),
3313
- prefixValues,
3314
- fullMatch: prefixValues.length === indexColumns.length,
3315
- satisfiesOrder,
3316
- direction: orderBy[0]?.direction === "DESC" ? "prev" : "next"
3317
- };
3318
- }
3319
- }
3320
- const appliedLimit = Boolean(best?.satisfiesOrder && options?.limit !== undefined);
3321
- const appliedOffset = Boolean(best?.satisfiesOrder && options?.offset !== undefined);
3322
- if (!best) {
3323
- return {
3324
- source: store,
3325
- range: undefined,
3326
- direction: orderBy[0]?.direction === "DESC" ? "prev" : "next",
3327
- satisfiesOrder: false,
3328
- appliedLimit: false,
3329
- appliedOffset: false,
3330
- skipRemaining: 0
3331
- };
3332
- }
3333
- const source = store.index(best.indexName);
3334
- const keyRange = best.fullMatch ? IDBKeyRange.only(best.prefixValues.length === 1 ? best.prefixValues[0] : best.prefixValues) : IDBKeyRange.bound(best.prefixValues, [...best.prefixValues, []]);
3335
- return {
3336
- source,
3337
- range: keyRange,
3338
- direction: best.direction,
3339
- satisfiesOrder: best.satisfiesOrder,
3340
- appliedLimit,
3341
- appliedOffset,
3342
- skipRemaining: appliedOffset ? options?.offset ?? 0 : 0
3343
- };
2441
+ return await this.inMemoryRepo.getBulk(offset, limit);
3344
2442
  }
3345
2443
  async query(criteria, options) {
3346
- this.validateQueryParams(criteria, options);
3347
- const db = await this.getDb();
3348
- return new Promise((resolve, reject) => {
3349
- const transaction = db.transaction(this.table, "readonly");
3350
- const store = transaction.objectStore(this.table);
3351
- const indexedQuery = this.createIndexedQuery(store, criteria, options);
3352
- const results = [];
3353
- const request = indexedQuery.source.openCursor(indexedQuery.range, indexedQuery.direction);
3354
- request.onsuccess = () => {
3355
- const cursor = request.result;
3356
- if (!cursor) {
3357
- let finalResults = results;
3358
- if (!indexedQuery.satisfiesOrder && options?.orderBy && options.orderBy.length > 0) {
3359
- finalResults = [...finalResults].sort((a, b) => this.compareByOrder(a, b, options));
3360
- }
3361
- if (!indexedQuery.appliedOffset && options?.offset !== undefined) {
3362
- finalResults = finalResults.slice(options.offset);
3363
- }
3364
- if (!indexedQuery.appliedLimit && options?.limit !== undefined) {
3365
- finalResults = finalResults.slice(0, options.limit);
3366
- }
3367
- const result = finalResults.length > 0 ? finalResults : undefined;
3368
- this.events.emit("query", criteria, result);
3369
- resolve(result);
3370
- return;
3371
- }
3372
- const record = cursor.value;
3373
- if (this.matchesCriteria(record, criteria)) {
3374
- if (indexedQuery.skipRemaining > 0) {
3375
- indexedQuery.skipRemaining -= 1;
3376
- } else {
3377
- results.push(record);
3378
- if (indexedQuery.appliedLimit && results.length === options?.limit) {
3379
- const result = results.length > 0 ? results : undefined;
3380
- this.events.emit("query", criteria, result);
3381
- resolve(result);
3382
- return;
3383
- }
3384
- }
3385
- }
3386
- cursor.continue();
3387
- };
3388
- request.onerror = () => reject(request.error);
3389
- });
2444
+ return await this.inMemoryRepo.query(criteria, options);
3390
2445
  }
3391
2446
  async queryIndex(criteria, options) {
3392
- this.validateSelect(options);
3393
- this.validateQueryParams(criteria, options);
3394
- const registered = this.indexes.map((cols) => {
3395
- const cs = Array.isArray(cols) ? cols : [cols];
3396
- return { name: cs.join("_"), keyPath: cs };
3397
- });
3398
- const picked = pickCoveringIndex({
3399
- table: this.table,
3400
- indexes: registered,
3401
- criteriaColumns: Object.keys(criteria),
3402
- orderByColumns: (options.orderBy ?? []).map((o) => ({
3403
- column: String(o.column),
3404
- direction: o.direction
3405
- })),
3406
- selectColumns: options.select.map(String),
3407
- primaryKeyColumns: this.primaryKeyColumns().map(String)
3408
- });
3409
- const db = await this.getDb();
3410
- return new Promise((resolve, reject) => {
3411
- const tx = db.transaction(this.table, "readonly");
3412
- const store = tx.objectStore(this.table);
3413
- const idx = store.index(picked.name);
3414
- const prefix = [];
3415
- for (const col of picked.keyPath) {
3416
- const c = criteria[col];
3417
- if (c === undefined && !(col in criteria))
3418
- break;
3419
- if (isSearchCondition(c)) {
3420
- if (c.operator !== "=")
3421
- break;
3422
- prefix.push(c.value);
3423
- } else {
3424
- prefix.push(c);
3425
- }
3426
- }
3427
- const range = prefix.length === 0 ? undefined : prefix.length === picked.keyPath.length ? IDBKeyRange.only(prefix.length === 1 ? prefix[0] : prefix) : IDBKeyRange.bound(prefix, [...prefix, []]);
3428
- const direction = picked.reverseDirection ? "prev" : "next";
3429
- const request = idx.openKeyCursor(range, direction);
3430
- const out = [];
3431
- let toSkip = options.offset ?? 0;
3432
- const keyPathPositions = new Map;
3433
- picked.keyPath.forEach((col, i) => keyPathPositions.set(col, i));
3434
- const pkCols = this.primaryKeyColumns().map(String);
3435
- const pkPositions = new Map;
3436
- pkCols.forEach((col, i) => pkPositions.set(col, i));
3437
- request.onsuccess = () => {
3438
- const cursor = request.result;
3439
- if (!cursor) {
3440
- resolve(out);
3441
- return;
3442
- }
3443
- const key = cursor.key;
3444
- const row = {};
3445
- for (const col of options.select) {
3446
- const colStr = String(col);
3447
- const pos = keyPathPositions.get(colStr);
3448
- if (pos !== undefined) {
3449
- row[colStr] = Array.isArray(key) ? key[pos] : key;
3450
- } else {
3451
- if (pkCols.length === 1 && colStr === pkCols[0]) {
3452
- row[colStr] = cursor.primaryKey;
3453
- } else {
3454
- const pkPos = pkPositions.get(colStr);
3455
- if (pkPos !== undefined) {
3456
- row[colStr] = Array.isArray(cursor.primaryKey) ? cursor.primaryKey[pkPos] : cursor.primaryKey;
3457
- }
3458
- }
3459
- }
3460
- }
3461
- let matches = true;
3462
- for (const [col, crit] of Object.entries(criteria)) {
3463
- const pos = keyPathPositions.get(col);
3464
- if (pos === undefined)
3465
- continue;
3466
- if (pos < prefix.length)
3467
- continue;
3468
- const valFromKey = Array.isArray(key) ? key[pos] : key;
3469
- const op = isSearchCondition(crit) ? crit.operator : "=";
3470
- const val = isSearchCondition(crit) ? crit.value : crit;
3471
- if (!compareWithOperator(valFromKey, op, val)) {
3472
- matches = false;
3473
- break;
3474
- }
3475
- }
3476
- if (matches) {
3477
- if (toSkip > 0) {
3478
- toSkip -= 1;
3479
- } else {
3480
- out.push(row);
3481
- if (options.limit !== undefined && out.length >= options.limit) {
3482
- resolve(out);
3483
- return;
3484
- }
3485
- }
3486
- }
3487
- cursor.continue();
3488
- };
3489
- request.onerror = () => reject(request.error);
3490
- });
2447
+ return await this.inMemoryRepo.queryIndex(criteria, options);
3491
2448
  }
3492
- getHybridManager() {
3493
- if (!this.hybridManager) {
3494
- const channelName = `indexeddb-tabular-${this.table}`;
3495
- this.hybridManager = new HybridSubscriptionManager(channelName, async () => {
3496
- const entities = await this.getAll() || [];
3497
- const map = new Map;
3498
- for (const entity of entities) {
3499
- const { key } = this.separateKeyValueFromCombined(entity);
3500
- const fingerprint = await makeFingerprint5(key);
3501
- map.set(fingerprint, entity);
3502
- }
3503
- return map;
3504
- }, compareEntitiesForChange, {
3505
- insert: (item) => ({ type: "INSERT", new: item }),
3506
- update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
3507
- delete: (item) => ({ type: "DELETE", old: item })
3508
- }, {
3509
- defaultIntervalMs: 1000,
3510
- useBroadcastChannel: this.hybridOptions.useBroadcastChannel,
3511
- backupPollingIntervalMs: this.hybridOptions.backupPollingIntervalMs
3512
- });
3513
- }
3514
- return this.hybridManager;
2449
+ async deleteSearch(criteria) {
2450
+ await this.inMemoryRepo.deleteSearch(criteria);
2451
+ this.broadcast({
2452
+ type: "DELETE_SEARCH",
2453
+ criteria
2454
+ });
3515
2455
  }
3516
2456
  subscribeToChanges(callback, options) {
3517
- const intervalMs = options?.pollingIntervalMs ?? 1000;
3518
- const manager = this.getHybridManager();
3519
- return manager.subscribe(callback, { intervalMs });
2457
+ return this.inMemoryRepo.subscribeToChanges(callback, options);
3520
2458
  }
3521
2459
  destroy() {
3522
- if (this.hybridManager) {
3523
- this.hybridManager.destroy();
3524
- this.hybridManager = null;
3525
- }
3526
- this.db?.close();
3527
- }
3528
- }
3529
- function compareWithOperator(a, op, b) {
3530
- const av = a;
3531
- const bv = b;
3532
- switch (op) {
3533
- case "=":
3534
- return av === bv;
3535
- case "<":
3536
- return av !== null && av !== undefined && av < bv;
3537
- case "<=":
3538
- return av !== null && av !== undefined && av <= bv;
3539
- case ">":
3540
- return av !== null && av !== undefined && av > bv;
3541
- case ">=":
3542
- return av !== null && av !== undefined && av >= bv;
3543
- }
3544
- }
3545
- // src/tabular/SharedInMemoryTabularStorage.ts
3546
- import { createServiceToken as createServiceToken13 } from "@workglow/util";
3547
- var SHARED_IN_MEMORY_TABULAR_REPOSITORY = createServiceToken13("storage.tabularRepository.sharedInMemory");
3548
- var SYNC_TIMEOUT = 1000;
3549
- var MAX_PENDING_MESSAGES = 1000;
3550
-
3551
- class SharedInMemoryTabularStorage extends BaseTabularStorage {
3552
- channel = null;
3553
- channelName;
3554
- inMemoryRepo;
3555
- isInitialized = false;
3556
- syncInProgress = false;
3557
- pendingMessages = [];
3558
- constructor(channelName = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
3559
- super(schema, primaryKeyNames, indexes, clientProvidedKeys);
3560
- this.channelName = channelName;
3561
- this.inMemoryRepo = new InMemoryTabularStorage(schema, primaryKeyNames, indexes, clientProvidedKeys);
3562
- this.setupEventForwarding();
3563
- this.initializeBroadcastChannel();
3564
- }
3565
- isBroadcastChannelAvailable() {
3566
- return typeof BroadcastChannel !== "undefined";
3567
- }
3568
- initializeBroadcastChannel() {
3569
- if (!this.isBroadcastChannelAvailable()) {
3570
- console.warn("BroadcastChannel is not available. Tab synchronization will not work.");
3571
- return;
3572
- }
3573
- try {
3574
- this.channel = new BroadcastChannel(this.channelName);
3575
- this.channel.onmessage = (event) => {
3576
- this.handleBroadcastMessage(event.data);
3577
- };
3578
- this.syncFromOtherTabs();
3579
- } catch (error) {
3580
- console.error("Failed to initialize BroadcastChannel:", error);
3581
- }
3582
- }
3583
- setupEventForwarding() {
3584
- this.inMemoryRepo.on("put", (entity) => {
3585
- this.events.emit("put", entity);
3586
- });
3587
- this.inMemoryRepo.on("get", (key, entity) => {
3588
- this.events.emit("get", key, entity);
3589
- });
3590
- this.inMemoryRepo.on("query", (key, entities) => {
3591
- this.events.emit("query", key, entities);
3592
- });
3593
- this.inMemoryRepo.on("delete", (key) => {
3594
- this.events.emit("delete", key);
3595
- });
3596
- this.inMemoryRepo.on("clearall", () => {
3597
- this.events.emit("clearall");
3598
- });
3599
- }
3600
- async handleBroadcastMessage(message) {
3601
- if (this.syncInProgress && message.type !== "SYNC_RESPONSE") {
3602
- if (this.pendingMessages.length < MAX_PENDING_MESSAGES) {
3603
- this.pendingMessages.push(message);
3604
- }
3605
- return;
3606
- }
3607
- switch (message.type) {
3608
- case "SYNC_REQUEST":
3609
- const all = await this.inMemoryRepo.getAll();
3610
- if (this.channel && all) {
3611
- this.channel.postMessage({
3612
- type: "SYNC_RESPONSE",
3613
- data: all
3614
- });
3615
- }
3616
- break;
3617
- case "SYNC_RESPONSE":
3618
- if (message.data && Array.isArray(message.data)) {
3619
- await this.copyDataFromArray(message.data);
3620
- }
3621
- this.syncInProgress = false;
3622
- await this.drainPendingMessages();
3623
- break;
3624
- case "PUT":
3625
- await this.inMemoryRepo.put(message.entity);
3626
- break;
3627
- case "PUT_BULK":
3628
- await this.inMemoryRepo.putBulk(message.entities);
3629
- break;
3630
- case "DELETE":
3631
- await this.inMemoryRepo.delete(message.key);
3632
- break;
3633
- case "DELETE_ALL":
3634
- await this.inMemoryRepo.deleteAll();
3635
- break;
3636
- case "DELETE_SEARCH":
3637
- await this.inMemoryRepo.deleteSearch(message.criteria);
3638
- break;
3639
- }
3640
- }
3641
- async drainPendingMessages() {
3642
- while (!this.syncInProgress && this.pendingMessages.length > 0) {
3643
- const messages = this.pendingMessages;
3644
- this.pendingMessages = [];
3645
- for (const message of messages) {
3646
- await this.handleBroadcastMessage(message);
3647
- if (this.syncInProgress) {
3648
- break;
3649
- }
3650
- }
3651
- }
3652
- }
3653
- syncFromOtherTabs() {
3654
- if (!this.channel)
3655
- return;
3656
- this.syncInProgress = true;
3657
- this.channel.postMessage({ type: "SYNC_REQUEST" });
3658
- setTimeout(() => {
3659
- if (this.syncInProgress) {
3660
- this.syncInProgress = false;
3661
- this.drainPendingMessages().catch((error) => {
3662
- console.error("Failed to drain pending messages after sync timeout", error);
3663
- });
3664
- }
3665
- }, SYNC_TIMEOUT);
3666
- }
3667
- async copyDataFromArray(entities) {
3668
- if (entities.length === 0)
3669
- return;
3670
- await this.inMemoryRepo.deleteAll();
3671
- await this.inMemoryRepo.putBulk(entities);
3672
- }
3673
- broadcast(message) {
3674
- if (this.channel) {
3675
- this.channel.postMessage(message);
3676
- }
3677
- }
3678
- async setupDatabase() {
3679
- if (this.isInitialized)
3680
- return;
3681
- this.isInitialized = true;
3682
- await this.syncFromOtherTabs();
3683
- }
3684
- async put(value) {
3685
- const result = await this.inMemoryRepo.put(value);
3686
- this.broadcast({ type: "PUT", entity: result });
3687
- return result;
3688
- }
3689
- async putBulk(values) {
3690
- const result = await this.inMemoryRepo.putBulk(values);
3691
- this.broadcast({ type: "PUT_BULK", entities: result });
3692
- return result;
3693
- }
3694
- async get(key) {
3695
- return await this.inMemoryRepo.get(key);
3696
- }
3697
- async delete(value) {
3698
- await this.inMemoryRepo.delete(value);
3699
- const { key } = this.separateKeyValueFromCombined(value);
3700
- this.broadcast({ type: "DELETE", key });
3701
- }
3702
- async deleteAll() {
3703
- await this.inMemoryRepo.deleteAll();
3704
- this.broadcast({ type: "DELETE_ALL" });
3705
- }
3706
- async getAll(options) {
3707
- return await this.inMemoryRepo.getAll(options);
3708
- }
3709
- async size() {
3710
- return await this.inMemoryRepo.size();
3711
- }
3712
- async getBulk(offset, limit) {
3713
- return await this.inMemoryRepo.getBulk(offset, limit);
3714
- }
3715
- async query(criteria, options) {
3716
- return await this.inMemoryRepo.query(criteria, options);
3717
- }
3718
- async queryIndex(criteria, options) {
3719
- return await this.inMemoryRepo.queryIndex(criteria, options);
3720
- }
3721
- async deleteSearch(criteria) {
3722
- await this.inMemoryRepo.deleteSearch(criteria);
3723
- this.broadcast({
3724
- type: "DELETE_SEARCH",
3725
- criteria
3726
- });
3727
- }
3728
- subscribeToChanges(callback, options) {
3729
- return this.inMemoryRepo.subscribeToChanges(callback, options);
3730
- }
3731
- destroy() {
3732
- if (this.channel) {
3733
- this.channel.close();
3734
- this.channel = null;
2460
+ if (this.channel) {
2461
+ this.channel.close();
2462
+ this.channel = null;
3735
2463
  }
3736
2464
  this.inMemoryRepo.destroy();
3737
2465
  }
3738
2466
  }
3739
- // src/tabular/SupabaseTabularStorage.ts
3740
- import { createServiceToken as createServiceToken14 } from "@workglow/util";
3741
-
3742
- // src/tabular/BaseSqlTabularStorage.ts
3743
- class BaseSqlTabularStorage extends BaseTabularStorage {
3744
- table;
3745
- _pkColsCache = new Map;
3746
- _valColsCache = new Map;
3747
- _pkColListCache = new Map;
3748
- _valColListCache = new Map;
3749
- constructor(table = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
3750
- super(schema, primaryKeyNames, indexes, clientProvidedKeys);
3751
- this.table = table;
3752
- this.validateTableAndSchema();
3753
- }
3754
- constructPrimaryKeyColumns($delimiter = "") {
3755
- let cached = this._pkColsCache.get($delimiter);
3756
- if (cached === undefined) {
3757
- cached = Object.entries(this.primaryKeySchema.properties).map(([key, typeDef]) => {
3758
- const sqlType = this.mapTypeToSQL(typeDef);
3759
- return `${$delimiter}${key}${$delimiter} ${sqlType} NOT NULL`;
3760
- }).join(", ");
3761
- this._pkColsCache.set($delimiter, cached);
3762
- }
3763
- return cached;
3764
- }
3765
- constructValueColumns($delimiter = "") {
3766
- let cached = this._valColsCache.get($delimiter);
3767
- if (cached === undefined) {
3768
- const requiredSet = new Set(this.valueSchema.required ?? []);
3769
- const cols = Object.entries(this.valueSchema.properties).map(([key, typeDef]) => {
3770
- const sqlType = this.mapTypeToSQL(typeDef);
3771
- const isRequired = requiredSet.has(key);
3772
- const nullable = !isRequired || this.isNullable(typeDef);
3773
- return `${$delimiter}${key}${$delimiter} ${sqlType}${nullable ? " NULL" : " NOT NULL"}`;
3774
- }).join(", ");
3775
- cached = cols.length > 0 ? `, ${cols}` : "";
3776
- this._valColsCache.set($delimiter, cached);
3777
- }
3778
- return cached;
3779
- }
3780
- isNullable(typeDef) {
3781
- if (typeof typeDef === "boolean")
3782
- return typeDef;
3783
- if (typeDef.type === "null") {
3784
- return true;
3785
- }
3786
- if (Array.isArray(typeDef.type)) {
3787
- return typeDef.type.includes("null");
3788
- }
3789
- if (typeDef.anyOf && Array.isArray(typeDef.anyOf)) {
3790
- return typeDef.anyOf.some((type) => type.type === "null");
3791
- }
3792
- if (typeDef.oneOf && Array.isArray(typeDef.oneOf)) {
3793
- return typeDef.oneOf.some((type) => type.type === "null");
3794
- }
3795
- return false;
3796
- }
3797
- primaryKeyColumnList($delimiter = "") {
3798
- let cached = this._pkColListCache.get($delimiter);
3799
- if (cached === undefined) {
3800
- cached = $delimiter + this.primaryKeyColumns().join(`${$delimiter}, ${$delimiter}`) + $delimiter;
3801
- this._pkColListCache.set($delimiter, cached);
3802
- }
3803
- return cached;
3804
- }
3805
- valueColumnList($delimiter = "") {
3806
- let cached = this._valColListCache.get($delimiter);
3807
- if (cached === undefined) {
3808
- cached = $delimiter + this.valueColumns().join(`${$delimiter}, ${$delimiter}`) + $delimiter;
3809
- this._valColListCache.set($delimiter, cached);
3810
- }
3811
- return cached;
3812
- }
3813
- getNonNullType(typeDef) {
3814
- if (typeof typeDef === "boolean")
3815
- return typeDef;
3816
- if (typeDef.anyOf && Array.isArray(typeDef.anyOf)) {
3817
- const nonNullType = typeDef.anyOf.find((t) => t.type !== "null");
3818
- if (nonNullType) {
3819
- return nonNullType;
3820
- }
3821
- }
3822
- if (typeDef.oneOf && Array.isArray(typeDef.oneOf)) {
3823
- const nonNullType = typeDef.oneOf.find((t) => t.type !== "null");
3824
- if (nonNullType) {
3825
- return nonNullType;
3826
- }
3827
- }
3828
- return typeDef;
3829
- }
3830
- getValueAsOrderedArray(value) {
3831
- const orderedParams = [];
3832
- const valueAsRecord = value;
3833
- const requiredSet = new Set(this.valueSchema.required ?? []);
3834
- for (const key in this.valueSchema.properties) {
3835
- if (Object.prototype.hasOwnProperty.call(valueAsRecord, key)) {
3836
- const val = valueAsRecord[key];
3837
- if (val === undefined && !requiredSet.has(key)) {
3838
- orderedParams.push(null);
3839
- } else {
3840
- orderedParams.push(this.jsToSqlValue(key, val));
3841
- }
3842
- } else {
3843
- if (requiredSet.has(key)) {
3844
- throw new Error(`Missing required value field: ${key}`);
3845
- }
3846
- orderedParams.push(null);
3847
- }
3848
- }
3849
- return orderedParams;
3850
- }
3851
- getPrimaryKeyAsOrderedArray(key) {
3852
- const orderedParams = [];
3853
- const keyObj = key;
3854
- for (const k of Object.keys(this.primaryKeySchema.properties)) {
3855
- if (k in keyObj) {
3856
- const value = keyObj[k];
3857
- if (value === null) {
3858
- throw new Error(`Primary key field ${k} cannot be null`);
3859
- }
3860
- orderedParams.push(this.jsToSqlValue(k, value));
3861
- } else {
3862
- throw new Error(`Missing required primary key field: ${k}`);
3863
- }
3864
- }
3865
- return orderedParams;
3866
- }
3867
- jsToSqlValue(column, value) {
3868
- const typeDef = this.schema.properties[column];
3869
- if (!typeDef) {
3870
- return value;
3871
- }
3872
- if (value === null && this.isNullable(typeDef)) {
3873
- return null;
3874
- }
3875
- const actualType = this.getNonNullType(typeDef);
3876
- if (typeof actualType === "boolean") {
3877
- return value;
3878
- }
3879
- if (actualType.contentEncoding === "blob") {
3880
- if (value instanceof Uint8Array) {
3881
- return value;
3882
- }
3883
- if (typeof Buffer !== "undefined" && value instanceof Buffer) {
3884
- return new Uint8Array(value);
3885
- }
3886
- if (Array.isArray(value)) {
3887
- return new Uint8Array(value);
3888
- }
3889
- return value;
3890
- } else if (value instanceof Date) {
3891
- return value.toISOString();
3892
- } else {
3893
- return value;
3894
- }
3895
- }
3896
- sqlToJsValue(column, value) {
3897
- const typeDef = this.schema.properties[column];
3898
- if (!typeDef) {
3899
- return value;
3900
- }
3901
- if (value === null && this.isNullable(typeDef)) {
3902
- return null;
3903
- }
3904
- const actualType = this.getNonNullType(typeDef);
3905
- if (typeof actualType === "boolean") {
3906
- return value;
3907
- }
3908
- if (actualType.contentEncoding === "blob") {
3909
- if (typeof Buffer !== "undefined" && value instanceof Buffer) {
3910
- return new Uint8Array(value);
3911
- }
3912
- if (value instanceof Uint8Array) {
3913
- return value;
3914
- }
3915
- return value;
3916
- } else {
3917
- return value;
3918
- }
3919
- }
3920
- validateTableAndSchema() {
3921
- if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(this.table)) {
3922
- throw new Error("Table name must start with a letter and contain only letters, digits, and underscores, got: " + this.table);
3923
- }
3924
- const validateSchemaKeys = (schema) => {
3925
- for (const key in schema.properties) {
3926
- if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(key)) {
3927
- throw new Error("Schema keys must start with a letter and contain only letters, digits, and underscores, got: " + key);
3928
- }
3929
- }
3930
- };
3931
- validateSchemaKeys(this.primaryKeySchema);
3932
- validateSchemaKeys(this.valueSchema);
3933
- const primaryKeys = new Set(Object.keys(this.primaryKeySchema.properties));
3934
- const valueKeys = Object.keys(this.valueSchema.properties);
3935
- const duplicates = valueKeys.filter((key) => primaryKeys.has(key));
3936
- if (duplicates.length > 0) {
3937
- throw new Error(`Duplicate keys found in schemas: ${duplicates.join(", ")}`);
3938
- }
3939
- }
3940
- }
3941
-
3942
- // src/tabular/SupabaseTabularStorage.ts
3943
- var SUPABASE_TABULAR_REPOSITORY = createServiceToken14("storage.tabularRepository.supabase");
3944
-
3945
- class SupabaseTabularStorage extends BaseSqlTabularStorage {
3946
- client;
3947
- realtimeChannel = null;
3948
- constructor(client, table = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
3949
- super(table, schema, primaryKeyNames, indexes, clientProvidedKeys);
3950
- this.client = client;
3951
- }
3952
- async setupDatabase() {
3953
- const sql = `
3954
- CREATE TABLE IF NOT EXISTS "${this.table}" (
3955
- ${this.constructPrimaryKeyColumns('"')} ${this.constructValueColumns('"')},
3956
- PRIMARY KEY (${this.primaryKeyColumnList()})
3957
- )
3958
- `;
3959
- const { error } = await this.client.rpc("exec_sql", { query: sql });
3960
- if (error && !error.message.includes("already exists")) {
3961
- throw error;
3962
- }
3963
- const pkColumns = this.primaryKeyColumns();
3964
- const createdIndexes = new Set;
3965
- for (const columns of this.indexes) {
3966
- if (columns.length <= pkColumns.length) {
3967
- const isPkPrefix = columns.every((col, idx) => col === pkColumns[idx]);
3968
- if (isPkPrefix)
3969
- continue;
3970
- }
3971
- const indexName = `${this.table}_${columns.join("_")}`;
3972
- const columnList = columns.map((col) => `"${String(col)}"`).join(", ");
3973
- const columnKey = columns.join(",");
3974
- if (createdIndexes.has(columnKey))
3975
- continue;
3976
- const isRedundant = Array.from(createdIndexes).some((existing) => {
3977
- const existingCols = existing.split(",");
3978
- return existingCols.length >= columns.length && columns.every((col, idx) => col === existingCols[idx]);
3979
- });
3980
- if (!isRedundant) {
3981
- const indexSql = `CREATE INDEX IF NOT EXISTS "${indexName}" ON "${this.table}" (${columnList})`;
3982
- const { error: indexError } = await this.client.rpc("exec_sql", { query: indexSql });
3983
- if (indexError && !indexError.message.includes("already exists")) {
3984
- console.warn(`Failed to create index ${indexName}:`, indexError);
3985
- }
3986
- createdIndexes.add(columnKey);
3987
- }
3988
- }
3989
- }
3990
- mapTypeToSQL(typeDef) {
3991
- const actualType = this.getNonNullType(typeDef);
3992
- if (typeof actualType === "boolean") {
3993
- return "TEXT /* boolean schema */";
3994
- }
3995
- if (actualType.contentEncoding === "blob")
3996
- return "BYTEA";
3997
- switch (actualType.type) {
3998
- case "string":
3999
- if (actualType.format === "date-time")
4000
- return "TIMESTAMP";
4001
- if (actualType.format === "date")
4002
- return "DATE";
4003
- if (actualType.format === "email")
4004
- return "VARCHAR(255)";
4005
- if (actualType.format === "uri")
4006
- return "VARCHAR(2048)";
4007
- if (actualType.format === "uuid")
4008
- return "UUID";
4009
- if (typeof actualType.maxLength === "number") {
4010
- return `VARCHAR(${actualType.maxLength})`;
4011
- }
4012
- return "TEXT";
4013
- case "number":
4014
- case "integer":
4015
- if (actualType.multipleOf === 1 || actualType.type === "integer") {
4016
- if (typeof actualType.minimum === "number") {
4017
- if (actualType.minimum >= 0) {
4018
- if (typeof actualType.maximum === "number") {
4019
- if (actualType.maximum <= 32767)
4020
- return "SMALLINT";
4021
- if (actualType.maximum <= 2147483647)
4022
- return "INTEGER";
4023
- }
4024
- return "BIGINT";
4025
- }
4026
- }
4027
- return "INTEGER";
4028
- }
4029
- if (actualType.format === "float")
4030
- return "REAL";
4031
- if (actualType.format === "double")
4032
- return "DOUBLE PRECISION";
4033
- if (typeof actualType.multipleOf === "number") {
4034
- const decimalPlaces = String(actualType.multipleOf).split(".")[1]?.length || 0;
4035
- if (decimalPlaces > 0) {
4036
- return `NUMERIC(38, ${decimalPlaces})`;
4037
- }
4038
- }
4039
- return "NUMERIC";
4040
- case "boolean":
4041
- return "BOOLEAN";
4042
- case "array":
4043
- if (actualType.items && typeof actualType.items === "object" && !Array.isArray(actualType.items)) {
4044
- const itemType = this.mapTypeToSQL(actualType.items);
4045
- const supportedArrayElementTypes = [
4046
- "TEXT",
4047
- "VARCHAR",
4048
- "CHAR",
4049
- "INTEGER",
4050
- "SMALLINT",
4051
- "BIGINT",
4052
- "REAL",
4053
- "DOUBLE PRECISION",
4054
- "NUMERIC",
4055
- "BOOLEAN",
4056
- "UUID",
4057
- "DATE",
4058
- "TIMESTAMP"
4059
- ];
4060
- const isSupported = supportedArrayElementTypes.some((type) => itemType === type || itemType.startsWith(type + "(") && type !== "VARCHAR");
4061
- if (isSupported) {
4062
- return `${itemType}[]`;
4063
- } else {
4064
- return "JSONB /* complex array */";
4065
- }
4066
- }
4067
- return "JSONB /* generic array */";
4068
- case "object":
4069
- return "JSONB /* object */";
4070
- default:
4071
- return "TEXT /* unknown type */";
4072
- }
4073
- }
4074
- constructPrimaryKeyColumns($delimiter = "") {
4075
- const cols = Object.entries(this.primaryKeySchema.properties).map(([key, typeDef]) => {
4076
- if (this.isAutoGeneratedKey(key)) {
4077
- if (this.autoGeneratedKeyStrategy === "autoincrement") {
4078
- const sqlType2 = this.mapTypeToSQL(typeDef);
4079
- const isSmallInt = sqlType2.includes("SMALLINT");
4080
- const isBigInt = sqlType2.includes("BIGINT");
4081
- const serialType = isBigInt ? "BIGSERIAL" : isSmallInt ? "SMALLSERIAL" : "SERIAL";
4082
- return `${$delimiter}${key}${$delimiter} ${serialType}`;
4083
- } else if (this.autoGeneratedKeyStrategy === "uuid") {
4084
- return `${$delimiter}${key}${$delimiter} UUID DEFAULT gen_random_uuid()`;
4085
- }
4086
- }
4087
- const sqlType = this.mapTypeToSQL(typeDef);
4088
- let constraints = "NOT NULL";
4089
- if (this.shouldBeUnsigned(typeDef)) {
4090
- constraints += ` CHECK (${$delimiter}${key}${$delimiter} >= 0)`;
4091
- }
4092
- return `${$delimiter}${key}${$delimiter} ${sqlType} ${constraints}`;
4093
- }).join(", ");
4094
- return cols;
4095
- }
4096
- constructValueColumns($delimiter = "") {
4097
- const delimiter = $delimiter || '"';
4098
- const requiredSet = new Set(this.valueSchema.required ?? []);
4099
- const cols = Object.entries(this.valueSchema.properties).map(([key, typeDef]) => {
4100
- const sqlType = this.mapTypeToSQL(typeDef);
4101
- const isRequired = requiredSet.has(key);
4102
- const nullable = !isRequired || this.isNullable(typeDef);
4103
- let constraints = nullable ? "NULL" : "NOT NULL";
4104
- if (this.shouldBeUnsigned(typeDef)) {
4105
- constraints += ` CHECK (${delimiter}${key}${delimiter} >= 0)`;
4106
- }
4107
- return `${delimiter}${key}${delimiter} ${sqlType} ${constraints}`;
4108
- }).join(", ");
4109
- if (cols.length > 0) {
4110
- return `, ${cols}`;
4111
- } else {
4112
- return "";
4113
- }
4114
- }
4115
- sqlToJsValue(column, value) {
4116
- const typeDef = this.schema.properties[column];
4117
- if (typeDef) {
4118
- if (value === null && this.isNullable(typeDef)) {
4119
- return null;
4120
- }
4121
- const actualType = this.getNonNullType(typeDef);
4122
- if (typeof actualType !== "boolean" && (actualType.type === "number" || actualType.type === "integer")) {
4123
- if (typeof value === "number")
4124
- return value;
4125
- if (typeof value === "string") {
4126
- const parsed = Number(value);
4127
- if (!isNaN(parsed))
4128
- return parsed;
4129
- }
4130
- }
4131
- }
4132
- return super.sqlToJsValue(column, value);
4133
- }
4134
- shouldBeUnsigned(typeDef) {
4135
- const actualType = this.getNonNullType(typeDef);
4136
- if (typeof actualType === "boolean") {
4137
- return false;
4138
- }
4139
- if ((actualType.type === "number" || actualType.type === "integer") && typeof actualType.minimum === "number" && actualType.minimum >= 0) {
4140
- return true;
4141
- }
4142
- return false;
4143
- }
4144
- async put(entity) {
4145
- let entityToInsert = { ...entity };
4146
- if (this.hasAutoGeneratedKey() && this.autoGeneratedKeyName) {
4147
- const keyName = String(this.autoGeneratedKeyName);
4148
- const entityRecord = entity;
4149
- const clientProvidedValue = entityRecord[keyName];
4150
- const hasClientValue = clientProvidedValue !== undefined && clientProvidedValue !== null;
4151
- let shouldOmitKey = false;
4152
- if (this.clientProvidedKeys === "never") {
4153
- shouldOmitKey = true;
4154
- } else if (this.clientProvidedKeys === "always") {
4155
- if (!hasClientValue) {
4156
- throw new Error(`Auto-generated key "${keyName}" is required when clientProvidedKeys is "always"`);
4157
- }
4158
- shouldOmitKey = false;
4159
- } else {
4160
- shouldOmitKey = !hasClientValue;
4161
- }
4162
- if (shouldOmitKey) {
4163
- delete entityToInsert[keyName];
4164
- }
4165
- }
4166
- const normalizedEntity = { ...entityToInsert };
4167
- const requiredSet = new Set(this.valueSchema.required ?? []);
4168
- for (const key in this.valueSchema.properties) {
4169
- if (!(key in normalizedEntity) || normalizedEntity[key] === undefined) {
4170
- if (!requiredSet.has(key)) {
4171
- normalizedEntity[key] = null;
4172
- }
4173
- }
4174
- }
4175
- const { data, error } = await this.client.from(this.table).upsert(normalizedEntity, { onConflict: this.primaryKeyColumnList() }).select().single();
4176
- if (error)
4177
- throw error;
4178
- const updatedEntity = data;
4179
- const updatedRecord = updatedEntity;
4180
- for (const key in this.schema.properties) {
4181
- updatedRecord[key] = this.sqlToJsValue(key, updatedRecord[key]);
4182
- }
4183
- this.events.emit("put", updatedEntity);
4184
- return updatedEntity;
4185
- }
4186
- async putBulk(entities) {
4187
- if (entities.length === 0)
4188
- return [];
4189
- return await Promise.all(entities.map((entity) => this.put(entity)));
4190
- }
4191
- async get(key) {
4192
- let query = this.client.from(this.table).select("*");
4193
- const keyRecord = key;
4194
- for (const pkName of this.primaryKeyNames) {
4195
- query = query.eq(String(pkName), keyRecord[String(pkName)]);
4196
- }
4197
- const { data, error } = await query.single();
4198
- if (error) {
4199
- if (error.code === "PGRST116") {
4200
- this.events.emit("get", key, undefined);
4201
- return;
4202
- }
4203
- throw error;
4204
- }
4205
- const val = data;
4206
- if (val) {
4207
- const valRecord = val;
4208
- for (const key2 in this.schema.properties) {
4209
- valRecord[key2] = this.sqlToJsValue(key2, valRecord[key2]);
4210
- }
4211
- }
4212
- this.events.emit("get", key, val);
4213
- return val;
4214
- }
4215
- async delete(value) {
4216
- const { key } = this.separateKeyValueFromCombined(value);
4217
- let query = this.client.from(this.table).delete();
4218
- const deleteKeyRecord = key;
4219
- for (const pkName of this.primaryKeyNames) {
4220
- query = query.eq(String(pkName), deleteKeyRecord[String(pkName)]);
4221
- }
4222
- const { error } = await query;
4223
- if (error)
4224
- throw error;
4225
- this.events.emit("delete", key);
4226
- }
4227
- async getAll(options) {
4228
- this.validateGetAllOptions(options);
4229
- let query = this.client.from(this.table).select("*");
4230
- if (options?.orderBy) {
4231
- for (const { column, direction } of options.orderBy) {
4232
- query = query.order(String(column), { ascending: direction === "ASC" });
4233
- }
4234
- }
4235
- if (options?.offset !== undefined || options?.limit !== undefined) {
4236
- const start = options?.offset ?? 0;
4237
- if (options?.limit !== undefined) {
4238
- query = query.range(start, start + options.limit - 1);
4239
- } else if (options?.offset !== undefined) {
4240
- query = query.range(start, start + 999999);
4241
- }
4242
- }
4243
- const { data, error } = await query;
4244
- if (error)
4245
- throw error;
4246
- if (data && data.length) {
4247
- for (const row of data) {
4248
- const record = row;
4249
- for (const key in this.schema.properties) {
4250
- record[key] = this.sqlToJsValue(key, record[key]);
4251
- }
4252
- }
4253
- return data;
4254
- }
4255
- return;
4256
- }
4257
- async deleteAll() {
4258
- const firstPkColumn = this.primaryKeyNames[0];
4259
- const { error } = await this.client.from(this.table).delete().neq(String(firstPkColumn), null);
4260
- if (error)
4261
- throw error;
4262
- this.events.emit("clearall");
4263
- }
4264
- async size() {
4265
- const { count, error } = await this.client.from(this.table).select("*", { count: "exact", head: true });
4266
- if (error)
4267
- throw error;
4268
- return count ?? 0;
4269
- }
4270
- async getBulk(offset, limit) {
4271
- let query = this.client.from(this.table).select("*");
4272
- for (const pkName of this.primaryKeyNames) {
4273
- query = query.order(String(pkName));
4274
- }
4275
- const { data, error } = await query.range(offset, offset + limit - 1);
4276
- if (error)
4277
- throw error;
4278
- if (!data || data.length === 0) {
4279
- return;
4280
- }
4281
- for (const row of data) {
4282
- const record = row;
4283
- for (const key in this.schema.properties) {
4284
- record[key] = this.sqlToJsValue(key, record[key]);
4285
- }
4286
- }
4287
- return data;
4288
- }
4289
- applyCriteriaToFilter(query, criteria) {
4290
- let q = query;
4291
- for (const column of Object.keys(criteria)) {
4292
- const criterion = criteria[column];
4293
- let operator = "=";
4294
- let value;
4295
- if (isSearchCondition(criterion)) {
4296
- operator = criterion.operator;
4297
- value = criterion.value;
4298
- } else {
4299
- value = criterion;
4300
- }
4301
- switch (operator) {
4302
- case "=":
4303
- q = q.eq(String(column), value);
4304
- break;
4305
- case "<":
4306
- q = q.lt(String(column), value);
4307
- break;
4308
- case "<=":
4309
- q = q.lte(String(column), value);
4310
- break;
4311
- case ">":
4312
- q = q.gt(String(column), value);
4313
- break;
4314
- case ">=":
4315
- q = q.gte(String(column), value);
4316
- break;
4317
- }
4318
- }
4319
- return q;
4320
- }
4321
- async count(criteria) {
4322
- if (!criteria || Object.keys(criteria).length === 0) {
4323
- return await this.size();
4324
- }
4325
- this.validateQueryParams(criteria);
4326
- const query = this.applyCriteriaToFilter(this.client.from(this.table).select("*", { count: "exact", head: true }), criteria);
4327
- const { count, error } = await query;
4328
- if (error)
4329
- throw error;
4330
- return count ?? 0;
4331
- }
4332
- async deleteSearch(criteria) {
4333
- const criteriaKeys = Object.keys(criteria);
4334
- if (criteriaKeys.length === 0) {
4335
- return;
4336
- }
4337
- let query = this.client.from(this.table).delete();
4338
- for (const column of criteriaKeys) {
4339
- if (!(column in this.schema.properties)) {
4340
- throw new Error(`Schema must have a ${String(column)} field to use deleteSearch`);
4341
- }
4342
- const criterion = criteria[column];
4343
- let operator = "=";
4344
- let value;
4345
- if (isSearchCondition(criterion)) {
4346
- operator = criterion.operator;
4347
- value = criterion.value;
4348
- } else {
4349
- value = criterion;
4350
- }
4351
- switch (operator) {
4352
- case "=":
4353
- query = query.eq(String(column), value);
4354
- break;
4355
- case "<":
4356
- query = query.lt(String(column), value);
4357
- break;
4358
- case "<=":
4359
- query = query.lte(String(column), value);
4360
- break;
4361
- case ">":
4362
- query = query.gt(String(column), value);
4363
- break;
4364
- case ">=":
4365
- query = query.gte(String(column), value);
4366
- break;
4367
- }
4368
- }
4369
- const { error } = await query;
4370
- if (error)
4371
- throw error;
4372
- this.events.emit("delete", criteriaKeys[0]);
4373
- }
4374
- async query(criteria, options) {
4375
- this.validateQueryParams(criteria, options);
4376
- let query = this.applyCriteriaToFilter(this.client.from(this.table).select("*"), criteria);
4377
- if (options?.orderBy) {
4378
- for (const { column, direction } of options.orderBy) {
4379
- query = query.order(String(column), { ascending: direction === "ASC" });
4380
- }
4381
- }
4382
- if (options?.offset !== undefined || options?.limit !== undefined) {
4383
- const start = options?.offset ?? 0;
4384
- if (options?.limit !== undefined) {
4385
- query = query.range(start, start + options.limit - 1);
4386
- } else if (options?.offset !== undefined) {
4387
- query = query.range(start, start + 999999);
4388
- }
4389
- } else if (options?.limit !== undefined) {
4390
- query = query.limit(options.limit);
4391
- }
4392
- const { data, error } = await query;
4393
- if (error)
4394
- throw error;
4395
- if (data && data.length > 0) {
4396
- for (const row of data) {
4397
- const record = row;
4398
- for (const key in this.schema.properties) {
4399
- record[key] = this.sqlToJsValue(key, record[key]);
4400
- }
4401
- }
4402
- this.events.emit("query", criteria, data);
4403
- return data;
4404
- }
4405
- this.events.emit("query", criteria, undefined);
4406
- return;
4407
- }
4408
- async queryIndex(criteria, options) {
4409
- this.validateSelect(options);
4410
- this.validateQueryParams(criteria, options);
4411
- const registered = this.indexes.map((cols, i) => {
4412
- const cs = Array.isArray(cols) ? cols : [cols];
4413
- return { name: `idx_${i}`, keyPath: cs };
4414
- });
4415
- pickCoveringIndex({
4416
- table: this.table,
4417
- indexes: registered,
4418
- criteriaColumns: Object.keys(criteria),
4419
- orderByColumns: (options.orderBy ?? []).map((o) => ({
4420
- column: String(o.column),
4421
- direction: o.direction
4422
- })),
4423
- selectColumns: options.select.map(String),
4424
- primaryKeyColumns: this.primaryKeyNames.map(String)
4425
- });
4426
- const colList = options.select.map(String).join(",");
4427
- let q = this.applyCriteriaToFilter(this.client.from(this.table).select(colList), criteria);
4428
- if (options.orderBy) {
4429
- for (const { column, direction } of options.orderBy) {
4430
- q = q.order(String(column), { ascending: direction === "ASC" });
4431
- }
4432
- }
4433
- if (options.offset !== undefined && options.limit === undefined) {
4434
- throw new StorageValidationError("queryIndex with offset requires limit (no implicit cap)");
4435
- }
4436
- if (options.offset !== undefined || options.limit !== undefined) {
4437
- const start = options.offset ?? 0;
4438
- if (options.limit !== undefined) {
4439
- q = q.range(start, start + options.limit - 1);
4440
- }
4441
- }
4442
- const { data, error } = await q;
4443
- if (error)
4444
- throw error;
4445
- if (!data)
4446
- return [];
4447
- const rows = data;
4448
- const sel = new Set(options.select.map(String));
4449
- for (const row of rows) {
4450
- for (const key of Object.keys(row)) {
4451
- if (sel.has(key)) {
4452
- row[key] = this.sqlToJsValue(key, row[key]);
4453
- }
4454
- }
4455
- }
4456
- return rows;
4457
- }
4458
- convertRealtimeRow(row) {
4459
- const entity = { ...row };
4460
- const record = entity;
4461
- for (const key in this.schema.properties) {
4462
- record[key] = this.sqlToJsValue(key, row[key]);
4463
- }
4464
- return entity;
4465
- }
4466
- subscribeToChanges(callback, options) {
4467
- const channelName = `tabular-${this.table}-${Date.now()}`;
4468
- this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
4469
- event: "*",
4470
- schema: "public",
4471
- table: this.table
4472
- }, (payload) => {
4473
- const change = {
4474
- type: payload.eventType.toUpperCase(),
4475
- old: payload.old && Object.keys(payload.old).length > 0 ? this.convertRealtimeRow(payload.old) : undefined,
4476
- new: payload.new && Object.keys(payload.new).length > 0 ? this.convertRealtimeRow(payload.new) : undefined
4477
- };
4478
- callback(change);
4479
- }).subscribe();
4480
- return () => {
4481
- if (this.realtimeChannel) {
4482
- this.client.removeChannel(this.realtimeChannel);
4483
- this.realtimeChannel = null;
4484
- }
4485
- };
4486
- }
4487
- destroy() {
4488
- if (this.realtimeChannel) {
4489
- this.client.removeChannel(this.realtimeChannel);
4490
- this.realtimeChannel = null;
4491
- }
4492
- }
4493
- }
4494
- // src/kv/IndexedDbKvStorage.ts
4495
- import { createServiceToken as createServiceToken15 } from "@workglow/util";
4496
- var IDB_KV_REPOSITORY = createServiceToken15("storage.kvRepository.indexedDb");
4497
-
4498
- class IndexedDbKvStorage extends KvViaTabularStorage {
4499
- dbName;
4500
- tabularRepository;
4501
- constructor(dbName, keySchema = { type: "string" }, valueSchema = {}) {
4502
- super(keySchema, valueSchema);
4503
- this.dbName = dbName;
4504
- this.tabularRepository = new IndexedDbTabularStorage(dbName, DefaultKeyValueSchema, DefaultKeyValueKey);
4505
- }
4506
- }
4507
- // src/kv/SupabaseKvStorage.ts
4508
- import { createServiceToken as createServiceToken16 } from "@workglow/util";
4509
- var SUPABASE_KV_REPOSITORY = createServiceToken16("storage.kvRepository.supabase");
4510
-
4511
- class SupabaseKvStorage extends KvViaTabularStorage {
4512
- client;
4513
- tableName;
4514
- tabularRepository;
4515
- constructor(client, tableName, keySchema = { type: "string" }, valueSchema = {}, tabularRepository) {
4516
- super(keySchema, valueSchema);
4517
- this.client = client;
4518
- this.tableName = tableName;
4519
- this.tabularRepository = tabularRepository ?? new SupabaseTabularStorage(client, tableName, DefaultKeyValueSchema, DefaultKeyValueKey);
4520
- }
4521
- }
4522
- // src/queue/IndexedDbQueueStorage.ts
4523
- import { createServiceToken as createServiceToken17, deepEqual as deepEqual3, makeFingerprint as makeFingerprint6, uuid4 as uuid45 } from "@workglow/util";
4524
- var INDEXED_DB_QUEUE_STORAGE = createServiceToken17("jobqueue.storage.indexedDb");
4525
-
4526
- class IndexedDbQueueStorage {
4527
- queueName;
4528
- scope = "process";
4529
- db;
4530
- tableName;
4531
- migrationOptions;
4532
- prefixes;
4533
- prefixValues;
4534
- hybridManager = null;
4535
- hybridOptions;
4536
- constructor(queueName, options = {}) {
4537
- this.queueName = queueName;
4538
- this.migrationOptions = options;
4539
- this.prefixes = options.prefixes ?? [];
4540
- this.prefixValues = options.prefixValues ?? {};
4541
- this.hybridOptions = {
4542
- useBroadcastChannel: options.useBroadcastChannel ?? true,
4543
- backupPollingIntervalMs: options.backupPollingIntervalMs ?? 5000
4544
- };
4545
- if (this.prefixes.length > 0) {
4546
- const prefixNames = this.prefixes.map((p) => p.name).join("_");
4547
- this.tableName = `jobs_${prefixNames}`;
4548
- } else {
4549
- this.tableName = "jobs";
4550
- }
4551
- }
4552
- getPrefixColumnNames() {
4553
- return this.prefixes.map((p) => p.name);
4554
- }
4555
- matchesPrefixes(job) {
4556
- for (const [key, value] of Object.entries(this.prefixValues)) {
4557
- if (job[key] !== value) {
4558
- return false;
4559
- }
4560
- }
4561
- return true;
4562
- }
4563
- getPrefixKeyValues() {
4564
- return this.prefixes.map((p) => this.prefixValues[p.name]);
4565
- }
4566
- async getDb() {
4567
- if (this.db)
4568
- return this.db;
4569
- await this.setupDatabase();
4570
- return this.db;
4571
- }
4572
- async setupDatabase() {
4573
- const prefixColumnNames = this.getPrefixColumnNames();
4574
- const buildKeyPath = (basePath) => {
4575
- return [...prefixColumnNames, ...basePath];
4576
- };
4577
- const expectedIndexes = [
4578
- {
4579
- name: "queue_status",
4580
- keyPath: buildKeyPath(["queue", "status"]),
4581
- options: { unique: false }
4582
- },
4583
- {
4584
- name: "queue_status_run_after",
4585
- keyPath: buildKeyPath(["queue", "status", "run_after"]),
4586
- options: { unique: false }
4587
- },
4588
- {
4589
- name: "queue_job_run_id",
4590
- keyPath: buildKeyPath(["queue", "job_run_id"]),
4591
- options: { unique: false }
4592
- },
4593
- {
4594
- name: "queue_fingerprint_status",
4595
- keyPath: buildKeyPath(["queue", "fingerprint", "status"]),
4596
- options: { unique: false }
4597
- }
4598
- ];
4599
- this.db = await ensureIndexedDbTable(this.tableName, "id", expectedIndexes, this.migrationOptions);
4600
- }
4601
- async add(job) {
4602
- const db = await this.getDb();
4603
- const now = new Date().toISOString();
4604
- const jobWithPrefixes = job;
4605
- jobWithPrefixes.id = jobWithPrefixes.id ?? uuid45();
4606
- jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid45();
4607
- jobWithPrefixes.queue = this.queueName;
4608
- jobWithPrefixes.fingerprint = await makeFingerprint6(jobWithPrefixes.input);
4609
- jobWithPrefixes.status = JobStatus.PENDING;
4610
- jobWithPrefixes.progress = 0;
4611
- jobWithPrefixes.progress_message = "";
4612
- jobWithPrefixes.progress_details = null;
4613
- jobWithPrefixes.created_at = now;
4614
- jobWithPrefixes.run_after = now;
4615
- for (const [key, value] of Object.entries(this.prefixValues)) {
4616
- jobWithPrefixes[key] = value;
4617
- }
4618
- const tx = db.transaction(this.tableName, "readwrite");
4619
- const store = tx.objectStore(this.tableName);
4620
- return new Promise((resolve, reject) => {
4621
- const request = store.add(jobWithPrefixes);
4622
- tx.oncomplete = () => {
4623
- this.hybridManager?.notifyLocalChange();
4624
- resolve(jobWithPrefixes.id);
4625
- };
4626
- tx.onerror = () => reject(tx.error);
4627
- request.onerror = () => reject(request.error);
4628
- });
4629
- }
4630
- async get(id) {
4631
- const db = await this.getDb();
4632
- const tx = db.transaction(this.tableName, "readonly");
4633
- const store = tx.objectStore(this.tableName);
4634
- const request = store.get(id);
4635
- return new Promise((resolve, reject) => {
4636
- request.onsuccess = () => {
4637
- const job = request.result;
4638
- if (job && job.queue === this.queueName && this.matchesPrefixes(job)) {
4639
- resolve(job);
4640
- } else {
4641
- resolve(undefined);
4642
- }
4643
- };
4644
- request.onerror = () => reject(request.error);
4645
- tx.onerror = () => reject(tx.error);
4646
- });
4647
- }
4648
- async peek(status = JobStatus.PENDING, num = 100) {
4649
- const db = await this.getDb();
4650
- const tx = db.transaction(this.tableName, "readonly");
4651
- const store = tx.objectStore(this.tableName);
4652
- const index = store.index("queue_status_run_after");
4653
- const prefixKeyValues = this.getPrefixKeyValues();
4654
- return new Promise((resolve, reject) => {
4655
- const ret = new Map;
4656
- const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, status, ""], [...prefixKeyValues, this.queueName, status, "￿"]);
4657
- const cursorRequest = index.openCursor(keyRange);
4658
- const handleCursor = (e) => {
4659
- const cursor = e.target.result;
4660
- if (!cursor || ret.size >= num) {
4661
- resolve(Array.from(ret.values()));
4662
- return;
4663
- }
4664
- const job = cursor.value;
4665
- if (this.matchesPrefixes(job)) {
4666
- ret.set(cursor.value.id, cursor.value);
4667
- }
4668
- cursor.continue();
4669
- };
4670
- cursorRequest.onsuccess = handleCursor;
4671
- cursorRequest.onerror = () => reject(cursorRequest.error);
4672
- tx.onerror = () => reject(tx.error);
4673
- });
4674
- }
4675
- async next(workerId) {
4676
- const db = await this.getDb();
4677
- const tx = db.transaction(this.tableName, "readwrite");
4678
- const store = tx.objectStore(this.tableName);
4679
- const index = store.index("queue_status_run_after");
4680
- const now = new Date().toISOString();
4681
- const prefixKeyValues = this.getPrefixKeyValues();
4682
- const claimToken = workerId;
4683
- const jobToReturn = await new Promise((resolve, reject) => {
4684
- const cursorRequest = index.openCursor(IDBKeyRange.bound([...prefixKeyValues, this.queueName, JobStatus.PENDING, ""], [...prefixKeyValues, this.queueName, JobStatus.PENDING, now], false, false));
4685
- let claimedJob;
4686
- let cursorStopped = false;
4687
- cursorRequest.onsuccess = (e) => {
4688
- const cursor = e.target.result;
4689
- if (!cursor) {
4690
- return;
4691
- }
4692
- if (cursorStopped) {
4693
- return;
4694
- }
4695
- const job = cursor.value;
4696
- if (job.queue !== this.queueName || job.status !== JobStatus.PENDING || !this.matchesPrefixes(job)) {
4697
- cursor.continue();
4698
- return;
4699
- }
4700
- job.status = JobStatus.PROCESSING;
4701
- job.last_ran_at = now;
4702
- job.worker_id = claimToken;
4703
- try {
4704
- const updateRequest = store.put(job);
4705
- updateRequest.onsuccess = () => {
4706
- claimedJob = job;
4707
- cursorStopped = true;
4708
- };
4709
- updateRequest.onerror = (err) => {
4710
- console.error("Failed to update job status:", err);
4711
- cursor.continue();
4712
- };
4713
- } catch (err) {
4714
- console.error("Error updating job:", err);
4715
- cursor.continue();
4716
- }
4717
- };
4718
- cursorRequest.onerror = () => reject(cursorRequest.error);
4719
- tx.oncomplete = () => {
4720
- if (claimedJob) {
4721
- this.hybridManager?.notifyLocalChange();
4722
- }
4723
- resolve(claimedJob);
4724
- };
4725
- tx.onerror = () => reject(tx.error);
4726
- });
4727
- if (!jobToReturn) {
4728
- return;
4729
- }
4730
- const verifiedJob = await this.get(jobToReturn.id);
4731
- if (!verifiedJob) {
4732
- return;
4733
- }
4734
- if (verifiedJob.worker_id !== claimToken) {
4735
- return;
4736
- }
4737
- if (verifiedJob.status !== JobStatus.PROCESSING) {
4738
- return;
4739
- }
4740
- return verifiedJob;
4741
- }
4742
- async size(status = JobStatus.PENDING) {
4743
- const db = await this.getDb();
4744
- const prefixKeyValues = this.getPrefixKeyValues();
4745
- return new Promise((resolve, reject) => {
4746
- const tx = db.transaction(this.tableName, "readonly");
4747
- const store = tx.objectStore(this.tableName);
4748
- const index = store.index("queue_status");
4749
- const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
4750
- const request = index.count(keyRange);
4751
- request.onsuccess = () => resolve(request.result);
4752
- request.onerror = () => reject(request.error);
4753
- tx.onerror = () => reject(tx.error);
4754
- });
4755
- }
4756
- async complete(job) {
4757
- const db = await this.getDb();
4758
- const tx = db.transaction(this.tableName, "readwrite");
4759
- const store = tx.objectStore(this.tableName);
4760
- return new Promise((resolve, reject) => {
4761
- const getReq = store.get(job.id);
4762
- getReq.onsuccess = () => {
4763
- const existing = getReq.result;
4764
- if (!existing || existing.queue !== this.queueName || !this.matchesPrefixes(existing)) {
4765
- reject(new Error(`Job ${job.id} not found or does not belong to queue ${this.queueName}`));
4766
- return;
4767
- }
4768
- const currentAttempts = existing.run_attempts ?? 0;
4769
- job.run_attempts = currentAttempts + 1;
4770
- job.queue = this.queueName;
4771
- const jobWithPrefixes = job;
4772
- for (const [key, value] of Object.entries(this.prefixValues)) {
4773
- jobWithPrefixes[key] = value;
4774
- }
4775
- const putReq = store.put(jobWithPrefixes);
4776
- putReq.onsuccess = () => {};
4777
- putReq.onerror = () => reject(putReq.error);
4778
- };
4779
- getReq.onerror = () => reject(getReq.error);
4780
- tx.oncomplete = () => {
4781
- this.hybridManager?.notifyLocalChange();
4782
- resolve();
4783
- };
4784
- tx.onerror = () => reject(tx.error);
4785
- });
4786
- }
4787
- async release(id) {
4788
- const job = await this.get(id);
4789
- if (!job)
4790
- return;
4791
- job.status = JobStatus.PENDING;
4792
- job.worker_id = null;
4793
- job.progress = 0;
4794
- job.progress_message = "";
4795
- job.progress_details = null;
4796
- await this.put(job);
4797
- }
4798
- async abort(id) {
4799
- const job = await this.get(id);
4800
- if (!job)
4801
- return;
4802
- job.status = JobStatus.ABORTING;
4803
- await this.complete(job);
4804
- }
4805
- async getByRunId(job_run_id) {
4806
- const db = await this.getDb();
4807
- const tx = db.transaction(this.tableName, "readonly");
4808
- const store = tx.objectStore(this.tableName);
4809
- const index = store.index("queue_job_run_id");
4810
- const prefixKeyValues = this.getPrefixKeyValues();
4811
- const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, job_run_id]);
4812
- const request = index.getAll(keyRange);
4813
- return new Promise((resolve, reject) => {
4814
- request.onsuccess = () => {
4815
- const results = (request.result || []).filter((job) => this.matchesPrefixes(job));
4816
- resolve(results);
4817
- };
4818
- request.onerror = () => reject(request.error);
4819
- tx.onerror = () => reject(tx.error);
4820
- });
4821
- }
4822
- async deleteAll() {
4823
- const db = await this.getDb();
4824
- const tx = db.transaction(this.tableName, "readwrite");
4825
- const store = tx.objectStore(this.tableName);
4826
- const index = store.index("queue_status");
4827
- const prefixKeyValues = this.getPrefixKeyValues();
4828
- return new Promise((resolve, reject) => {
4829
- const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, "￿"]);
4830
- const request = index.openCursor(keyRange);
4831
- request.onsuccess = (event) => {
4832
- const cursor = event.target.result;
4833
- if (cursor) {
4834
- const job = cursor.value;
4835
- if (job.queue === this.queueName && this.matchesPrefixes(job)) {
4836
- const deleteRequest = cursor.delete();
4837
- deleteRequest.onsuccess = () => {
4838
- cursor.continue();
4839
- };
4840
- deleteRequest.onerror = () => {
4841
- cursor.continue();
4842
- };
4843
- } else {
4844
- cursor.continue();
4845
- }
4846
- }
4847
- };
4848
- tx.oncomplete = () => {
4849
- this.hybridManager?.notifyLocalChange();
4850
- resolve();
4851
- };
4852
- tx.onerror = () => reject(tx.error);
4853
- request.onerror = () => reject(request.error);
4854
- });
4855
- }
4856
- async outputForInput(input) {
4857
- const fingerprint = await makeFingerprint6(input);
4858
- const db = await this.getDb();
4859
- const tx = db.transaction(this.tableName, "readonly");
4860
- const store = tx.objectStore(this.tableName);
4861
- const index = store.index("queue_fingerprint_status");
4862
- const prefixKeyValues = this.getPrefixKeyValues();
4863
- const request = index.get([
4864
- ...prefixKeyValues,
4865
- this.queueName,
4866
- fingerprint,
4867
- JobStatus.COMPLETED
4868
- ]);
4869
- return new Promise((resolve, reject) => {
4870
- request.onsuccess = () => {
4871
- const job = request.result;
4872
- if (job && this.matchesPrefixes(job)) {
4873
- resolve(job.output ?? null);
4874
- } else {
4875
- resolve(null);
4876
- }
4877
- };
4878
- request.onerror = () => reject(request.error);
4879
- tx.onerror = () => reject(tx.error);
4880
- });
4881
- }
4882
- async saveProgress(id, progress, message, details) {
4883
- const job = await this.get(id);
4884
- if (!job)
4885
- throw new Error(`Job ${id} not found`);
4886
- job.progress = progress;
4887
- job.progress_message = message;
4888
- job.progress_details = details;
4889
- await this.put(job);
4890
- }
4891
- async put(job) {
4892
- const db = await this.getDb();
4893
- const tx = db.transaction(this.tableName, "readwrite");
4894
- const store = tx.objectStore(this.tableName);
4895
- job.queue = this.queueName;
4896
- const jobWithPrefixes = job;
4897
- for (const [key, value] of Object.entries(this.prefixValues)) {
4898
- jobWithPrefixes[key] = value;
4899
- }
4900
- return new Promise((resolve, reject) => {
4901
- const putReq = store.put(jobWithPrefixes);
4902
- putReq.onerror = () => reject(putReq.error);
4903
- tx.oncomplete = () => {
4904
- this.hybridManager?.notifyLocalChange();
4905
- resolve();
4906
- };
4907
- tx.onerror = () => reject(tx.error);
4908
- });
4909
- }
4910
- async delete(id) {
4911
- const job = await this.get(id);
4912
- if (!job)
4913
- return;
4914
- const db = await this.getDb();
4915
- const tx = db.transaction(this.tableName, "readwrite");
4916
- const store = tx.objectStore(this.tableName);
4917
- const request = store.delete(id);
4918
- return new Promise((resolve, reject) => {
4919
- request.onsuccess = () => resolve();
4920
- request.onerror = () => reject(request.error);
4921
- tx.oncomplete = () => {
4922
- this.hybridManager?.notifyLocalChange();
4923
- };
4924
- tx.onerror = () => reject(tx.error);
4925
- });
4926
- }
4927
- async deleteJobsByStatusAndAge(status, olderThanMs) {
4928
- const db = await this.getDb();
4929
- const tx = db.transaction(this.tableName, "readwrite");
4930
- const store = tx.objectStore(this.tableName);
4931
- const index = store.index("queue_status");
4932
- const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
4933
- const prefixKeyValues = this.getPrefixKeyValues();
4934
- const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
4935
- return new Promise((resolve, reject) => {
4936
- const request = index.openCursor(keyRange);
4937
- request.onsuccess = (event) => {
4938
- const cursor = event.target.result;
4939
- if (cursor) {
4940
- const job = cursor.value;
4941
- if (job.queue === this.queueName && this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
4942
- cursor.delete();
4943
- }
4944
- cursor.continue();
4945
- }
4946
- };
4947
- tx.oncomplete = () => {
4948
- this.hybridManager?.notifyLocalChange();
4949
- resolve();
4950
- };
4951
- tx.onerror = () => reject(tx.error);
4952
- request.onerror = () => reject(request.error);
4953
- });
4954
- }
4955
- async getAllJobs() {
4956
- const db = await this.getDb();
4957
- const tx = db.transaction(this.tableName, "readonly");
4958
- const store = tx.objectStore(this.tableName);
4959
- const index = store.index("queue_status");
4960
- const prefixKeyValues = this.getPrefixKeyValues();
4961
- return new Promise((resolve, reject) => {
4962
- const jobs = [];
4963
- const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, "￿"]);
4964
- const request = index.openCursor(keyRange);
4965
- request.onsuccess = (event) => {
4966
- const cursor = event.target.result;
4967
- if (cursor) {
4968
- const job = cursor.value;
4969
- if (job.queue === this.queueName && this.matchesPrefixes(job)) {
4970
- jobs.push(job);
4971
- }
4972
- cursor.continue();
4973
- }
4974
- };
4975
- tx.oncomplete = () => resolve(jobs);
4976
- tx.onerror = () => reject(tx.error);
4977
- request.onerror = () => reject(request.error);
4978
- });
4979
- }
4980
- async getAllJobsWithFilter(prefixFilter) {
4981
- const db = await this.getDb();
4982
- const tx = db.transaction(this.tableName, "readonly");
4983
- const store = tx.objectStore(this.tableName);
4984
- return new Promise((resolve, reject) => {
4985
- const jobs = [];
4986
- const request = store.openCursor();
4987
- request.onsuccess = (event) => {
4988
- const cursor = event.target.result;
4989
- if (cursor) {
4990
- const job = cursor.value;
4991
- if (job.queue !== this.queueName) {
4992
- cursor.continue();
4993
- return;
4994
- }
4995
- if (Object.keys(prefixFilter).length === 0) {
4996
- jobs.push(job);
4997
- } else {
4998
- let matches = true;
4999
- for (const [key, value] of Object.entries(prefixFilter)) {
5000
- if (job[key] !== value) {
5001
- matches = false;
5002
- break;
5003
- }
5004
- }
5005
- if (matches) {
5006
- jobs.push(job);
5007
- }
5008
- }
5009
- cursor.continue();
5010
- }
5011
- };
5012
- tx.oncomplete = () => resolve(jobs);
5013
- tx.onerror = () => reject(tx.error);
5014
- request.onerror = () => reject(request.error);
5015
- });
5016
- }
5017
- isCustomPrefixFilter(prefixFilter) {
5018
- if (prefixFilter === undefined) {
5019
- return false;
5020
- }
5021
- if (Object.keys(prefixFilter).length === 0) {
5022
- return true;
5023
- }
5024
- const instanceKeys = Object.keys(this.prefixValues);
5025
- const filterKeys = Object.keys(prefixFilter);
5026
- if (instanceKeys.length !== filterKeys.length) {
5027
- return true;
5028
- }
5029
- for (const key of instanceKeys) {
5030
- if (this.prefixValues[key] !== prefixFilter[key]) {
5031
- return true;
5032
- }
5033
- }
5034
- return false;
5035
- }
5036
- getHybridManager() {
5037
- if (!this.hybridManager) {
5038
- const channelName = `indexeddb-queue-${this.tableName}-${this.queueName}`;
5039
- this.hybridManager = new HybridSubscriptionManager(channelName, async () => {
5040
- const jobs = await this.getAllJobs();
5041
- return new Map(jobs.map((j) => [j.id, j]));
5042
- }, (a, b) => deepEqual3(a, b), {
5043
- insert: (item) => ({ type: "INSERT", new: item }),
5044
- update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
5045
- delete: (item) => ({ type: "DELETE", old: item })
5046
- }, {
5047
- defaultIntervalMs: 1000,
5048
- useBroadcastChannel: this.hybridOptions.useBroadcastChannel,
5049
- backupPollingIntervalMs: this.hybridOptions.backupPollingIntervalMs
5050
- });
5051
- }
5052
- return this.hybridManager;
5053
- }
5054
- subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
5055
- let lastKnownJobs = new Map;
5056
- let cancelled = false;
5057
- const poll = async () => {
5058
- if (cancelled)
5059
- return;
5060
- try {
5061
- const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
5062
- if (cancelled)
5063
- return;
5064
- const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
5065
- for (const [id, job] of currentMap) {
5066
- const old = lastKnownJobs.get(id);
5067
- if (!old) {
5068
- callback({ type: "INSERT", new: job });
5069
- } else if (!deepEqual3(old, job)) {
5070
- callback({ type: "UPDATE", old, new: job });
5071
- }
5072
- }
5073
- for (const [id, job] of lastKnownJobs) {
5074
- if (!currentMap.has(id)) {
5075
- callback({ type: "DELETE", old: job });
5076
- }
5077
- }
5078
- lastKnownJobs = currentMap;
5079
- } catch {}
5080
- };
5081
- const intervalId = setInterval(poll, intervalMs);
5082
- poll();
5083
- return () => {
5084
- cancelled = true;
5085
- clearInterval(intervalId);
5086
- };
5087
- }
5088
- subscribeToChanges(callback, options) {
5089
- const intervalMs = options?.pollingIntervalMs ?? 1000;
5090
- if (this.isCustomPrefixFilter(options?.prefixFilter)) {
5091
- return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
5092
- }
5093
- const manager = this.getHybridManager();
5094
- return manager.subscribe(callback, { intervalMs });
5095
- }
5096
- destroy() {
5097
- if (this.hybridManager) {
5098
- this.hybridManager.destroy();
5099
- this.hybridManager = null;
5100
- }
5101
- }
5102
- }
5103
- // src/queue/SupabaseQueueStorage.ts
5104
- import { createServiceToken as createServiceToken18, deepEqual as deepEqual4, makeFingerprint as makeFingerprint7, uuid4 as uuid46 } from "@workglow/util";
5105
- var SUPABASE_QUEUE_STORAGE = createServiceToken18("jobqueue.storage.supabase");
5106
-
5107
- class SupabaseQueueStorage {
5108
- queueName;
5109
- scope = "cluster";
5110
- client;
5111
- prefixes;
5112
- prefixValues;
5113
- tableName;
5114
- realtimeChannel = null;
5115
- pollingManager = null;
5116
- constructor(client, queueName, options) {
5117
- this.queueName = queueName;
5118
- this.client = client;
5119
- this.prefixes = options?.prefixes ?? [];
5120
- this.prefixValues = options?.prefixValues ?? {};
5121
- if (this.prefixes.length > 0) {
5122
- const prefixNames = this.prefixes.map((p) => p.name).join("_");
5123
- this.tableName = `job_queue_${prefixNames}`;
5124
- } else {
5125
- this.tableName = "job_queue";
5126
- }
5127
- }
5128
- getPrefixColumnType(type) {
5129
- return type === "uuid" ? "UUID" : "INTEGER";
5130
- }
5131
- buildPrefixColumnsSql() {
5132
- if (this.prefixes.length === 0)
5133
- return "";
5134
- return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
5135
- `) + `,
5136
- `;
5137
- }
5138
- getPrefixColumnNames() {
5139
- return this.prefixes.map((p) => p.name);
5140
- }
5141
- applyPrefixFilters(query) {
5142
- let result = query;
5143
- for (const prefix of this.prefixes) {
5144
- result = result.eq(prefix.name, this.prefixValues[prefix.name]);
5145
- }
5146
- return result;
5147
- }
5148
- getPrefixInsertValues() {
5149
- const values = {};
5150
- for (const prefix of this.prefixes) {
5151
- values[prefix.name] = this.prefixValues[prefix.name];
5152
- }
5153
- return values;
5154
- }
5155
- buildPrefixWhereSql() {
5156
- if (this.prefixes.length === 0) {
5157
- return "";
5158
- }
5159
- const conditions = this.prefixes.map((p) => {
5160
- const value = this.prefixValues[p.name];
5161
- if (p.type === "uuid") {
5162
- const validated = this.validateSqlValue(String(value), `prefix "${p.name}"`);
5163
- return `${p.name} = '${this.escapeSqlString(validated)}'`;
5164
- }
5165
- const numValue = Number(value ?? 0);
5166
- if (!Number.isFinite(numValue)) {
5167
- throw new Error(`Invalid numeric prefix value for "${p.name}": ${value}`);
5168
- }
5169
- return `${p.name} = ${numValue}`;
5170
- }).join(" AND ");
5171
- return " AND " + conditions;
5172
- }
5173
- static SAFE_SQL_VALUE_RE = /^[a-zA-Z0-9_\-.:]+$/;
5174
- validateSqlValue(value, context) {
5175
- if (!SupabaseQueueStorage.SAFE_SQL_VALUE_RE.test(value)) {
5176
- throw new Error(`Unsafe value for ${context}: "${value}". Values must match /^[a-zA-Z0-9_\\-.:]+$/.`);
5177
- }
5178
- return value;
5179
- }
5180
- escapeSqlString(value) {
5181
- return value.replace(/'/g, "''");
5182
- }
5183
- async setupDatabase() {
5184
- const createTypeSql = `CREATE TYPE job_status AS ENUM (${Object.values(JobStatus).map((v) => `'${v}'`).join(",")})`;
5185
- const { error: typeError } = await this.client.rpc("exec_sql", { query: createTypeSql });
5186
- if (typeError && typeError.code !== "42710") {
5187
- throw typeError;
5188
- }
5189
- const prefixColumnsSql = this.buildPrefixColumnsSql();
5190
- const prefixColumnNames = this.getPrefixColumnNames();
5191
- const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
5192
- const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
5193
- const createTableSql = `
5194
- CREATE TABLE IF NOT EXISTS ${this.tableName} (
5195
- id SERIAL NOT NULL,
5196
- ${prefixColumnsSql}fingerprint text NOT NULL,
5197
- queue text NOT NULL,
5198
- job_run_id text NOT NULL,
5199
- status job_status NOT NULL default 'PENDING',
5200
- input jsonb NOT NULL,
5201
- output jsonb,
5202
- run_attempts integer default 0,
5203
- max_retries integer default 20,
5204
- run_after timestamp with time zone DEFAULT now(),
5205
- last_ran_at timestamp with time zone,
5206
- created_at timestamp with time zone DEFAULT now(),
5207
- deadline_at timestamp with time zone,
5208
- completed_at timestamp with time zone,
5209
- error text,
5210
- error_code text,
5211
- progress real DEFAULT 0,
5212
- progress_message text DEFAULT '',
5213
- progress_details jsonb,
5214
- worker_id text
5215
- )`;
5216
- const { error: tableError } = await this.client.rpc("exec_sql", { query: createTableSql });
5217
- if (tableError) {
5218
- if (tableError.code !== "42P07") {
5219
- throw tableError;
5220
- }
5221
- }
5222
- const indexes = [
5223
- `CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`,
5224
- `CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`,
5225
- `CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`
5226
- ];
5227
- for (const indexSql of indexes) {
5228
- await this.client.rpc("exec_sql", { query: indexSql });
5229
- }
5230
- }
5231
- async add(job) {
5232
- const now = new Date().toISOString();
5233
- job.queue = this.queueName;
5234
- job.job_run_id = job.job_run_id ?? uuid46();
5235
- job.fingerprint = await makeFingerprint7(job.input);
5236
- job.status = JobStatus.PENDING;
5237
- job.progress = 0;
5238
- job.progress_message = "";
5239
- job.progress_details = null;
5240
- job.created_at = now;
5241
- job.run_after = now;
5242
- const prefixInsertValues = this.getPrefixInsertValues();
5243
- const { data, error } = await this.client.from(this.tableName).insert({
5244
- ...prefixInsertValues,
5245
- queue: job.queue,
5246
- fingerprint: job.fingerprint,
5247
- input: job.input,
5248
- run_after: job.run_after,
5249
- created_at: job.created_at,
5250
- deadline_at: job.deadline_at,
5251
- max_retries: job.max_retries,
5252
- job_run_id: job.job_run_id,
5253
- progress: job.progress,
5254
- progress_message: job.progress_message,
5255
- progress_details: job.progress_details
5256
- }).select("id").single();
5257
- if (error)
5258
- throw error;
5259
- if (!data)
5260
- throw new Error("Failed to add to queue");
5261
- job.id = data.id;
5262
- return job.id;
5263
- }
5264
- async get(id) {
5265
- let query = this.client.from(this.tableName).select("*").eq("id", id).eq("queue", this.queueName);
5266
- query = this.applyPrefixFilters(query);
5267
- const { data, error } = await query.single();
5268
- if (error) {
5269
- if (error.code === "PGRST116")
5270
- return;
5271
- throw error;
5272
- }
5273
- return data;
5274
- }
5275
- async peek(status = JobStatus.PENDING, num = 100) {
5276
- num = Number(num) || 100;
5277
- let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", status);
5278
- query = this.applyPrefixFilters(query);
5279
- const { data, error } = await query.order("run_after", { ascending: true }).limit(num);
5280
- if (error)
5281
- throw error;
5282
- return data ?? [];
5283
- }
5284
- async next(workerId) {
5285
- const prefixConditions = this.buildPrefixWhereSql();
5286
- const validatedQueueName = this.validateSqlValue(this.queueName, "queueName");
5287
- const validatedWorkerId = this.validateSqlValue(workerId, "workerId");
5288
- const escapedQueueName = this.escapeSqlString(validatedQueueName);
5289
- const escapedWorkerId = this.escapeSqlString(validatedWorkerId);
5290
- const sql = `
5291
- UPDATE ${this.tableName}
5292
- SET status = '${JobStatus.PROCESSING}', last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = '${escapedWorkerId}'
5293
- WHERE id = (
5294
- SELECT id
5295
- FROM ${this.tableName}
5296
- WHERE queue = '${escapedQueueName}'
5297
- AND status = '${JobStatus.PENDING}'
5298
- ${prefixConditions}
5299
- AND run_after <= NOW() AT TIME ZONE 'UTC'
5300
- ORDER BY run_after ASC
5301
- FOR UPDATE SKIP LOCKED
5302
- LIMIT 1
5303
- )
5304
- RETURNING *`;
5305
- const { data, error } = await this.client.rpc("exec_sql", { query: sql });
5306
- if (error)
5307
- throw error;
5308
- if (!data || !Array.isArray(data) || data.length === 0) {
5309
- return;
5310
- }
5311
- return data[0];
5312
- }
5313
- async size(status = JobStatus.PENDING) {
5314
- let query = this.client.from(this.tableName).select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
5315
- query = this.applyPrefixFilters(query);
5316
- const { count, error } = await query;
5317
- if (error)
5318
- throw error;
5319
- return count ?? 0;
5320
- }
5321
- async getAllJobs() {
5322
- let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
5323
- query = this.applyPrefixFilters(query);
5324
- const { data, error } = await query;
5325
- if (error)
5326
- throw error;
5327
- return data ?? [];
5328
- }
5329
- async complete(jobDetails) {
5330
- const now = new Date().toISOString();
5331
- if (jobDetails.status === JobStatus.DISABLED) {
5332
- let query2 = this.client.from(this.tableName).update({
5333
- status: jobDetails.status,
5334
- progress: 100,
5335
- progress_message: "",
5336
- progress_details: null,
5337
- completed_at: now,
5338
- last_ran_at: now
5339
- }).eq("id", jobDetails.id).eq("queue", this.queueName);
5340
- query2 = this.applyPrefixFilters(query2);
5341
- const { error: error2 } = await query2;
5342
- if (error2)
5343
- throw error2;
5344
- return;
5345
- }
5346
- let getQuery = this.client.from(this.tableName).select("run_attempts, max_retries").eq("id", jobDetails.id).eq("queue", this.queueName);
5347
- getQuery = this.applyPrefixFilters(getQuery);
5348
- const { data: current, error: getError } = await getQuery.single();
5349
- if (getError)
5350
- throw getError;
5351
- const currentAttempts = current?.run_attempts ?? 0;
5352
- const maxRetries = current?.max_retries ?? jobDetails.max_retries ?? 10;
5353
- const nextAttempts = currentAttempts + 1;
5354
- if (jobDetails.status === JobStatus.PENDING) {
5355
- if (nextAttempts > maxRetries) {
5356
- let failQuery = this.client.from(this.tableName).update({
5357
- status: JobStatus.FAILED,
5358
- error: "Max retries reached",
5359
- error_code: "MAX_RETRIES_REACHED",
5360
- progress: 100,
5361
- progress_message: "",
5362
- progress_details: null,
5363
- completed_at: now,
5364
- last_ran_at: now
5365
- }).eq("id", jobDetails.id).eq("queue", this.queueName);
5366
- failQuery = this.applyPrefixFilters(failQuery);
5367
- const { error: failError } = await failQuery;
5368
- if (failError)
5369
- throw failError;
5370
- return;
5371
- }
5372
- let query2 = this.client.from(this.tableName).update({
5373
- error: jobDetails.error ?? null,
5374
- error_code: jobDetails.error_code ?? null,
5375
- status: jobDetails.status,
5376
- run_after: jobDetails.run_after,
5377
- progress: 0,
5378
- progress_message: "",
5379
- progress_details: null,
5380
- run_attempts: nextAttempts,
5381
- last_ran_at: now
5382
- }).eq("id", jobDetails.id).eq("queue", this.queueName);
5383
- query2 = this.applyPrefixFilters(query2);
5384
- const { error: error2 } = await query2;
5385
- if (error2)
5386
- throw error2;
5387
- return;
5388
- }
5389
- if (jobDetails.status === JobStatus.COMPLETED || jobDetails.status === JobStatus.FAILED) {
5390
- let query2 = this.client.from(this.tableName).update({
5391
- output: jobDetails.output ?? null,
5392
- error: jobDetails.error ?? null,
5393
- error_code: jobDetails.error_code ?? null,
5394
- status: jobDetails.status,
5395
- progress: 100,
5396
- progress_message: "",
5397
- progress_details: null,
5398
- run_attempts: nextAttempts,
5399
- completed_at: now,
5400
- last_ran_at: now
5401
- }).eq("id", jobDetails.id).eq("queue", this.queueName);
5402
- query2 = this.applyPrefixFilters(query2);
5403
- const { error: error2 } = await query2;
5404
- if (error2)
5405
- throw error2;
5406
- return;
5407
- }
5408
- let query = this.client.from(this.tableName).update({
5409
- status: jobDetails.status,
5410
- output: jobDetails.output ?? null,
5411
- error: jobDetails.error ?? null,
5412
- error_code: jobDetails.error_code ?? null,
5413
- run_after: jobDetails.run_after ?? null,
5414
- run_attempts: nextAttempts,
5415
- last_ran_at: now
5416
- }).eq("id", jobDetails.id).eq("queue", this.queueName);
5417
- query = this.applyPrefixFilters(query);
5418
- const { error } = await query;
5419
- if (error)
5420
- throw error;
5421
- }
5422
- async release(jobId) {
5423
- let query = this.client.from(this.tableName).update({
5424
- status: JobStatus.PENDING,
5425
- worker_id: null,
5426
- progress: 0,
5427
- progress_message: "",
5428
- progress_details: null
5429
- }).eq("id", jobId).eq("queue", this.queueName);
5430
- query = this.applyPrefixFilters(query);
5431
- const { error } = await query;
5432
- if (error)
5433
- throw error;
5434
- }
5435
- async deleteAll() {
5436
- let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
5437
- query = this.applyPrefixFilters(query);
5438
- const { error } = await query;
5439
- if (error)
5440
- throw error;
5441
- }
5442
- async outputForInput(input) {
5443
- const fingerprint = await makeFingerprint7(input);
5444
- let query = this.client.from(this.tableName).select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", JobStatus.COMPLETED);
5445
- query = this.applyPrefixFilters(query);
5446
- const { data, error } = await query.single();
5447
- if (error) {
5448
- if (error.code === "PGRST116")
5449
- return null;
5450
- throw error;
5451
- }
5452
- return data?.output ?? null;
5453
- }
5454
- async abort(jobId) {
5455
- let query = this.client.from(this.tableName).update({ status: JobStatus.ABORTING }).eq("id", jobId).eq("queue", this.queueName);
5456
- query = this.applyPrefixFilters(query);
5457
- const { error } = await query;
5458
- if (error)
5459
- throw error;
5460
- }
5461
- async getByRunId(job_run_id) {
5462
- let query = this.client.from(this.tableName).select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
5463
- query = this.applyPrefixFilters(query);
5464
- const { data, error } = await query;
5465
- if (error)
5466
- throw error;
5467
- return data ?? [];
5468
- }
5469
- async saveProgress(jobId, progress, message, details) {
5470
- let query = this.client.from(this.tableName).update({
5471
- progress,
5472
- progress_message: message,
5473
- progress_details: details
5474
- }).eq("id", jobId).eq("queue", this.queueName);
5475
- query = this.applyPrefixFilters(query);
5476
- const { error } = await query;
5477
- if (error)
5478
- throw error;
5479
- }
5480
- async delete(jobId) {
5481
- let query = this.client.from(this.tableName).delete().eq("id", jobId).eq("queue", this.queueName);
5482
- query = this.applyPrefixFilters(query);
5483
- const { error } = await query;
5484
- if (error)
5485
- throw error;
5486
- }
5487
- async deleteJobsByStatusAndAge(status, olderThanMs) {
5488
- const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
5489
- let query = this.client.from(this.tableName).delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
5490
- query = this.applyPrefixFilters(query);
5491
- const { error } = await query;
5492
- if (error)
5493
- throw error;
5494
- }
5495
- matchesPrefixFilter(job, prefixFilter) {
5496
- if (!job)
5497
- return false;
5498
- if (job.queue !== this.queueName) {
5499
- return false;
5500
- }
5501
- if (prefixFilter && Object.keys(prefixFilter).length === 0) {
5502
- return true;
5503
- }
5504
- const filterValues = prefixFilter ?? this.prefixValues;
5505
- if (Object.keys(filterValues).length === 0) {
5506
- return true;
5507
- }
5508
- for (const [key, value] of Object.entries(filterValues)) {
5509
- if (job[key] !== value) {
5510
- return false;
5511
- }
5512
- }
5513
- return true;
5514
- }
5515
- isCustomPrefixFilter(prefixFilter) {
5516
- if (prefixFilter === undefined) {
5517
- return false;
5518
- }
5519
- if (Object.keys(prefixFilter).length === 0) {
5520
- return true;
5521
- }
5522
- const instanceKeys = Object.keys(this.prefixValues);
5523
- const filterKeys = Object.keys(prefixFilter);
5524
- if (instanceKeys.length !== filterKeys.length) {
5525
- return true;
5526
- }
5527
- for (const key of instanceKeys) {
5528
- if (this.prefixValues[key] !== prefixFilter[key]) {
5529
- return true;
5530
- }
5531
- }
5532
- return false;
5533
- }
5534
- async getAllJobsWithFilter(prefixFilter) {
5535
- let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
5536
- for (const [key, value] of Object.entries(prefixFilter)) {
5537
- query = query.eq(key, value);
5538
- }
5539
- const { data, error } = await query;
5540
- if (error)
5541
- throw error;
5542
- return data ?? [];
5543
- }
5544
- subscribeToChanges(callback, options) {
5545
- return this.subscribeToChangesWithRealtime(callback, options?.prefixFilter);
5546
- }
5547
- subscribeToChangesWithRealtime(callback, prefixFilter) {
5548
- const channelName = `queue-${this.tableName}-${this.queueName}-${Date.now()}`;
5549
- this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
5550
- event: "*",
5551
- schema: "public",
5552
- table: this.tableName,
5553
- filter: `queue=eq.${this.queueName}`
5554
- }, (payload) => {
5555
- const newJob = payload.new;
5556
- const oldJob = payload.old;
5557
- const newMatches = this.matchesPrefixFilter(newJob, prefixFilter);
5558
- const oldMatches = this.matchesPrefixFilter(oldJob, prefixFilter);
5559
- if (!newMatches && !oldMatches) {
5560
- return;
5561
- }
5562
- callback({
5563
- type: payload.eventType.toUpperCase(),
5564
- old: oldJob && Object.keys(oldJob).length > 0 ? oldJob : undefined,
5565
- new: newJob && Object.keys(newJob).length > 0 ? newJob : undefined
5566
- });
5567
- }).subscribe();
5568
- return () => {
5569
- if (this.realtimeChannel) {
5570
- this.client.removeChannel(this.realtimeChannel);
5571
- this.realtimeChannel = null;
5572
- }
5573
- };
5574
- }
5575
- getPollingManager() {
5576
- if (!this.pollingManager) {
5577
- this.pollingManager = new PollingSubscriptionManager(async () => {
5578
- const jobs = await this.getAllJobs();
5579
- return new Map(jobs.map((j) => [j.id, j]));
5580
- }, (a, b) => deepEqual4(a, b), {
5581
- insert: (item) => ({ type: "INSERT", new: item }),
5582
- update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
5583
- delete: (item) => ({ type: "DELETE", old: item })
5584
- });
5585
- }
5586
- return this.pollingManager;
5587
- }
5588
- subscribeWithCustomPrefixFilterPolling(callback, prefixFilter, intervalMs) {
5589
- let lastKnownJobs = new Map;
5590
- let cancelled = false;
5591
- const poll = async () => {
5592
- if (cancelled)
5593
- return;
5594
- try {
5595
- const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
5596
- if (cancelled)
5597
- return;
5598
- const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
5599
- for (const [id, job] of currentMap) {
5600
- const old = lastKnownJobs.get(id);
5601
- if (!old) {
5602
- callback({ type: "INSERT", new: job });
5603
- } else if (!deepEqual4(old, job)) {
5604
- callback({ type: "UPDATE", old, new: job });
5605
- }
5606
- }
5607
- for (const [id, job] of lastKnownJobs) {
5608
- if (!currentMap.has(id)) {
5609
- callback({ type: "DELETE", old: job });
5610
- }
5611
- }
5612
- lastKnownJobs = currentMap;
5613
- } catch {}
5614
- };
5615
- const intervalId = setInterval(poll, intervalMs);
5616
- poll();
5617
- return () => {
5618
- cancelled = true;
5619
- clearInterval(intervalId);
5620
- };
5621
- }
5622
- subscribeToChangesWithPolling(callback, options) {
5623
- const intervalMs = options?.pollingIntervalMs ?? 1000;
5624
- if (this.isCustomPrefixFilter(options?.prefixFilter)) {
5625
- return this.subscribeWithCustomPrefixFilterPolling(callback, options.prefixFilter, intervalMs);
5626
- }
5627
- const manager = this.getPollingManager();
5628
- return manager.subscribe(callback, { intervalMs });
5629
- }
5630
- }
5631
- // src/queue-limiter/IndexedDbRateLimiterStorage.ts
5632
- import { createServiceToken as createServiceToken19 } from "@workglow/util";
5633
- var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken19("ratelimiter.storage.indexedDb");
5634
-
5635
- class IndexedDbRateLimiterStorage {
5636
- scope = "process";
5637
- executionDb;
5638
- nextAvailableDb;
5639
- executionTableName;
5640
- nextAvailableTableName;
5641
- migrationOptions;
5642
- prefixes;
5643
- prefixValues;
5644
- constructor(options = {}) {
5645
- this.migrationOptions = options;
5646
- this.prefixes = options.prefixes ?? [];
5647
- this.prefixValues = options.prefixValues ?? {};
5648
- if (this.prefixes.length > 0) {
5649
- const prefixNames = this.prefixes.map((p) => p.name).join("_");
5650
- this.executionTableName = `rate_limit_executions_${prefixNames}`;
5651
- this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
5652
- } else {
5653
- this.executionTableName = "rate_limit_executions";
5654
- this.nextAvailableTableName = "rate_limit_next_available";
5655
- }
5656
- }
5657
- getPrefixColumnNames() {
5658
- return this.prefixes.map((p) => p.name);
5659
- }
5660
- matchesPrefixes(record) {
5661
- for (const [key, value] of Object.entries(this.prefixValues)) {
5662
- if (record[key] !== value) {
5663
- return false;
5664
- }
5665
- }
5666
- return true;
5667
- }
5668
- getPrefixKeyValues() {
5669
- return this.prefixes.map((p) => this.prefixValues[p.name]);
5670
- }
5671
- async getExecutionDb() {
5672
- if (this.executionDb)
5673
- return this.executionDb;
5674
- await this.setupDatabase();
5675
- return this.executionDb;
5676
- }
5677
- async getNextAvailableDb() {
5678
- if (this.nextAvailableDb)
5679
- return this.nextAvailableDb;
5680
- await this.setupDatabase();
5681
- return this.nextAvailableDb;
5682
- }
5683
- async setupDatabase() {
5684
- const prefixColumnNames = this.getPrefixColumnNames();
5685
- const buildKeyPath = (basePath) => {
5686
- return [...prefixColumnNames, ...basePath];
5687
- };
5688
- const executionIndexes = [
5689
- {
5690
- name: "queue_executed_at",
5691
- keyPath: buildKeyPath(["queue_name", "executed_at"]),
5692
- options: { unique: false }
5693
- }
5694
- ];
5695
- this.executionDb = await ensureIndexedDbTable(this.executionTableName, "id", executionIndexes, this.migrationOptions);
5696
- const nextAvailableIndexes = [
5697
- {
5698
- name: "queue_name",
5699
- keyPath: buildKeyPath(["queue_name"]),
5700
- options: { unique: true }
5701
- }
5702
- ];
5703
- this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
5704
- }
5705
- async tryReserveExecution(queueName, maxExecutions, windowMs) {
5706
- const nextIso = await this.getNextAvailableTime(queueName);
5707
- if (nextIso && new Date(nextIso).getTime() > Date.now()) {
5708
- return null;
5709
- }
5710
- const execDb = await this.getExecutionDb();
5711
- const prefixKeyValues = this.getPrefixKeyValues();
5712
- const windowStartIso = new Date(Date.now() - windowMs).toISOString();
5713
- const execTx = execDb.transaction(this.executionTableName, "readwrite");
5714
- const execStore = execTx.objectStore(this.executionTableName);
5715
- const insertedId = crypto.randomUUID();
5716
- return new Promise((resolve, reject) => {
5717
- let liveCount = 0;
5718
- let didInsert = false;
5719
- const liveRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartIso], [...prefixKeyValues, queueName, "￿"], true, false);
5720
- const cursorReq = execStore.index("queue_executed_at").openCursor(liveRange);
5721
- cursorReq.onsuccess = (event) => {
5722
- const cursor = event.target.result;
5723
- if (cursor) {
5724
- const record2 = cursor.value;
5725
- if (this.matchesPrefixes(record2)) {
5726
- liveCount++;
5727
- }
5728
- cursor.continue();
5729
- return;
5730
- }
5731
- if (liveCount >= maxExecutions) {
5732
- execTx.abort();
5733
- return;
5734
- }
5735
- const record = {
5736
- id: insertedId,
5737
- queue_name: queueName,
5738
- executed_at: new Date().toISOString()
5739
- };
5740
- for (const [k, v] of Object.entries(this.prefixValues)) {
5741
- record[k] = v;
5742
- }
5743
- const addReq = execStore.add(record);
5744
- didInsert = true;
5745
- addReq.onerror = () => {
5746
- try {
5747
- execTx.abort();
5748
- } catch {}
5749
- reject(addReq.error);
5750
- };
5751
- };
5752
- cursorReq.onerror = () => reject(cursorReq.error);
5753
- execTx.oncomplete = () => resolve(didInsert ? insertedId : null);
5754
- execTx.onerror = () => reject(execTx.error);
5755
- execTx.onabort = () => resolve(null);
5756
- });
5757
- }
5758
- async releaseExecution(queueName, token) {
5759
- if (token === null || token === undefined)
5760
- return;
5761
- const db = await this.getExecutionDb();
5762
- const tx = db.transaction(this.executionTableName, "readwrite");
5763
- const store = tx.objectStore(this.executionTableName);
5764
- return new Promise((resolve, reject) => {
5765
- const req = store.delete(token);
5766
- req.onerror = () => reject(req.error);
5767
- tx.oncomplete = () => resolve();
5768
- tx.onerror = () => reject(tx.error);
5769
- });
5770
- }
5771
- async recordExecution(queueName) {
5772
- const db = await this.getExecutionDb();
5773
- const tx = db.transaction(this.executionTableName, "readwrite");
5774
- const store = tx.objectStore(this.executionTableName);
5775
- const record = {
5776
- id: crypto.randomUUID(),
5777
- queue_name: queueName,
5778
- executed_at: new Date().toISOString()
5779
- };
5780
- for (const [key, value] of Object.entries(this.prefixValues)) {
5781
- record[key] = value;
5782
- }
5783
- return new Promise((resolve, reject) => {
5784
- const request = store.add(record);
5785
- tx.oncomplete = () => resolve();
5786
- tx.onerror = () => reject(tx.error);
5787
- request.onerror = () => reject(request.error);
5788
- });
5789
- }
5790
- async getExecutionCount(queueName, windowStartTime) {
5791
- const db = await this.getExecutionDb();
5792
- const tx = db.transaction(this.executionTableName, "readonly");
5793
- const store = tx.objectStore(this.executionTableName);
5794
- const index = store.index("queue_executed_at");
5795
- const prefixKeyValues = this.getPrefixKeyValues();
5796
- return new Promise((resolve, reject) => {
5797
- let count = 0;
5798
- const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartTime], [...prefixKeyValues, queueName, "￿"], true, false);
5799
- const request = index.openCursor(keyRange);
5800
- request.onsuccess = (event) => {
5801
- const cursor = event.target.result;
5802
- if (cursor) {
5803
- const record = cursor.value;
5804
- if (this.matchesPrefixes(record)) {
5805
- count++;
5806
- }
5807
- cursor.continue();
5808
- }
5809
- };
5810
- tx.oncomplete = () => resolve(count);
5811
- tx.onerror = () => reject(tx.error);
5812
- request.onerror = () => reject(request.error);
5813
- });
5814
- }
5815
- async getOldestExecutionAtOffset(queueName, offset) {
5816
- const db = await this.getExecutionDb();
5817
- const tx = db.transaction(this.executionTableName, "readonly");
5818
- const store = tx.objectStore(this.executionTableName);
5819
- const index = store.index("queue_executed_at");
5820
- const prefixKeyValues = this.getPrefixKeyValues();
5821
- return new Promise((resolve, reject) => {
5822
- const executions = [];
5823
- const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, "￿"]);
5824
- const request = index.openCursor(keyRange);
5825
- request.onsuccess = (event) => {
5826
- const cursor = event.target.result;
5827
- if (cursor) {
5828
- const record = cursor.value;
5829
- if (this.matchesPrefixes(record)) {
5830
- executions.push(record.executed_at);
5831
- }
5832
- cursor.continue();
5833
- }
5834
- };
5835
- tx.oncomplete = () => {
5836
- executions.sort();
5837
- resolve(executions[offset]);
5838
- };
5839
- tx.onerror = () => reject(tx.error);
5840
- request.onerror = () => reject(request.error);
5841
- });
5842
- }
5843
- async getNextAvailableTime(queueName) {
5844
- const db = await this.getNextAvailableDb();
5845
- const tx = db.transaction(this.nextAvailableTableName, "readonly");
5846
- const store = tx.objectStore(this.nextAvailableTableName);
5847
- const prefixKeyValues = this.getPrefixKeyValues();
5848
- const key = [...prefixKeyValues, queueName].join("_");
5849
- return new Promise((resolve, reject) => {
5850
- const request = store.get(key);
5851
- request.onsuccess = () => {
5852
- const record = request.result;
5853
- if (record && this.matchesPrefixes(record)) {
5854
- resolve(record.next_available_at);
5855
- } else {
5856
- resolve(undefined);
5857
- }
5858
- };
5859
- request.onerror = () => reject(request.error);
5860
- tx.onerror = () => reject(tx.error);
5861
- });
5862
- }
5863
- async setNextAvailableTime(queueName, nextAvailableAt) {
5864
- const db = await this.getNextAvailableDb();
5865
- const tx = db.transaction(this.nextAvailableTableName, "readwrite");
5866
- const store = tx.objectStore(this.nextAvailableTableName);
5867
- const prefixKeyValues = this.getPrefixKeyValues();
5868
- const key = [...prefixKeyValues, queueName].join("_");
5869
- const record = {
5870
- queue_name: queueName,
5871
- next_available_at: nextAvailableAt
5872
- };
5873
- for (const [k, value] of Object.entries(this.prefixValues)) {
5874
- record[k] = value;
5875
- }
5876
- record[this.getPrefixColumnNames().concat(["queue_name"]).join("_")] = key;
5877
- return new Promise((resolve, reject) => {
5878
- const request = store.put(record);
5879
- tx.oncomplete = () => resolve();
5880
- tx.onerror = () => reject(tx.error);
5881
- request.onerror = () => reject(request.error);
5882
- });
5883
- }
5884
- async clear(queueName) {
5885
- const execDb = await this.getExecutionDb();
5886
- const execTx = execDb.transaction(this.executionTableName, "readwrite");
5887
- const execStore = execTx.objectStore(this.executionTableName);
5888
- const execIndex = execStore.index("queue_executed_at");
5889
- const prefixKeyValues = this.getPrefixKeyValues();
5890
- await new Promise((resolve, reject) => {
5891
- const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, "￿"]);
5892
- const request = execIndex.openCursor(keyRange);
5893
- request.onsuccess = (event) => {
5894
- const cursor = event.target.result;
5895
- if (cursor) {
5896
- const record = cursor.value;
5897
- if (this.matchesPrefixes(record)) {
5898
- cursor.delete();
5899
- }
5900
- cursor.continue();
5901
- }
5902
- };
5903
- execTx.oncomplete = () => resolve();
5904
- execTx.onerror = () => reject(execTx.error);
5905
- request.onerror = () => reject(request.error);
5906
- });
5907
- const nextDb = await this.getNextAvailableDb();
5908
- const nextTx = nextDb.transaction(this.nextAvailableTableName, "readwrite");
5909
- const nextStore = nextTx.objectStore(this.nextAvailableTableName);
5910
- const key = [...prefixKeyValues, queueName].join("_");
5911
- await new Promise((resolve, reject) => {
5912
- const request = nextStore.delete(key);
5913
- nextTx.oncomplete = () => resolve();
5914
- nextTx.onerror = () => reject(nextTx.error);
5915
- request.onerror = () => reject(request.error);
5916
- });
5917
- }
5918
- }
5919
- // src/queue-limiter/SupabaseRateLimiterStorage.ts
5920
- import { createServiceToken as createServiceToken20 } from "@workglow/util";
5921
- var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken20("ratelimiter.storage.supabase");
5922
-
5923
- class SupabaseRateLimiterStorage {
5924
- scope = "cluster";
5925
- client;
5926
- prefixes;
5927
- prefixValues;
5928
- executionTableName;
5929
- nextAvailableTableName;
5930
- constructor(client, options) {
5931
- this.client = client;
5932
- this.prefixes = options?.prefixes ?? [];
5933
- this.prefixValues = options?.prefixValues ?? {};
5934
- if (this.prefixes.length > 0) {
5935
- const prefixNames = this.prefixes.map((p) => p.name).join("_");
5936
- this.executionTableName = `rate_limit_executions_${prefixNames}`;
5937
- this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
5938
- } else {
5939
- this.executionTableName = "rate_limit_executions";
5940
- this.nextAvailableTableName = "rate_limit_next_available";
5941
- }
5942
- }
5943
- getPrefixColumnType(type) {
5944
- return type === "uuid" ? "UUID" : "INTEGER";
5945
- }
5946
- buildPrefixColumnsSql() {
5947
- if (this.prefixes.length === 0)
5948
- return "";
5949
- return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
5950
- `) + `,
5951
- `;
5952
- }
5953
- getPrefixColumnNames() {
5954
- return this.prefixes.map((p) => p.name);
5955
- }
5956
- applyPrefixFilters(query) {
5957
- let result = query;
5958
- for (const prefix of this.prefixes) {
5959
- result = result.eq(prefix.name, this.prefixValues[prefix.name]);
5960
- }
5961
- return result;
5962
- }
5963
- getPrefixInsertValues() {
5964
- const values = {};
5965
- for (const prefix of this.prefixes) {
5966
- values[prefix.name] = this.prefixValues[prefix.name];
5967
- }
5968
- return values;
5969
- }
5970
- async setupDatabase() {
5971
- const prefixColumnsSql = this.buildPrefixColumnsSql();
5972
- const prefixColumnNames = this.getPrefixColumnNames();
5973
- const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
5974
- const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
5975
- const createExecTableSql = `
5976
- CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
5977
- id SERIAL PRIMARY KEY,
5978
- ${prefixColumnsSql}queue_name TEXT NOT NULL,
5979
- executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
5980
- )
5981
- `;
5982
- const { error: execTableError } = await this.client.rpc("exec_sql", {
5983
- query: createExecTableSql
5984
- });
5985
- if (execTableError && execTableError.code !== "42P07") {
5986
- throw execTableError;
5987
- }
5988
- const createExecIndexSql = `
5989
- CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
5990
- ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
5991
- `;
5992
- await this.client.rpc("exec_sql", { query: createExecIndexSql });
5993
- const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
5994
- const createNextTableSql = `
5995
- CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
5996
- ${prefixColumnsSql}queue_name TEXT NOT NULL,
5997
- next_available_at TIMESTAMP WITH TIME ZONE,
5998
- PRIMARY KEY (${primaryKeyColumns})
5999
- )
6000
- `;
6001
- const { error: nextTableError } = await this.client.rpc("exec_sql", {
6002
- query: createNextTableSql
6003
- });
6004
- if (nextTableError && nextTableError.code !== "42P07") {
6005
- throw nextTableError;
6006
- }
6007
- const fnName = this.atomicReserveFunctionName();
6008
- const prefixSig = this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)}`).join(", ");
6009
- const prefixSigPrefix = prefixSig ? prefixSig + ", " : "";
6010
- const prefixWhere = this.prefixes.length > 0 ? " AND " + this.prefixes.map((p) => `${p.name} = _${p.name}`).join(" AND ") : "";
6011
- const prefixInsertCols = this.prefixes.length > 0 ? this.prefixes.map((p) => p.name).join(", ") + ", " : "";
6012
- const prefixInsertVals = this.prefixes.length > 0 ? this.prefixes.map((p) => `_${p.name}`).join(", ") + ", " : "";
6013
- const lockKeyParts = [`'${this.executionTableName}'`, ...this.prefixes.map((p) => `_${p.name}::text`), `_queue_name::text`];
6014
- const lockKeyExpr = `hashtextextended(${lockKeyParts.join(" || '|' || ")}, 0)`;
6015
- const createFnSql = `
6016
- CREATE OR REPLACE FUNCTION ${fnName}(
6017
- ${prefixSigPrefix}_queue_name TEXT, _window_start TIMESTAMPTZ, _max_exec INT
6018
- ) RETURNS BIGINT AS $fn$
6019
- DECLARE
6020
- _count INT;
6021
- _next TIMESTAMPTZ;
6022
- _new_id BIGINT;
6023
- BEGIN
6024
- PERFORM pg_advisory_xact_lock(${lockKeyExpr});
6025
- SELECT COUNT(*) INTO _count FROM ${this.executionTableName}
6026
- WHERE queue_name = _queue_name AND executed_at > _window_start${prefixWhere};
6027
- IF _count >= _max_exec THEN RETURN NULL; END IF;
6028
- SELECT next_available_at INTO _next FROM ${this.nextAvailableTableName}
6029
- WHERE queue_name = _queue_name${prefixWhere};
6030
- IF _next IS NOT NULL AND _next > NOW() THEN RETURN NULL; END IF;
6031
- INSERT INTO ${this.executionTableName} (${prefixInsertCols}queue_name)
6032
- VALUES (${prefixInsertVals}_queue_name)
6033
- RETURNING id INTO _new_id;
6034
- RETURN _new_id;
6035
- END;
6036
- $fn$ LANGUAGE plpgsql;
6037
- `;
6038
- const { error: fnError } = await this.client.rpc("exec_sql", { query: createFnSql });
6039
- if (fnError) {
6040
- throw fnError;
6041
- }
6042
- }
6043
- atomicReserveFunctionName() {
6044
- return `${this.executionTableName}_try_reserve`.slice(0, 63);
6045
- }
6046
- async tryReserveExecution(queueName, maxExecutions, windowMs) {
6047
- const args = {
6048
- _queue_name: queueName,
6049
- _window_start: new Date(Date.now() - windowMs).toISOString(),
6050
- _max_exec: maxExecutions
6051
- };
6052
- for (const p of this.prefixes) {
6053
- args[`_${p.name}`] = this.prefixValues[p.name];
6054
- }
6055
- const { data, error } = await this.client.rpc(this.atomicReserveFunctionName(), args);
6056
- if (error)
6057
- throw error;
6058
- if (data === null || data === undefined)
6059
- return null;
6060
- if (Array.isArray(data)) {
6061
- if (data.length === 0)
6062
- return null;
6063
- const first = Object.values(data[0])[0];
6064
- return first ?? null;
6065
- }
6066
- return data;
6067
- }
6068
- async releaseExecution(queueName, token) {
6069
- if (token === null || token === undefined)
6070
- return;
6071
- let del = this.client.from(this.executionTableName).delete().eq("id", token).eq("queue_name", queueName);
6072
- del = this.applyPrefixFilters(del);
6073
- const { error: delError } = await del;
6074
- if (delError)
6075
- throw delError;
6076
- }
6077
- async recordExecution(queueName) {
6078
- const prefixInsertValues = this.getPrefixInsertValues();
6079
- const { error } = await this.client.from(this.executionTableName).insert({
6080
- ...prefixInsertValues,
6081
- queue_name: queueName
6082
- });
6083
- if (error)
6084
- throw error;
6085
- }
6086
- async getExecutionCount(queueName, windowStartTime) {
6087
- let query = this.client.from(this.executionTableName).select("*", { count: "exact", head: true }).eq("queue_name", queueName).gt("executed_at", windowStartTime);
6088
- query = this.applyPrefixFilters(query);
6089
- const { count, error } = await query;
6090
- if (error)
6091
- throw error;
6092
- return count ?? 0;
6093
- }
6094
- async getOldestExecutionAtOffset(queueName, offset) {
6095
- let query = this.client.from(this.executionTableName).select("executed_at").eq("queue_name", queueName);
6096
- query = this.applyPrefixFilters(query);
6097
- const { data, error } = await query.order("executed_at", { ascending: true }).range(offset, offset);
6098
- if (error)
6099
- throw error;
6100
- if (!data || data.length === 0)
6101
- return;
6102
- return new Date(data[0].executed_at).toISOString();
6103
- }
6104
- async getNextAvailableTime(queueName) {
6105
- let query = this.client.from(this.nextAvailableTableName).select("next_available_at").eq("queue_name", queueName);
6106
- query = this.applyPrefixFilters(query);
6107
- const { data, error } = await query.single();
6108
- if (error) {
6109
- if (error.code === "PGRST116")
6110
- return;
6111
- throw error;
6112
- }
6113
- if (!data?.next_available_at)
6114
- return;
6115
- return new Date(data.next_available_at).toISOString();
6116
- }
6117
- async setNextAvailableTime(queueName, nextAvailableAt) {
6118
- const prefixInsertValues = this.getPrefixInsertValues();
6119
- const { error } = await this.client.from(this.nextAvailableTableName).upsert({
6120
- ...prefixInsertValues,
6121
- queue_name: queueName,
6122
- next_available_at: nextAvailableAt
6123
- }, {
6124
- onConflict: this.prefixes.length > 0 ? `${this.getPrefixColumnNames().join(",")},queue_name` : "queue_name"
6125
- });
6126
- if (error)
6127
- throw error;
6128
- }
6129
- async clear(queueName) {
6130
- let execQuery = this.client.from(this.executionTableName).delete().eq("queue_name", queueName);
6131
- execQuery = this.applyPrefixFilters(execQuery);
6132
- const { error: execError } = await execQuery;
6133
- if (execError)
6134
- throw execError;
6135
- let nextQuery = this.client.from(this.nextAvailableTableName).delete().eq("queue_name", queueName);
6136
- nextQuery = this.applyPrefixFilters(nextQuery);
6137
- const { error: nextError } = await nextQuery;
6138
- if (nextError)
6139
- throw nextError;
6140
- }
6141
- }
6142
- // src/vector/IndexedDbVectorStorage.ts
6143
- import { createServiceToken as createServiceToken21 } from "@workglow/util";
6144
- import { cosineSimilarity as cosineSimilarity2 } from "@workglow/util/schema";
6145
- var IDB_VECTOR_REPOSITORY = createServiceToken21("storage.vectorRepository.indexedDb");
6146
- function matchesFilter2(metadata, filter) {
6147
- for (const [key, value] of Object.entries(filter)) {
6148
- if (metadata[key] !== value) {
6149
- return false;
6150
- }
6151
- }
6152
- return true;
6153
- }
6154
- function textRelevance2(text, query) {
6155
- const textLower = text.toLowerCase();
6156
- const queryLower = query.toLowerCase();
6157
- const queryWords = queryLower.split(/\s+/).filter((w) => w.length > 0);
6158
- if (queryWords.length === 0) {
6159
- return 0;
6160
- }
6161
- let matches = 0;
6162
- for (const word of queryWords) {
6163
- if (textLower.includes(word)) {
6164
- matches++;
6165
- }
6166
- }
6167
- return matches / queryWords.length;
6168
- }
6169
-
6170
- class IndexedDbVectorStorage extends IndexedDbTabularStorage {
6171
- vectorDimensions;
6172
- vectorPropertyName;
6173
- metadataPropertyName;
6174
- constructor(table = "vectors", schema, primaryKeyNames, indexes = [], dimensions, _vectorCtor = Float32Array, migrationOptions = {}, clientProvidedKeys = "if-missing") {
6175
- super(table, schema, primaryKeyNames, indexes, migrationOptions, clientProvidedKeys);
6176
- this.vectorDimensions = dimensions;
6177
- const vectorProp = getVectorProperty(schema);
6178
- if (!vectorProp) {
6179
- throw new Error("Schema must have a property with type array and format TypedArray");
6180
- }
6181
- this.vectorPropertyName = vectorProp;
6182
- this.metadataPropertyName = getMetadataProperty(schema);
6183
- }
6184
- getVectorDimensions() {
6185
- return this.vectorDimensions;
6186
- }
6187
- async similaritySearch(query, options = {}) {
6188
- const { topK = 10, filter, scoreThreshold = 0 } = options;
6189
- const results = [];
6190
- const allEntities = await this.getAll() || [];
6191
- for (const entity of allEntities) {
6192
- const vector = entity[this.vectorPropertyName];
6193
- const metadata = this.metadataPropertyName ? entity[this.metadataPropertyName] : {};
6194
- if (filter && !matchesFilter2(metadata, filter)) {
6195
- continue;
6196
- }
6197
- const score = cosineSimilarity2(query, vector);
6198
- if (score < scoreThreshold) {
6199
- continue;
6200
- }
6201
- results.push({
6202
- ...entity,
6203
- score
6204
- });
6205
- }
6206
- results.sort((a, b) => b.score - a.score);
6207
- const topResults = results.slice(0, topK);
6208
- return topResults;
6209
- }
6210
- async hybridSearch(query, options) {
6211
- const { topK = 10, filter, scoreThreshold = 0, textQuery, vectorWeight = 0.7 } = options;
6212
- if (!textQuery || textQuery.trim().length === 0) {
6213
- return this.similaritySearch(query, { topK, filter, scoreThreshold });
6214
- }
6215
- const results = [];
6216
- const allEntities = await this.getAll() || [];
6217
- for (const entity of allEntities) {
6218
- const vector = entity[this.vectorPropertyName];
6219
- const metadata = this.metadataPropertyName ? entity[this.metadataPropertyName] : {};
6220
- if (filter && !matchesFilter2(metadata, filter)) {
6221
- continue;
6222
- }
6223
- const vectorScore = cosineSimilarity2(query, vector);
6224
- const metadataText = Object.values(metadata).join(" ").toLowerCase();
6225
- const textScore = textRelevance2(metadataText, textQuery);
6226
- const combinedScore = vectorWeight * vectorScore + (1 - vectorWeight) * textScore;
6227
- if (combinedScore < scoreThreshold) {
6228
- continue;
6229
- }
6230
- results.push({
6231
- ...entity,
6232
- score: combinedScore
6233
- });
6234
- }
6235
- results.sort((a, b) => b.score - a.score);
6236
- const topResults = results.slice(0, topK);
6237
- return topResults;
6238
- }
6239
- }
6240
2467
  export {
6241
- traced,
6242
2468
  registerTabularRepository,
6243
2469
  pickCoveringIndex,
6244
2470
  isSearchCondition,
@@ -6246,18 +2472,11 @@ export {
6246
2472
  getTabularRepository,
6247
2473
  getMetadataProperty,
6248
2474
  getGlobalTabularRepositories,
6249
- ensureIndexedDbTable,
6250
- dropIndexedDbTable,
6251
2475
  TelemetryVectorStorage,
6252
2476
  TelemetryTabularStorage,
6253
- TelemetryQueueStorage,
6254
2477
  TelemetryKvStorage,
6255
2478
  TABULAR_REPOSITORY,
6256
2479
  TABULAR_REPOSITORIES,
6257
- SupabaseTabularStorage,
6258
- SupabaseRateLimiterStorage,
6259
- SupabaseQueueStorage,
6260
- SupabaseKvStorage,
6261
2480
  StorageValidationError,
6262
2481
  StorageUnsupportedError,
6263
2482
  StorageInvalidLimitError,
@@ -6265,13 +2484,7 @@ export {
6265
2484
  StorageError,
6266
2485
  StorageEmptyCriteriaError,
6267
2486
  SharedInMemoryTabularStorage,
6268
- SUPABASE_TABULAR_REPOSITORY,
6269
- SUPABASE_RATE_LIMITER_STORAGE,
6270
- SUPABASE_QUEUE_STORAGE,
6271
- SUPABASE_KV_REPOSITORY,
6272
2487
  SHARED_IN_MEMORY_TABULAR_REPOSITORY,
6273
- RATE_LIMITER_STORAGE,
6274
- QUEUE_STORAGE,
6275
2488
  PollingSubscriptionManager,
6276
2489
  MEMORY_TABULAR_REPOSITORY,
6277
2490
  MEMORY_KV_REPOSITORY,
@@ -6279,24 +2492,9 @@ export {
6279
2492
  KvViaTabularStorage,
6280
2493
  KvStorage,
6281
2494
  KV_REPOSITORY,
6282
- JobStatus,
6283
- IndexedDbVectorStorage,
6284
- IndexedDbTabularStorage,
6285
- IndexedDbRateLimiterStorage,
6286
- IndexedDbQueueStorage,
6287
- IndexedDbKvStorage,
6288
2495
  InMemoryVectorStorage,
6289
2496
  InMemoryTabularStorage,
6290
- InMemoryRateLimiterStorage,
6291
- InMemoryQueueStorage,
6292
2497
  InMemoryKvStorage,
6293
- IN_MEMORY_RATE_LIMITER_STORAGE,
6294
- IN_MEMORY_QUEUE_STORAGE,
6295
- INDEXED_DB_RATE_LIMITER_STORAGE,
6296
- INDEXED_DB_QUEUE_STORAGE,
6297
- IDB_VECTOR_REPOSITORY,
6298
- IDB_TABULAR_REPOSITORY,
6299
- IDB_KV_REPOSITORY,
6300
2498
  HybridSubscriptionManager,
6301
2499
  HuggingFaceTabularStorage,
6302
2500
  HF_TABULAR_REPOSITORY,
@@ -6306,7 +2504,8 @@ export {
6306
2504
  CoveringIndexMissingError,
6307
2505
  CachedTabularStorage,
6308
2506
  CACHED_TABULAR_REPOSITORY,
6309
- BaseTabularStorage
2507
+ BaseTabularStorage,
2508
+ BaseSqlTabularStorage
6310
2509
  };
6311
2510
 
6312
- //# debugId=E2D716C3197417F164756E2164756E21
2511
+ //# debugId=23715A15DDE7420E64756E2164756E21