@workglow/storage 0.2.27 → 0.2.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.d.ts +0 -10
- package/dist/browser.d.ts.map +1 -1
- package/dist/browser.js +406 -4207
- package/dist/browser.js.map +8 -24
- package/dist/bun.js +341 -7235
- package/dist/bun.js.map +8 -35
- package/dist/common-server.d.ts +0 -21
- package/dist/common-server.d.ts.map +1 -1
- package/dist/common.d.ts +1 -6
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +341 -7236
- package/dist/node.js.map +8 -35
- package/package.json +5 -71
- package/dist/kv/IndexedDbKvStorage.d.ts +0 -27
- package/dist/kv/IndexedDbKvStorage.d.ts.map +0 -1
- package/dist/kv/PostgresKvStorage.d.ts +0 -28
- package/dist/kv/PostgresKvStorage.d.ts.map +0 -1
- package/dist/kv/SqliteKvStorage.d.ts +0 -28
- package/dist/kv/SqliteKvStorage.d.ts.map +0 -1
- package/dist/kv/SupabaseKvStorage.d.ts +0 -33
- package/dist/kv/SupabaseKvStorage.d.ts.map +0 -1
- package/dist/postgres/browser.d.ts +0 -32
- package/dist/postgres/browser.d.ts.map +0 -1
- package/dist/postgres/browser.js +0 -150
- package/dist/postgres/browser.js.map +0 -11
- package/dist/postgres/node-bun.d.ts +0 -26
- package/dist/postgres/node-bun.d.ts.map +0 -1
- package/dist/postgres/node-bun.js +0 -41
- package/dist/postgres/node-bun.js.map +0 -10
- package/dist/postgres/pglite-pool.d.ts +0 -21
- package/dist/postgres/pglite-pool.d.ts.map +0 -1
- package/dist/queue/IQueueStorage.d.ts +0 -229
- package/dist/queue/IQueueStorage.d.ts.map +0 -1
- package/dist/queue/InMemoryQueueStorage.d.ts +0 -149
- package/dist/queue/InMemoryQueueStorage.d.ts.map +0 -1
- package/dist/queue/IndexedDbQueueStorage.d.ts +0 -166
- package/dist/queue/IndexedDbQueueStorage.d.ts.map +0 -1
- package/dist/queue/PostgresQueueStorage.d.ts +0 -154
- package/dist/queue/PostgresQueueStorage.d.ts.map +0 -1
- package/dist/queue/SqliteQueueStorage.d.ts +0 -149
- package/dist/queue/SqliteQueueStorage.d.ts.map +0 -1
- package/dist/queue/SupabaseQueueStorage.d.ts +0 -195
- package/dist/queue/SupabaseQueueStorage.d.ts.map +0 -1
- package/dist/queue/TelemetryQueueStorage.d.ts +0 -33
- package/dist/queue/TelemetryQueueStorage.d.ts.map +0 -1
- package/dist/queue-limiter/IRateLimiterStorage.d.ts +0 -127
- package/dist/queue-limiter/IRateLimiterStorage.d.ts.map +0 -1
- package/dist/queue-limiter/InMemoryRateLimiterStorage.d.ts +0 -43
- package/dist/queue-limiter/InMemoryRateLimiterStorage.d.ts.map +0 -1
- package/dist/queue-limiter/IndexedDbRateLimiterStorage.d.ts +0 -79
- package/dist/queue-limiter/IndexedDbRateLimiterStorage.d.ts.map +0 -1
- package/dist/queue-limiter/PostgresRateLimiterStorage.d.ts +0 -57
- package/dist/queue-limiter/PostgresRateLimiterStorage.d.ts.map +0 -1
- package/dist/queue-limiter/SqliteRateLimiterStorage.d.ts +0 -62
- package/dist/queue-limiter/SqliteRateLimiterStorage.d.ts.map +0 -1
- package/dist/queue-limiter/SupabaseRateLimiterStorage.d.ts +0 -54
- package/dist/queue-limiter/SupabaseRateLimiterStorage.d.ts.map +0 -1
- package/dist/sqlite/browser.d.ts +0 -37
- package/dist/sqlite/browser.d.ts.map +0 -1
- package/dist/sqlite/browser.js +0 -125
- package/dist/sqlite/browser.js.map +0 -10
- package/dist/sqlite/bun.d.ts +0 -32
- package/dist/sqlite/bun.d.ts.map +0 -1
- package/dist/sqlite/bun.js +0 -84
- package/dist/sqlite/bun.js.map +0 -10
- package/dist/sqlite/canonical-api.d.ts +0 -34
- package/dist/sqlite/canonical-api.d.ts.map +0 -1
- package/dist/sqlite/node.d.ts +0 -34
- package/dist/sqlite/node.d.ts.map +0 -1
- package/dist/sqlite/node.js +0 -65
- package/dist/sqlite/node.js.map +0 -10
- package/dist/tabular/IndexedDbTabularStorage.d.ts +0 -199
- package/dist/tabular/IndexedDbTabularStorage.d.ts.map +0 -1
- package/dist/tabular/PostgresTabularStorage.d.ts +0 -196
- package/dist/tabular/PostgresTabularStorage.d.ts.map +0 -1
- package/dist/tabular/SqliteTabularStorage.d.ts +0 -167
- package/dist/tabular/SqliteTabularStorage.d.ts.map +0 -1
- package/dist/tabular/SupabaseTabularStorage.d.ts +0 -174
- package/dist/tabular/SupabaseTabularStorage.d.ts.map +0 -1
- package/dist/util/IndexedDbTable.d.ts +0 -40
- package/dist/util/IndexedDbTable.d.ts.map +0 -1
- package/dist/util/traced.d.ts +0 -10
- package/dist/util/traced.d.ts.map +0 -1
- package/dist/vector/IndexedDbVectorStorage.d.ts +0 -53
- package/dist/vector/IndexedDbVectorStorage.d.ts.map +0 -1
- package/dist/vector/PostgresVectorStorage.d.ts +0 -39
- package/dist/vector/PostgresVectorStorage.d.ts.map +0 -1
- package/dist/vector/SqliteAiVectorStorage.d.ts +0 -100
- package/dist/vector/SqliteAiVectorStorage.d.ts.map +0 -1
- package/dist/vector/SqliteVectorStorage.d.ts +0 -49
- package/dist/vector/SqliteVectorStorage.d.ts.map +0 -1
- package/src/queue/README.md +0 -41
package/dist/browser.js
CHANGED
|
@@ -414,6 +414,205 @@ class BaseTabularStorage {
|
|
|
414
414
|
this.destroy();
|
|
415
415
|
}
|
|
416
416
|
}
|
|
417
|
+
// src/tabular/BaseSqlTabularStorage.ts
|
|
418
|
+
class BaseSqlTabularStorage extends BaseTabularStorage {
|
|
419
|
+
table;
|
|
420
|
+
_pkColsCache = new Map;
|
|
421
|
+
_valColsCache = new Map;
|
|
422
|
+
_pkColListCache = new Map;
|
|
423
|
+
_valColListCache = new Map;
|
|
424
|
+
constructor(table = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
|
|
425
|
+
super(schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
426
|
+
this.table = table;
|
|
427
|
+
this.validateTableAndSchema();
|
|
428
|
+
}
|
|
429
|
+
constructPrimaryKeyColumns($delimiter = "") {
|
|
430
|
+
let cached = this._pkColsCache.get($delimiter);
|
|
431
|
+
if (cached === undefined) {
|
|
432
|
+
cached = Object.entries(this.primaryKeySchema.properties).map(([key, typeDef]) => {
|
|
433
|
+
const sqlType = this.mapTypeToSQL(typeDef);
|
|
434
|
+
return `${$delimiter}${key}${$delimiter} ${sqlType} NOT NULL`;
|
|
435
|
+
}).join(", ");
|
|
436
|
+
this._pkColsCache.set($delimiter, cached);
|
|
437
|
+
}
|
|
438
|
+
return cached;
|
|
439
|
+
}
|
|
440
|
+
constructValueColumns($delimiter = "") {
|
|
441
|
+
let cached = this._valColsCache.get($delimiter);
|
|
442
|
+
if (cached === undefined) {
|
|
443
|
+
const requiredSet = new Set(this.valueSchema.required ?? []);
|
|
444
|
+
const cols = Object.entries(this.valueSchema.properties).map(([key, typeDef]) => {
|
|
445
|
+
const sqlType = this.mapTypeToSQL(typeDef);
|
|
446
|
+
const isRequired = requiredSet.has(key);
|
|
447
|
+
const nullable = !isRequired || this.isNullable(typeDef);
|
|
448
|
+
return `${$delimiter}${key}${$delimiter} ${sqlType}${nullable ? " NULL" : " NOT NULL"}`;
|
|
449
|
+
}).join(", ");
|
|
450
|
+
cached = cols.length > 0 ? `, ${cols}` : "";
|
|
451
|
+
this._valColsCache.set($delimiter, cached);
|
|
452
|
+
}
|
|
453
|
+
return cached;
|
|
454
|
+
}
|
|
455
|
+
isNullable(typeDef) {
|
|
456
|
+
if (typeof typeDef === "boolean")
|
|
457
|
+
return typeDef;
|
|
458
|
+
if (typeDef.type === "null") {
|
|
459
|
+
return true;
|
|
460
|
+
}
|
|
461
|
+
if (Array.isArray(typeDef.type)) {
|
|
462
|
+
return typeDef.type.includes("null");
|
|
463
|
+
}
|
|
464
|
+
if (typeDef.anyOf && Array.isArray(typeDef.anyOf)) {
|
|
465
|
+
return typeDef.anyOf.some((type) => type.type === "null");
|
|
466
|
+
}
|
|
467
|
+
if (typeDef.oneOf && Array.isArray(typeDef.oneOf)) {
|
|
468
|
+
return typeDef.oneOf.some((type) => type.type === "null");
|
|
469
|
+
}
|
|
470
|
+
return false;
|
|
471
|
+
}
|
|
472
|
+
primaryKeyColumnList($delimiter = "") {
|
|
473
|
+
let cached = this._pkColListCache.get($delimiter);
|
|
474
|
+
if (cached === undefined) {
|
|
475
|
+
cached = $delimiter + this.primaryKeyColumns().join(`${$delimiter}, ${$delimiter}`) + $delimiter;
|
|
476
|
+
this._pkColListCache.set($delimiter, cached);
|
|
477
|
+
}
|
|
478
|
+
return cached;
|
|
479
|
+
}
|
|
480
|
+
valueColumnList($delimiter = "") {
|
|
481
|
+
let cached = this._valColListCache.get($delimiter);
|
|
482
|
+
if (cached === undefined) {
|
|
483
|
+
cached = $delimiter + this.valueColumns().join(`${$delimiter}, ${$delimiter}`) + $delimiter;
|
|
484
|
+
this._valColListCache.set($delimiter, cached);
|
|
485
|
+
}
|
|
486
|
+
return cached;
|
|
487
|
+
}
|
|
488
|
+
getNonNullType(typeDef) {
|
|
489
|
+
if (typeof typeDef === "boolean")
|
|
490
|
+
return typeDef;
|
|
491
|
+
if (typeDef.anyOf && Array.isArray(typeDef.anyOf)) {
|
|
492
|
+
const nonNullType = typeDef.anyOf.find((t) => t.type !== "null");
|
|
493
|
+
if (nonNullType) {
|
|
494
|
+
return nonNullType;
|
|
495
|
+
}
|
|
496
|
+
}
|
|
497
|
+
if (typeDef.oneOf && Array.isArray(typeDef.oneOf)) {
|
|
498
|
+
const nonNullType = typeDef.oneOf.find((t) => t.type !== "null");
|
|
499
|
+
if (nonNullType) {
|
|
500
|
+
return nonNullType;
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
return typeDef;
|
|
504
|
+
}
|
|
505
|
+
getValueAsOrderedArray(value) {
|
|
506
|
+
const orderedParams = [];
|
|
507
|
+
const valueAsRecord = value;
|
|
508
|
+
const requiredSet = new Set(this.valueSchema.required ?? []);
|
|
509
|
+
for (const key in this.valueSchema.properties) {
|
|
510
|
+
if (Object.prototype.hasOwnProperty.call(valueAsRecord, key)) {
|
|
511
|
+
const val = valueAsRecord[key];
|
|
512
|
+
if (val === undefined && !requiredSet.has(key)) {
|
|
513
|
+
orderedParams.push(null);
|
|
514
|
+
} else {
|
|
515
|
+
orderedParams.push(this.jsToSqlValue(key, val));
|
|
516
|
+
}
|
|
517
|
+
} else {
|
|
518
|
+
if (requiredSet.has(key)) {
|
|
519
|
+
throw new Error(`Missing required value field: ${key}`);
|
|
520
|
+
}
|
|
521
|
+
orderedParams.push(null);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
return orderedParams;
|
|
525
|
+
}
|
|
526
|
+
getPrimaryKeyAsOrderedArray(key) {
|
|
527
|
+
const orderedParams = [];
|
|
528
|
+
const keyObj = key;
|
|
529
|
+
for (const k of Object.keys(this.primaryKeySchema.properties)) {
|
|
530
|
+
if (k in keyObj) {
|
|
531
|
+
const value = keyObj[k];
|
|
532
|
+
if (value === null) {
|
|
533
|
+
throw new Error(`Primary key field ${k} cannot be null`);
|
|
534
|
+
}
|
|
535
|
+
orderedParams.push(this.jsToSqlValue(k, value));
|
|
536
|
+
} else {
|
|
537
|
+
throw new Error(`Missing required primary key field: ${k}`);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
return orderedParams;
|
|
541
|
+
}
|
|
542
|
+
jsToSqlValue(column, value) {
|
|
543
|
+
const typeDef = this.schema.properties[column];
|
|
544
|
+
if (!typeDef) {
|
|
545
|
+
return value;
|
|
546
|
+
}
|
|
547
|
+
if (value === null && this.isNullable(typeDef)) {
|
|
548
|
+
return null;
|
|
549
|
+
}
|
|
550
|
+
const actualType = this.getNonNullType(typeDef);
|
|
551
|
+
if (typeof actualType === "boolean") {
|
|
552
|
+
return value;
|
|
553
|
+
}
|
|
554
|
+
if (actualType.contentEncoding === "blob") {
|
|
555
|
+
if (value instanceof Uint8Array) {
|
|
556
|
+
return value;
|
|
557
|
+
}
|
|
558
|
+
if (typeof Buffer !== "undefined" && value instanceof Buffer) {
|
|
559
|
+
return new Uint8Array(value);
|
|
560
|
+
}
|
|
561
|
+
if (Array.isArray(value)) {
|
|
562
|
+
return new Uint8Array(value);
|
|
563
|
+
}
|
|
564
|
+
return value;
|
|
565
|
+
} else if (value instanceof Date) {
|
|
566
|
+
return value.toISOString();
|
|
567
|
+
} else {
|
|
568
|
+
return value;
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
sqlToJsValue(column, value) {
|
|
572
|
+
const typeDef = this.schema.properties[column];
|
|
573
|
+
if (!typeDef) {
|
|
574
|
+
return value;
|
|
575
|
+
}
|
|
576
|
+
if (value === null && this.isNullable(typeDef)) {
|
|
577
|
+
return null;
|
|
578
|
+
}
|
|
579
|
+
const actualType = this.getNonNullType(typeDef);
|
|
580
|
+
if (typeof actualType === "boolean") {
|
|
581
|
+
return value;
|
|
582
|
+
}
|
|
583
|
+
if (actualType.contentEncoding === "blob") {
|
|
584
|
+
if (typeof Buffer !== "undefined" && value instanceof Buffer) {
|
|
585
|
+
return new Uint8Array(value);
|
|
586
|
+
}
|
|
587
|
+
if (value instanceof Uint8Array) {
|
|
588
|
+
return value;
|
|
589
|
+
}
|
|
590
|
+
return value;
|
|
591
|
+
} else {
|
|
592
|
+
return value;
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
validateTableAndSchema() {
|
|
596
|
+
if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(this.table)) {
|
|
597
|
+
throw new Error("Table name must start with a letter and contain only letters, digits, and underscores, got: " + this.table);
|
|
598
|
+
}
|
|
599
|
+
const validateSchemaKeys = (schema) => {
|
|
600
|
+
for (const key in schema.properties) {
|
|
601
|
+
if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(key)) {
|
|
602
|
+
throw new Error("Schema keys must start with a letter and contain only letters, digits, and underscores, got: " + key);
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
};
|
|
606
|
+
validateSchemaKeys(this.primaryKeySchema);
|
|
607
|
+
validateSchemaKeys(this.valueSchema);
|
|
608
|
+
const primaryKeys = new Set(Object.keys(this.primaryKeySchema.properties));
|
|
609
|
+
const valueKeys = Object.keys(this.valueSchema.properties);
|
|
610
|
+
const duplicates = valueKeys.filter((key) => primaryKeys.has(key));
|
|
611
|
+
if (duplicates.length > 0) {
|
|
612
|
+
throw new Error(`Duplicate keys found in schemas: ${duplicates.join(", ")}`);
|
|
613
|
+
}
|
|
614
|
+
}
|
|
615
|
+
}
|
|
417
616
|
// src/tabular/CachedTabularStorage.ts
|
|
418
617
|
import { createServiceToken as createServiceToken3, getLogger } from "@workglow/util";
|
|
419
618
|
|
|
@@ -1272,28 +1471,9 @@ registerInputCompactor("storage:tabular", (value, _format, registry) => {
|
|
|
1272
1471
|
}
|
|
1273
1472
|
return;
|
|
1274
1473
|
});
|
|
1275
|
-
// src/util/traced.ts
|
|
1276
|
-
import { getTelemetryProvider, SpanStatusCode } from "@workglow/util";
|
|
1277
|
-
async function traced(spanName, storageName, fn) {
|
|
1278
|
-
const telemetry = getTelemetryProvider();
|
|
1279
|
-
if (!telemetry.isEnabled)
|
|
1280
|
-
return fn();
|
|
1281
|
-
const span = telemetry.startSpan(spanName, {
|
|
1282
|
-
attributes: { "workglow.storage.name": storageName }
|
|
1283
|
-
});
|
|
1284
|
-
try {
|
|
1285
|
-
const result = await fn();
|
|
1286
|
-
span.setStatus(SpanStatusCode.OK);
|
|
1287
|
-
return result;
|
|
1288
|
-
} catch (err) {
|
|
1289
|
-
span.setStatus(SpanStatusCode.ERROR, err instanceof Error ? err.message : String(err));
|
|
1290
|
-
throw err;
|
|
1291
|
-
} finally {
|
|
1292
|
-
span.end();
|
|
1293
|
-
}
|
|
1294
|
-
}
|
|
1295
|
-
|
|
1296
1474
|
// src/tabular/TelemetryTabularStorage.ts
|
|
1475
|
+
import { traced } from "@workglow/util";
|
|
1476
|
+
|
|
1297
1477
|
class TelemetryTabularStorage {
|
|
1298
1478
|
storageName;
|
|
1299
1479
|
inner;
|
|
@@ -1500,6 +1680,8 @@ class InMemoryKvStorage extends KvViaTabularStorage {
|
|
|
1500
1680
|
}
|
|
1501
1681
|
}
|
|
1502
1682
|
// src/kv/TelemetryKvStorage.ts
|
|
1683
|
+
import { traced as traced2 } from "@workglow/util";
|
|
1684
|
+
|
|
1503
1685
|
class TelemetryKvStorage {
|
|
1504
1686
|
storageName;
|
|
1505
1687
|
inner;
|
|
@@ -1508,25 +1690,25 @@ class TelemetryKvStorage {
|
|
|
1508
1690
|
this.inner = inner;
|
|
1509
1691
|
}
|
|
1510
1692
|
put(key, value) {
|
|
1511
|
-
return
|
|
1693
|
+
return traced2("workglow.storage.kv.put", this.storageName, () => this.inner.put(key, value));
|
|
1512
1694
|
}
|
|
1513
1695
|
putBulk(items) {
|
|
1514
|
-
return
|
|
1696
|
+
return traced2("workglow.storage.kv.putBulk", this.storageName, () => this.inner.putBulk(items));
|
|
1515
1697
|
}
|
|
1516
1698
|
get(key) {
|
|
1517
|
-
return
|
|
1699
|
+
return traced2("workglow.storage.kv.get", this.storageName, () => this.inner.get(key));
|
|
1518
1700
|
}
|
|
1519
1701
|
delete(key) {
|
|
1520
|
-
return
|
|
1702
|
+
return traced2("workglow.storage.kv.delete", this.storageName, () => this.inner.delete(key));
|
|
1521
1703
|
}
|
|
1522
1704
|
getAll() {
|
|
1523
|
-
return
|
|
1705
|
+
return traced2("workglow.storage.kv.getAll", this.storageName, () => this.inner.getAll());
|
|
1524
1706
|
}
|
|
1525
1707
|
deleteAll() {
|
|
1526
|
-
return
|
|
1708
|
+
return traced2("workglow.storage.kv.deleteAll", this.storageName, () => this.inner.deleteAll());
|
|
1527
1709
|
}
|
|
1528
1710
|
size() {
|
|
1529
|
-
return
|
|
1711
|
+
return traced2("workglow.storage.kv.size", this.storageName, () => this.inner.size());
|
|
1530
1712
|
}
|
|
1531
1713
|
getObjectAsIdString(object) {
|
|
1532
1714
|
return this.inner.getObjectAsIdString(object);
|
|
@@ -1547,456 +1729,47 @@ class TelemetryKvStorage {
|
|
|
1547
1729
|
return this.inner.waitOn(name);
|
|
1548
1730
|
}
|
|
1549
1731
|
}
|
|
1550
|
-
// src/
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
class InMemoryQueueStorage {
|
|
1576
|
-
queueName;
|
|
1577
|
-
scope = "process";
|
|
1578
|
-
prefixValues;
|
|
1579
|
-
events = new EventEmitter3;
|
|
1580
|
-
constructor(queueName, options) {
|
|
1581
|
-
this.queueName = queueName;
|
|
1582
|
-
this.jobQueue = [];
|
|
1583
|
-
this.prefixValues = options?.prefixValues ?? {};
|
|
1584
|
-
}
|
|
1585
|
-
jobQueue;
|
|
1586
|
-
matchesPrefixes(job) {
|
|
1587
|
-
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
1588
|
-
if (job[key] !== value) {
|
|
1589
|
-
return false;
|
|
1590
|
-
}
|
|
1732
|
+
// src/util/HybridSubscriptionManager.ts
|
|
1733
|
+
class HybridSubscriptionManager {
|
|
1734
|
+
subscribers = new Set;
|
|
1735
|
+
lastKnownState = new Map;
|
|
1736
|
+
initialized = false;
|
|
1737
|
+
channel = null;
|
|
1738
|
+
backupPollingIntervalId = null;
|
|
1739
|
+
fetchState;
|
|
1740
|
+
compareItems;
|
|
1741
|
+
payloadFactory;
|
|
1742
|
+
options;
|
|
1743
|
+
hasBroadcastChannel;
|
|
1744
|
+
constructor(channelName, fetchState, compareItems, payloadFactory, options) {
|
|
1745
|
+
this.fetchState = fetchState;
|
|
1746
|
+
this.compareItems = compareItems;
|
|
1747
|
+
this.payloadFactory = payloadFactory;
|
|
1748
|
+
this.options = {
|
|
1749
|
+
defaultIntervalMs: options?.defaultIntervalMs ?? 1000,
|
|
1750
|
+
backupPollingIntervalMs: options?.backupPollingIntervalMs ?? 5000,
|
|
1751
|
+
useBroadcastChannel: options?.useBroadcastChannel ?? true,
|
|
1752
|
+
broadcastChannelName: options?.broadcastChannelName ?? channelName
|
|
1753
|
+
};
|
|
1754
|
+
this.hasBroadcastChannel = this.options.useBroadcastChannel && typeof BroadcastChannel !== "undefined";
|
|
1755
|
+
if (this.hasBroadcastChannel) {
|
|
1756
|
+
this.initializeBroadcastChannel();
|
|
1591
1757
|
}
|
|
1592
|
-
return true;
|
|
1593
1758
|
}
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid42();
|
|
1604
|
-
jobWithPrefixes.queue = this.queueName;
|
|
1605
|
-
jobWithPrefixes.fingerprint = await makeFingerprint4(jobWithPrefixes.input);
|
|
1606
|
-
jobWithPrefixes.status = JobStatus.PENDING;
|
|
1607
|
-
jobWithPrefixes.progress = 0;
|
|
1608
|
-
jobWithPrefixes.progress_message = "";
|
|
1609
|
-
jobWithPrefixes.progress_details = null;
|
|
1610
|
-
jobWithPrefixes.created_at = now;
|
|
1611
|
-
jobWithPrefixes.run_after = now;
|
|
1612
|
-
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
1613
|
-
jobWithPrefixes[key] = value;
|
|
1614
|
-
}
|
|
1615
|
-
this.jobQueue.push(jobWithPrefixes);
|
|
1616
|
-
this.events.emit("change", { type: "INSERT", new: jobWithPrefixes });
|
|
1617
|
-
return jobWithPrefixes.id;
|
|
1618
|
-
}
|
|
1619
|
-
async get(id) {
|
|
1620
|
-
await sleep(0);
|
|
1621
|
-
const job = this.jobQueue.find((j) => j.id === id);
|
|
1622
|
-
if (job && this.matchesPrefixes(job)) {
|
|
1623
|
-
return job;
|
|
1759
|
+
initializeBroadcastChannel() {
|
|
1760
|
+
try {
|
|
1761
|
+
this.channel = new BroadcastChannel(this.options.broadcastChannelName);
|
|
1762
|
+
this.channel.onmessage = (event) => {
|
|
1763
|
+
this.handleBroadcastMessage(event.data);
|
|
1764
|
+
};
|
|
1765
|
+
} catch (error) {
|
|
1766
|
+
console.error("Failed to initialize BroadcastChannel:", error);
|
|
1767
|
+
this.channel = null;
|
|
1624
1768
|
}
|
|
1625
|
-
return;
|
|
1626
1769
|
}
|
|
1627
|
-
async
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
return this.jobQueue.filter((j) => this.matchesPrefixes(j)).sort((a, b) => (a.run_after || "").localeCompare(b.run_after || "")).filter((j) => j.status === status).slice(0, num);
|
|
1631
|
-
}
|
|
1632
|
-
async next(workerId) {
|
|
1633
|
-
await sleep(0);
|
|
1634
|
-
const top = this.pendingQueue();
|
|
1635
|
-
const job = top[0];
|
|
1636
|
-
if (job) {
|
|
1637
|
-
const oldJob = { ...job };
|
|
1638
|
-
job.status = JobStatus.PROCESSING;
|
|
1639
|
-
job.last_ran_at = new Date().toISOString();
|
|
1640
|
-
job.worker_id = workerId;
|
|
1641
|
-
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1642
|
-
return job;
|
|
1643
|
-
}
|
|
1644
|
-
}
|
|
1645
|
-
async size(status = JobStatus.PENDING) {
|
|
1646
|
-
await sleep(0);
|
|
1647
|
-
return this.jobQueue.filter((j) => this.matchesPrefixes(j) && j.status === status).length;
|
|
1648
|
-
}
|
|
1649
|
-
async saveProgress(id, progress, message, details) {
|
|
1650
|
-
await sleep(0);
|
|
1651
|
-
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
1652
|
-
if (!job) {
|
|
1653
|
-
const jobWithAnyPrefix = this.jobQueue.find((j) => j.id === id);
|
|
1654
|
-
getLogger2().warn("Job not found for progress update", {
|
|
1655
|
-
id,
|
|
1656
|
-
reason: jobWithAnyPrefix ? "prefix_mismatch" : "missing",
|
|
1657
|
-
existingStatus: jobWithAnyPrefix?.status,
|
|
1658
|
-
queueName: this.queueName,
|
|
1659
|
-
prefixValues: this.prefixValues
|
|
1660
|
-
});
|
|
1661
|
-
return;
|
|
1662
|
-
}
|
|
1663
|
-
if (job.status === JobStatus.COMPLETED || job.status === JobStatus.FAILED) {
|
|
1664
|
-
getLogger2().warn("Job already completed or failed for progress update", {
|
|
1665
|
-
id,
|
|
1666
|
-
status: job.status,
|
|
1667
|
-
completedAt: job.completed_at,
|
|
1668
|
-
error: job.error
|
|
1669
|
-
});
|
|
1670
|
-
return;
|
|
1671
|
-
}
|
|
1672
|
-
const oldJob = { ...job };
|
|
1673
|
-
job.progress = progress;
|
|
1674
|
-
job.progress_message = message;
|
|
1675
|
-
job.progress_details = details;
|
|
1676
|
-
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1677
|
-
}
|
|
1678
|
-
async complete(job) {
|
|
1679
|
-
await sleep(0);
|
|
1680
|
-
const jobWithPrefixes = job;
|
|
1681
|
-
const index = this.jobQueue.findIndex((j) => j.id === job.id && this.matchesPrefixes(j));
|
|
1682
|
-
if (index !== -1) {
|
|
1683
|
-
const existing = this.jobQueue[index];
|
|
1684
|
-
const currentAttempts = existing?.run_attempts ?? 0;
|
|
1685
|
-
jobWithPrefixes.run_attempts = currentAttempts + 1;
|
|
1686
|
-
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
1687
|
-
jobWithPrefixes[key] = value;
|
|
1688
|
-
}
|
|
1689
|
-
this.jobQueue[index] = jobWithPrefixes;
|
|
1690
|
-
this.events.emit("change", { type: "UPDATE", old: existing, new: jobWithPrefixes });
|
|
1691
|
-
}
|
|
1692
|
-
}
|
|
1693
|
-
async release(id) {
|
|
1694
|
-
await sleep(0);
|
|
1695
|
-
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
1696
|
-
if (job) {
|
|
1697
|
-
const oldJob = { ...job };
|
|
1698
|
-
job.status = JobStatus.PENDING;
|
|
1699
|
-
job.worker_id = null;
|
|
1700
|
-
job.progress = 0;
|
|
1701
|
-
job.progress_message = "";
|
|
1702
|
-
job.progress_details = null;
|
|
1703
|
-
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1704
|
-
}
|
|
1705
|
-
}
|
|
1706
|
-
async abort(id) {
|
|
1707
|
-
await sleep(0);
|
|
1708
|
-
const job = this.jobQueue.find((j) => j.id === id && this.matchesPrefixes(j));
|
|
1709
|
-
if (job) {
|
|
1710
|
-
const oldJob = { ...job };
|
|
1711
|
-
job.status = JobStatus.ABORTING;
|
|
1712
|
-
this.events.emit("change", { type: "UPDATE", old: oldJob, new: job });
|
|
1713
|
-
}
|
|
1714
|
-
}
|
|
1715
|
-
async getByRunId(runId) {
|
|
1716
|
-
await sleep(0);
|
|
1717
|
-
return this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.job_run_id === runId);
|
|
1718
|
-
}
|
|
1719
|
-
async deleteAll() {
|
|
1720
|
-
await sleep(0);
|
|
1721
|
-
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job));
|
|
1722
|
-
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job));
|
|
1723
|
-
for (const job of deletedJobs) {
|
|
1724
|
-
this.events.emit("change", { type: "DELETE", old: job });
|
|
1725
|
-
}
|
|
1726
|
-
}
|
|
1727
|
-
async outputForInput(input) {
|
|
1728
|
-
await sleep(0);
|
|
1729
|
-
const fingerprint = await makeFingerprint4(input);
|
|
1730
|
-
return this.jobQueue.find((j) => this.matchesPrefixes(j) && j.fingerprint === fingerprint && j.status === JobStatus.COMPLETED)?.output ?? null;
|
|
1731
|
-
}
|
|
1732
|
-
async delete(id) {
|
|
1733
|
-
await sleep(0);
|
|
1734
|
-
const deletedJob = this.jobQueue.find((job) => job.id === id && this.matchesPrefixes(job));
|
|
1735
|
-
this.jobQueue = this.jobQueue.filter((job) => !(job.id === id && this.matchesPrefixes(job)));
|
|
1736
|
-
if (deletedJob) {
|
|
1737
|
-
this.events.emit("change", { type: "DELETE", old: deletedJob });
|
|
1738
|
-
}
|
|
1739
|
-
}
|
|
1740
|
-
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
1741
|
-
await sleep(0);
|
|
1742
|
-
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
1743
|
-
const deletedJobs = this.jobQueue.filter((job) => this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate);
|
|
1744
|
-
this.jobQueue = this.jobQueue.filter((job) => !this.matchesPrefixes(job) || job.status !== status || !job.completed_at || job.completed_at > cutoffDate);
|
|
1745
|
-
for (const job of deletedJobs) {
|
|
1746
|
-
this.events.emit("change", { type: "DELETE", old: job });
|
|
1747
|
-
}
|
|
1748
|
-
}
|
|
1749
|
-
async setupDatabase() {}
|
|
1750
|
-
matchesPrefixFilter(job, prefixFilter) {
|
|
1751
|
-
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
1752
|
-
return true;
|
|
1753
|
-
}
|
|
1754
|
-
const filterValues = prefixFilter ?? this.prefixValues;
|
|
1755
|
-
if (Object.keys(filterValues).length === 0) {
|
|
1756
|
-
return true;
|
|
1757
|
-
}
|
|
1758
|
-
const jobWithPrefixes = job;
|
|
1759
|
-
for (const [key, value] of Object.entries(filterValues)) {
|
|
1760
|
-
if (jobWithPrefixes[key] !== value) {
|
|
1761
|
-
return false;
|
|
1762
|
-
}
|
|
1763
|
-
}
|
|
1764
|
-
return true;
|
|
1765
|
-
}
|
|
1766
|
-
subscribeToChanges(callback, options) {
|
|
1767
|
-
const prefixFilter = options?.prefixFilter;
|
|
1768
|
-
const filteredCallback = (change) => {
|
|
1769
|
-
const newMatches = change.new ? this.matchesPrefixFilter(change.new, prefixFilter) : false;
|
|
1770
|
-
const oldMatches = change.old ? this.matchesPrefixFilter(change.old, prefixFilter) : false;
|
|
1771
|
-
if (!newMatches && !oldMatches) {
|
|
1772
|
-
return;
|
|
1773
|
-
}
|
|
1774
|
-
callback(change);
|
|
1775
|
-
};
|
|
1776
|
-
return this.events.subscribe("change", filteredCallback);
|
|
1777
|
-
}
|
|
1778
|
-
}
|
|
1779
|
-
// src/queue/TelemetryQueueStorage.ts
|
|
1780
|
-
class TelemetryQueueStorage {
|
|
1781
|
-
storageName;
|
|
1782
|
-
inner;
|
|
1783
|
-
constructor(storageName, inner) {
|
|
1784
|
-
this.storageName = storageName;
|
|
1785
|
-
this.inner = inner;
|
|
1786
|
-
}
|
|
1787
|
-
get scope() {
|
|
1788
|
-
return this.inner.scope;
|
|
1789
|
-
}
|
|
1790
|
-
add(job) {
|
|
1791
|
-
return traced("workglow.storage.queue.add", this.storageName, () => this.inner.add(job));
|
|
1792
|
-
}
|
|
1793
|
-
get(id) {
|
|
1794
|
-
return traced("workglow.storage.queue.get", this.storageName, () => this.inner.get(id));
|
|
1795
|
-
}
|
|
1796
|
-
next(workerId) {
|
|
1797
|
-
return traced("workglow.storage.queue.next", this.storageName, () => this.inner.next(workerId));
|
|
1798
|
-
}
|
|
1799
|
-
peek(status, num) {
|
|
1800
|
-
return traced("workglow.storage.queue.peek", this.storageName, () => this.inner.peek(status, num));
|
|
1801
|
-
}
|
|
1802
|
-
size(status) {
|
|
1803
|
-
return traced("workglow.storage.queue.size", this.storageName, () => this.inner.size(status));
|
|
1804
|
-
}
|
|
1805
|
-
complete(job) {
|
|
1806
|
-
return traced("workglow.storage.queue.complete", this.storageName, () => this.inner.complete(job));
|
|
1807
|
-
}
|
|
1808
|
-
release(id) {
|
|
1809
|
-
return traced("workglow.storage.queue.release", this.storageName, () => this.inner.release(id));
|
|
1810
|
-
}
|
|
1811
|
-
deleteAll() {
|
|
1812
|
-
return traced("workglow.storage.queue.deleteAll", this.storageName, () => this.inner.deleteAll());
|
|
1813
|
-
}
|
|
1814
|
-
outputForInput(input) {
|
|
1815
|
-
return traced("workglow.storage.queue.outputForInput", this.storageName, () => this.inner.outputForInput(input));
|
|
1816
|
-
}
|
|
1817
|
-
abort(id) {
|
|
1818
|
-
return traced("workglow.storage.queue.abort", this.storageName, () => this.inner.abort(id));
|
|
1819
|
-
}
|
|
1820
|
-
getByRunId(runId) {
|
|
1821
|
-
return traced("workglow.storage.queue.getByRunId", this.storageName, () => this.inner.getByRunId(runId));
|
|
1822
|
-
}
|
|
1823
|
-
saveProgress(id, progress, message, details) {
|
|
1824
|
-
return traced("workglow.storage.queue.saveProgress", this.storageName, () => this.inner.saveProgress(id, progress, message, details));
|
|
1825
|
-
}
|
|
1826
|
-
delete(id) {
|
|
1827
|
-
return traced("workglow.storage.queue.delete", this.storageName, () => this.inner.delete(id));
|
|
1828
|
-
}
|
|
1829
|
-
deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
1830
|
-
return traced("workglow.storage.queue.deleteJobsByStatusAndAge", this.storageName, () => this.inner.deleteJobsByStatusAndAge(status, olderThanMs));
|
|
1831
|
-
}
|
|
1832
|
-
setupDatabase() {
|
|
1833
|
-
return this.inner.setupDatabase();
|
|
1834
|
-
}
|
|
1835
|
-
subscribeToChanges(callback, options) {
|
|
1836
|
-
return this.inner.subscribeToChanges(callback, options);
|
|
1837
|
-
}
|
|
1838
|
-
}
|
|
1839
|
-
// src/queue-limiter/InMemoryRateLimiterStorage.ts
|
|
1840
|
-
import { createServiceToken as createServiceToken10, sleep as sleep2, uuid4 as uuid43 } from "@workglow/util";
|
|
1841
|
-
var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken10("ratelimiter.storage.inMemory");
|
|
1842
|
-
|
|
1843
|
-
class InMemoryRateLimiterStorage {
|
|
1844
|
-
scope = "process";
|
|
1845
|
-
prefixValues;
|
|
1846
|
-
executions = new Map;
|
|
1847
|
-
nextAvailableTimes = new Map;
|
|
1848
|
-
reserveChains = new Map;
|
|
1849
|
-
constructor(options) {
|
|
1850
|
-
this.prefixValues = options?.prefixValues ?? {};
|
|
1851
|
-
}
|
|
1852
|
-
makeKey(queueName) {
|
|
1853
|
-
const prefixPart = Object.entries(this.prefixValues).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}:${v}`).join("|");
|
|
1854
|
-
return prefixPart ? `${prefixPart}|${queueName}` : queueName;
|
|
1855
|
-
}
|
|
1856
|
-
async setupDatabase() {}
|
|
1857
|
-
async withKeyLock(key, fn) {
|
|
1858
|
-
const previous = this.reserveChains.get(key) ?? Promise.resolve();
|
|
1859
|
-
let release;
|
|
1860
|
-
const next = new Promise((resolve) => {
|
|
1861
|
-
release = resolve;
|
|
1862
|
-
});
|
|
1863
|
-
this.reserveChains.set(key, next);
|
|
1864
|
-
try {
|
|
1865
|
-
await previous;
|
|
1866
|
-
return await fn();
|
|
1867
|
-
} finally {
|
|
1868
|
-
release(undefined);
|
|
1869
|
-
if (this.reserveChains.get(key) === next) {
|
|
1870
|
-
this.reserveChains.delete(key);
|
|
1871
|
-
}
|
|
1872
|
-
}
|
|
1873
|
-
}
|
|
1874
|
-
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
1875
|
-
const key = this.makeKey(queueName);
|
|
1876
|
-
return this.withKeyLock(key, () => {
|
|
1877
|
-
const now = Date.now();
|
|
1878
|
-
const windowStart = new Date(now - windowMs);
|
|
1879
|
-
const executions = this.executions.get(key) ?? [];
|
|
1880
|
-
const live = executions.filter((e) => e.executedAt > windowStart);
|
|
1881
|
-
if (live.length >= maxExecutions) {
|
|
1882
|
-
if (live.length !== executions.length) {
|
|
1883
|
-
this.executions.set(key, live);
|
|
1884
|
-
}
|
|
1885
|
-
return null;
|
|
1886
|
-
}
|
|
1887
|
-
const next = this.nextAvailableTimes.get(key);
|
|
1888
|
-
if (next && next.getTime() > now) {
|
|
1889
|
-
return null;
|
|
1890
|
-
}
|
|
1891
|
-
const id = uuid43();
|
|
1892
|
-
live.push({ id, queueName, executedAt: new Date(now) });
|
|
1893
|
-
this.executions.set(key, live);
|
|
1894
|
-
return id;
|
|
1895
|
-
});
|
|
1896
|
-
}
|
|
1897
|
-
async releaseExecution(queueName, token) {
|
|
1898
|
-
if (token === null || token === undefined)
|
|
1899
|
-
return;
|
|
1900
|
-
const key = this.makeKey(queueName);
|
|
1901
|
-
await this.withKeyLock(key, () => {
|
|
1902
|
-
const executions = this.executions.get(key);
|
|
1903
|
-
if (!executions || executions.length === 0)
|
|
1904
|
-
return;
|
|
1905
|
-
const idx = executions.findIndex((e) => e.id === token);
|
|
1906
|
-
if (idx === -1)
|
|
1907
|
-
return;
|
|
1908
|
-
executions.splice(idx, 1);
|
|
1909
|
-
this.executions.set(key, executions);
|
|
1910
|
-
});
|
|
1911
|
-
}
|
|
1912
|
-
async recordExecution(queueName) {
|
|
1913
|
-
await sleep2(0);
|
|
1914
|
-
const key = this.makeKey(queueName);
|
|
1915
|
-
const executions = this.executions.get(key) ?? [];
|
|
1916
|
-
executions.push({
|
|
1917
|
-
id: uuid43(),
|
|
1918
|
-
queueName,
|
|
1919
|
-
executedAt: new Date
|
|
1920
|
-
});
|
|
1921
|
-
this.executions.set(key, executions);
|
|
1922
|
-
}
|
|
1923
|
-
async getExecutionCount(queueName, windowStartTime) {
|
|
1924
|
-
await sleep2(0);
|
|
1925
|
-
const key = this.makeKey(queueName);
|
|
1926
|
-
const executions = this.executions.get(key) ?? [];
|
|
1927
|
-
const windowStart = new Date(windowStartTime);
|
|
1928
|
-
return executions.filter((e) => e.executedAt > windowStart).length;
|
|
1929
|
-
}
|
|
1930
|
-
async getOldestExecutionAtOffset(queueName, offset) {
|
|
1931
|
-
await sleep2(0);
|
|
1932
|
-
const key = this.makeKey(queueName);
|
|
1933
|
-
const executions = this.executions.get(key) ?? [];
|
|
1934
|
-
const sorted = [...executions].sort((a, b) => a.executedAt.getTime() - b.executedAt.getTime());
|
|
1935
|
-
const execution = sorted[offset];
|
|
1936
|
-
return execution?.executedAt.toISOString();
|
|
1937
|
-
}
|
|
1938
|
-
async getNextAvailableTime(queueName) {
|
|
1939
|
-
await sleep2(0);
|
|
1940
|
-
const key = this.makeKey(queueName);
|
|
1941
|
-
const time = this.nextAvailableTimes.get(key);
|
|
1942
|
-
return time?.toISOString();
|
|
1943
|
-
}
|
|
1944
|
-
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
1945
|
-
await sleep2(0);
|
|
1946
|
-
const key = this.makeKey(queueName);
|
|
1947
|
-
this.nextAvailableTimes.set(key, new Date(nextAvailableAt));
|
|
1948
|
-
}
|
|
1949
|
-
async clear(queueName) {
|
|
1950
|
-
await sleep2(0);
|
|
1951
|
-
const key = this.makeKey(queueName);
|
|
1952
|
-
this.executions.delete(key);
|
|
1953
|
-
this.nextAvailableTimes.delete(key);
|
|
1954
|
-
}
|
|
1955
|
-
}
|
|
1956
|
-
// src/queue-limiter/IRateLimiterStorage.ts
|
|
1957
|
-
import { createServiceToken as createServiceToken11 } from "@workglow/util";
|
|
1958
|
-
var RATE_LIMITER_STORAGE = createServiceToken11("ratelimiter.storage");
|
|
1959
|
-
// src/util/HybridSubscriptionManager.ts
|
|
1960
|
-
class HybridSubscriptionManager {
|
|
1961
|
-
subscribers = new Set;
|
|
1962
|
-
lastKnownState = new Map;
|
|
1963
|
-
initialized = false;
|
|
1964
|
-
channel = null;
|
|
1965
|
-
backupPollingIntervalId = null;
|
|
1966
|
-
fetchState;
|
|
1967
|
-
compareItems;
|
|
1968
|
-
payloadFactory;
|
|
1969
|
-
options;
|
|
1970
|
-
hasBroadcastChannel;
|
|
1971
|
-
constructor(channelName, fetchState, compareItems, payloadFactory, options) {
|
|
1972
|
-
this.fetchState = fetchState;
|
|
1973
|
-
this.compareItems = compareItems;
|
|
1974
|
-
this.payloadFactory = payloadFactory;
|
|
1975
|
-
this.options = {
|
|
1976
|
-
defaultIntervalMs: options?.defaultIntervalMs ?? 1000,
|
|
1977
|
-
backupPollingIntervalMs: options?.backupPollingIntervalMs ?? 5000,
|
|
1978
|
-
useBroadcastChannel: options?.useBroadcastChannel ?? true,
|
|
1979
|
-
broadcastChannelName: options?.broadcastChannelName ?? channelName
|
|
1980
|
-
};
|
|
1981
|
-
this.hasBroadcastChannel = this.options.useBroadcastChannel && typeof BroadcastChannel !== "undefined";
|
|
1982
|
-
if (this.hasBroadcastChannel) {
|
|
1983
|
-
this.initializeBroadcastChannel();
|
|
1984
|
-
}
|
|
1985
|
-
}
|
|
1986
|
-
initializeBroadcastChannel() {
|
|
1987
|
-
try {
|
|
1988
|
-
this.channel = new BroadcastChannel(this.options.broadcastChannelName);
|
|
1989
|
-
this.channel.onmessage = (event) => {
|
|
1990
|
-
this.handleBroadcastMessage(event.data);
|
|
1991
|
-
};
|
|
1992
|
-
} catch (error) {
|
|
1993
|
-
console.error("Failed to initialize BroadcastChannel:", error);
|
|
1994
|
-
this.channel = null;
|
|
1995
|
-
}
|
|
1996
|
-
}
|
|
1997
|
-
async handleBroadcastMessage(message) {
|
|
1998
|
-
if (message.type === "CHANGE") {
|
|
1999
|
-
await this.pollAndNotify();
|
|
1770
|
+
async handleBroadcastMessage(message) {
|
|
1771
|
+
if (message.type === "CHANGE") {
|
|
1772
|
+
await this.pollAndNotify();
|
|
2000
1773
|
}
|
|
2001
1774
|
}
|
|
2002
1775
|
notifyLocalChange() {
|
|
@@ -2353,6 +2126,8 @@ class InMemoryVectorStorage extends InMemoryTabularStorage {
|
|
|
2353
2126
|
}
|
|
2354
2127
|
}
|
|
2355
2128
|
// src/vector/TelemetryVectorStorage.ts
|
|
2129
|
+
import { traced as traced3 } from "@workglow/util";
|
|
2130
|
+
|
|
2356
2131
|
class TelemetryVectorStorage extends TelemetryTabularStorage {
|
|
2357
2132
|
vectorInner;
|
|
2358
2133
|
constructor(storageName, inner) {
|
|
@@ -2363,13 +2138,13 @@ class TelemetryVectorStorage extends TelemetryTabularStorage {
|
|
|
2363
2138
|
return this.vectorInner.getVectorDimensions();
|
|
2364
2139
|
}
|
|
2365
2140
|
similaritySearch(query, options) {
|
|
2366
|
-
return
|
|
2141
|
+
return traced3("workglow.storage.vector.similaritySearch", this.storageName, () => this.vectorInner.similaritySearch(query, options));
|
|
2367
2142
|
}
|
|
2368
2143
|
hybridSearch(query, options) {
|
|
2369
2144
|
if (!this.vectorInner.hybridSearch) {
|
|
2370
2145
|
throw new Error("hybridSearch is not supported by the underlying storage implementation");
|
|
2371
2146
|
}
|
|
2372
|
-
return
|
|
2147
|
+
return traced3("workglow.storage.vector.hybridSearch", this.storageName, () => this.vectorInner.hybridSearch(query, options));
|
|
2373
2148
|
}
|
|
2374
2149
|
}
|
|
2375
2150
|
// src/credentials/EncryptedKvCredentialStore.ts
|
|
@@ -2495,3750 +2270,201 @@ class LazyEncryptedCredentialStore {
|
|
|
2495
2270
|
return this.inner.deleteAll();
|
|
2496
2271
|
}
|
|
2497
2272
|
}
|
|
2498
|
-
// src/tabular/
|
|
2499
|
-
import { createServiceToken as
|
|
2273
|
+
// src/tabular/SharedInMemoryTabularStorage.ts
|
|
2274
|
+
import { createServiceToken as createServiceToken8 } from "@workglow/util";
|
|
2275
|
+
var SHARED_IN_MEMORY_TABULAR_REPOSITORY = createServiceToken8("storage.tabularRepository.sharedInMemory");
|
|
2276
|
+
var SYNC_TIMEOUT = 1000;
|
|
2277
|
+
var MAX_PENDING_MESSAGES = 1000;
|
|
2500
2278
|
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
|
|
2504
|
-
|
|
2505
|
-
|
|
2279
|
+
class SharedInMemoryTabularStorage extends BaseTabularStorage {
|
|
2280
|
+
channel = null;
|
|
2281
|
+
channelName;
|
|
2282
|
+
inMemoryRepo;
|
|
2283
|
+
isInitialized = false;
|
|
2284
|
+
syncInProgress = false;
|
|
2285
|
+
pendingMessages = [];
|
|
2286
|
+
constructor(channelName = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
|
|
2287
|
+
super(schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
2288
|
+
this.channelName = channelName;
|
|
2289
|
+
this.inMemoryRepo = new InMemoryTabularStorage(schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
2290
|
+
this.setupEventForwarding();
|
|
2291
|
+
this.initializeBroadcastChannel();
|
|
2292
|
+
}
|
|
2293
|
+
isBroadcastChannelAvailable() {
|
|
2294
|
+
return typeof BroadcastChannel !== "undefined";
|
|
2295
|
+
}
|
|
2296
|
+
initializeBroadcastChannel() {
|
|
2297
|
+
if (!this.isBroadcastChannelAvailable()) {
|
|
2298
|
+
console.warn("BroadcastChannel is not available. Tab synchronization will not work.");
|
|
2299
|
+
return;
|
|
2300
|
+
}
|
|
2506
2301
|
try {
|
|
2507
|
-
|
|
2508
|
-
|
|
2509
|
-
|
|
2510
|
-
request.onsuccess = () => resolve();
|
|
2511
|
-
request.onerror = () => reject(request.error);
|
|
2512
|
-
transaction.onerror = () => reject(transaction.error);
|
|
2513
|
-
} catch (err) {
|
|
2514
|
-
resolve();
|
|
2515
|
-
}
|
|
2516
|
-
});
|
|
2517
|
-
}
|
|
2518
|
-
async function openIndexedDbTable(tableName, version, upgradeNeededCallback) {
|
|
2519
|
-
return new Promise((resolve, reject) => {
|
|
2520
|
-
const openRequest = indexedDB.open(tableName, version);
|
|
2521
|
-
openRequest.onsuccess = (event) => {
|
|
2522
|
-
const db = event.target.result;
|
|
2523
|
-
db.onversionchange = () => {
|
|
2524
|
-
db.close();
|
|
2302
|
+
this.channel = new BroadcastChannel(this.channelName);
|
|
2303
|
+
this.channel.onmessage = (event) => {
|
|
2304
|
+
this.handleBroadcastMessage(event.data);
|
|
2525
2305
|
};
|
|
2526
|
-
|
|
2527
|
-
}
|
|
2528
|
-
|
|
2529
|
-
if (upgradeNeededCallback) {
|
|
2530
|
-
upgradeNeededCallback(event);
|
|
2531
|
-
}
|
|
2532
|
-
};
|
|
2533
|
-
openRequest.onerror = () => {
|
|
2534
|
-
const error = openRequest.error;
|
|
2535
|
-
if (error && error.name === "VersionError") {
|
|
2536
|
-
reject(new Error(`Database ${tableName} exists at a higher version. Cannot open at version ${version || "current"}.`));
|
|
2537
|
-
} else {
|
|
2538
|
-
reject(error);
|
|
2539
|
-
}
|
|
2540
|
-
};
|
|
2541
|
-
openRequest.onblocked = () => {
|
|
2542
|
-
reject(new Error(`Database ${tableName} is blocked. Close all other tabs using this database.`));
|
|
2543
|
-
};
|
|
2544
|
-
});
|
|
2545
|
-
}
|
|
2546
|
-
async function deleteIndexedDbTable(tableName) {
|
|
2547
|
-
return new Promise((resolve, reject) => {
|
|
2548
|
-
const deleteRequest = indexedDB.deleteDatabase(tableName);
|
|
2549
|
-
deleteRequest.onsuccess = () => resolve();
|
|
2550
|
-
deleteRequest.onerror = () => reject(deleteRequest.error);
|
|
2551
|
-
deleteRequest.onblocked = () => {
|
|
2552
|
-
reject(new Error(`Cannot delete database ${tableName}. Close all other tabs using this database.`));
|
|
2553
|
-
};
|
|
2554
|
-
});
|
|
2555
|
-
}
|
|
2556
|
-
function compareSchemas(store, expectedPrimaryKey, expectedIndexes) {
|
|
2557
|
-
const diff = {
|
|
2558
|
-
indexesToAdd: [],
|
|
2559
|
-
indexesToRemove: [],
|
|
2560
|
-
indexesToModify: [],
|
|
2561
|
-
primaryKeyChanged: false,
|
|
2562
|
-
needsObjectStoreRecreation: false
|
|
2563
|
-
};
|
|
2564
|
-
const actualKeyPath = store.keyPath;
|
|
2565
|
-
const normalizedExpected = Array.isArray(expectedPrimaryKey) ? expectedPrimaryKey : expectedPrimaryKey;
|
|
2566
|
-
const normalizedActual = Array.isArray(actualKeyPath) ? actualKeyPath : actualKeyPath;
|
|
2567
|
-
if (!deepEqual(normalizedExpected, normalizedActual)) {
|
|
2568
|
-
diff.primaryKeyChanged = true;
|
|
2569
|
-
diff.needsObjectStoreRecreation = true;
|
|
2570
|
-
return diff;
|
|
2571
|
-
}
|
|
2572
|
-
const existingIndexes = new Map;
|
|
2573
|
-
for (let i = 0;i < store.indexNames.length; i++) {
|
|
2574
|
-
const indexName = store.indexNames[i];
|
|
2575
|
-
existingIndexes.set(indexName, store.index(indexName));
|
|
2576
|
-
}
|
|
2577
|
-
for (const expectedIdx of expectedIndexes) {
|
|
2578
|
-
const existingIdx = existingIndexes.get(expectedIdx.name);
|
|
2579
|
-
if (!existingIdx) {
|
|
2580
|
-
diff.indexesToAdd.push(expectedIdx);
|
|
2581
|
-
} else {
|
|
2582
|
-
const expectedKeyPath = Array.isArray(expectedIdx.keyPath) ? expectedIdx.keyPath : [expectedIdx.keyPath];
|
|
2583
|
-
const actualKeyPath2 = Array.isArray(existingIdx.keyPath) ? existingIdx.keyPath : [existingIdx.keyPath];
|
|
2584
|
-
const keyPathChanged = !deepEqual(expectedKeyPath, actualKeyPath2);
|
|
2585
|
-
const uniqueChanged = existingIdx.unique !== (expectedIdx.options?.unique ?? false);
|
|
2586
|
-
const multiEntryChanged = existingIdx.multiEntry !== (expectedIdx.options?.multiEntry ?? false);
|
|
2587
|
-
if (keyPathChanged || uniqueChanged || multiEntryChanged) {
|
|
2588
|
-
diff.indexesToModify.push(expectedIdx);
|
|
2589
|
-
}
|
|
2590
|
-
existingIndexes.delete(expectedIdx.name);
|
|
2306
|
+
this.syncFromOtherTabs();
|
|
2307
|
+
} catch (error) {
|
|
2308
|
+
console.error("Failed to initialize BroadcastChannel:", error);
|
|
2591
2309
|
}
|
|
2592
2310
|
}
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
}
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
}
|
|
2615
|
-
for (const indexDef of diff.indexesToModify) {
|
|
2616
|
-
options.onMigrationProgress?.(`Updating index: ${indexDef.name}`, 0.4);
|
|
2617
|
-
if (store.indexNames.contains(indexDef.name)) {
|
|
2618
|
-
store.deleteIndex(indexDef.name);
|
|
2311
|
+
setupEventForwarding() {
|
|
2312
|
+
this.inMemoryRepo.on("put", (entity) => {
|
|
2313
|
+
this.events.emit("put", entity);
|
|
2314
|
+
});
|
|
2315
|
+
this.inMemoryRepo.on("get", (key, entity) => {
|
|
2316
|
+
this.events.emit("get", key, entity);
|
|
2317
|
+
});
|
|
2318
|
+
this.inMemoryRepo.on("query", (key, entities) => {
|
|
2319
|
+
this.events.emit("query", key, entities);
|
|
2320
|
+
});
|
|
2321
|
+
this.inMemoryRepo.on("delete", (key) => {
|
|
2322
|
+
this.events.emit("delete", key);
|
|
2323
|
+
});
|
|
2324
|
+
this.inMemoryRepo.on("clearall", () => {
|
|
2325
|
+
this.events.emit("clearall");
|
|
2326
|
+
});
|
|
2327
|
+
}
|
|
2328
|
+
async handleBroadcastMessage(message) {
|
|
2329
|
+
if (this.syncInProgress && message.type !== "SYNC_RESPONSE") {
|
|
2330
|
+
if (this.pendingMessages.length < MAX_PENDING_MESSAGES) {
|
|
2331
|
+
this.pendingMessages.push(message);
|
|
2619
2332
|
}
|
|
2620
|
-
|
|
2621
|
-
}
|
|
2622
|
-
for (const indexDef of diff.indexesToAdd) {
|
|
2623
|
-
options.onMigrationProgress?.(`Adding index: ${indexDef.name}`, 0.6);
|
|
2624
|
-
store.createIndex(indexDef.name, indexDef.keyPath, indexDef.options);
|
|
2333
|
+
return;
|
|
2625
2334
|
}
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
const newVersion = currentVersion + 1;
|
|
2635
|
-
options.onMigrationProgress?.(`Performing destructive migration of ${tableName}. Reading existing data...`, 0);
|
|
2636
|
-
let existingData = [];
|
|
2637
|
-
try {
|
|
2638
|
-
const transaction = db.transaction(tableName, "readonly");
|
|
2639
|
-
const store = transaction.objectStore(tableName);
|
|
2640
|
-
existingData = await readAllData(store);
|
|
2641
|
-
options.onMigrationProgress?.(`Read ${existingData.length} records`, 0.3);
|
|
2642
|
-
} catch (err) {
|
|
2643
|
-
options.onMigrationWarning?.(`Failed to read existing data during migration: ${err}`, err);
|
|
2644
|
-
}
|
|
2645
|
-
db.close();
|
|
2646
|
-
if (options.dataTransformer && existingData.length > 0) {
|
|
2647
|
-
options.onMigrationProgress?.(`Transforming ${existingData.length} records...`, 0.4);
|
|
2648
|
-
try {
|
|
2649
|
-
const transformed = [];
|
|
2650
|
-
for (let i = 0;i < existingData.length; i++) {
|
|
2651
|
-
const record = existingData[i];
|
|
2652
|
-
const transformedRecord = await options.dataTransformer(record);
|
|
2653
|
-
if (transformedRecord !== undefined && transformedRecord !== null) {
|
|
2654
|
-
transformed.push(transformedRecord);
|
|
2655
|
-
}
|
|
2656
|
-
if (i % 100 === 0) {
|
|
2657
|
-
options.onMigrationProgress?.(`Transformed ${i}/${existingData.length} records`, 0.4 + i / existingData.length * 0.3);
|
|
2335
|
+
switch (message.type) {
|
|
2336
|
+
case "SYNC_REQUEST":
|
|
2337
|
+
const all = await this.inMemoryRepo.getAll();
|
|
2338
|
+
if (this.channel && all) {
|
|
2339
|
+
this.channel.postMessage({
|
|
2340
|
+
type: "SYNC_RESPONSE",
|
|
2341
|
+
data: all
|
|
2342
|
+
});
|
|
2658
2343
|
}
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
options.onMigrationWarning?.(`Data transformation failed: ${err}. Some data may be lost.`, err);
|
|
2664
|
-
existingData = [];
|
|
2665
|
-
}
|
|
2666
|
-
}
|
|
2667
|
-
options.onMigrationProgress?.(`Recreating object store...`, 0.75);
|
|
2668
|
-
const newDb = await openIndexedDbTable(tableName, newVersion, (event) => {
|
|
2669
|
-
const db2 = event.target.result;
|
|
2670
|
-
if (db2.objectStoreNames.contains(tableName)) {
|
|
2671
|
-
db2.deleteObjectStore(tableName);
|
|
2672
|
-
}
|
|
2673
|
-
const store = db2.createObjectStore(tableName, { keyPath: primaryKey, autoIncrement });
|
|
2674
|
-
for (const idx of expectedIndexes) {
|
|
2675
|
-
store.createIndex(idx.name, idx.keyPath, idx.options);
|
|
2676
|
-
}
|
|
2677
|
-
if (existingData.length > 0) {
|
|
2678
|
-
options.onMigrationProgress?.(`Restoring ${existingData.length} records...`, 0.8);
|
|
2679
|
-
for (const record of existingData) {
|
|
2680
|
-
try {
|
|
2681
|
-
store.put(record);
|
|
2682
|
-
} catch (err) {
|
|
2683
|
-
options.onMigrationWarning?.(`Failed to restore record: ${err}`, err);
|
|
2344
|
+
break;
|
|
2345
|
+
case "SYNC_RESPONSE":
|
|
2346
|
+
if (message.data && Array.isArray(message.data)) {
|
|
2347
|
+
await this.copyDataFromArray(message.data);
|
|
2684
2348
|
}
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
const store = db2.createObjectStore(tableName, { keyPath: primaryKey, autoIncrement });
|
|
2704
|
-
for (const idx of expectedIndexes) {
|
|
2705
|
-
store.createIndex(idx.name, idx.keyPath, idx.options);
|
|
2706
|
-
}
|
|
2707
|
-
});
|
|
2708
|
-
const snapshot = {
|
|
2709
|
-
version: db.version,
|
|
2710
|
-
primaryKey,
|
|
2711
|
-
indexes: expectedIndexes,
|
|
2712
|
-
recordCount: 0,
|
|
2713
|
-
timestamp: Date.now()
|
|
2714
|
-
};
|
|
2715
|
-
await saveSchemaMetadata(db, tableName, snapshot);
|
|
2716
|
-
options.onMigrationProgress?.(`Database created successfully`, 1);
|
|
2717
|
-
return db;
|
|
2718
|
-
}
|
|
2719
|
-
async function ensureIndexedDbTable(tableName, primaryKey, expectedIndexes = [], options = {}, autoIncrement = false) {
|
|
2720
|
-
try {
|
|
2721
|
-
let db;
|
|
2722
|
-
let wasJustCreated = false;
|
|
2723
|
-
try {
|
|
2724
|
-
db = await openIndexedDbTable(tableName);
|
|
2725
|
-
if (db.version === 1 && !db.objectStoreNames.contains(tableName)) {
|
|
2726
|
-
wasJustCreated = true;
|
|
2727
|
-
db.close();
|
|
2728
|
-
}
|
|
2729
|
-
} catch (err) {
|
|
2730
|
-
options.onMigrationProgress?.(`Database ${tableName} does not exist or has version conflict, creating...`, 0);
|
|
2731
|
-
return await createNewDatabase(tableName, primaryKey, expectedIndexes, options, autoIncrement);
|
|
2349
|
+
this.syncInProgress = false;
|
|
2350
|
+
await this.drainPendingMessages();
|
|
2351
|
+
break;
|
|
2352
|
+
case "PUT":
|
|
2353
|
+
await this.inMemoryRepo.put(message.entity);
|
|
2354
|
+
break;
|
|
2355
|
+
case "PUT_BULK":
|
|
2356
|
+
await this.inMemoryRepo.putBulk(message.entities);
|
|
2357
|
+
break;
|
|
2358
|
+
case "DELETE":
|
|
2359
|
+
await this.inMemoryRepo.delete(message.key);
|
|
2360
|
+
break;
|
|
2361
|
+
case "DELETE_ALL":
|
|
2362
|
+
await this.inMemoryRepo.deleteAll();
|
|
2363
|
+
break;
|
|
2364
|
+
case "DELETE_SEARCH":
|
|
2365
|
+
await this.inMemoryRepo.deleteSearch(message.criteria);
|
|
2366
|
+
break;
|
|
2732
2367
|
}
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
|
|
2740
|
-
|
|
2741
|
-
|
|
2742
|
-
db2.createObjectStore(METADATA_STORE_NAME, { keyPath: "tableName" });
|
|
2743
|
-
}
|
|
2744
|
-
const store2 = db2.createObjectStore(tableName, { keyPath: primaryKey, autoIncrement });
|
|
2745
|
-
for (const idx of expectedIndexes) {
|
|
2746
|
-
store2.createIndex(idx.name, idx.keyPath, idx.options);
|
|
2747
|
-
}
|
|
2748
|
-
});
|
|
2749
|
-
const snapshot2 = {
|
|
2750
|
-
version: db.version,
|
|
2751
|
-
primaryKey,
|
|
2752
|
-
indexes: expectedIndexes,
|
|
2753
|
-
recordCount: 0,
|
|
2754
|
-
timestamp: Date.now()
|
|
2755
|
-
};
|
|
2756
|
-
await saveSchemaMetadata(db, tableName, snapshot2);
|
|
2757
|
-
options.onMigrationProgress?.(`Database created successfully`, 1);
|
|
2758
|
-
return db;
|
|
2759
|
-
}
|
|
2760
|
-
if (!db.objectStoreNames.contains(METADATA_STORE_NAME)) {
|
|
2761
|
-
const currentVersion = db.version;
|
|
2762
|
-
db.close();
|
|
2763
|
-
db = await openIndexedDbTable(tableName, currentVersion + 1, (event) => {
|
|
2764
|
-
const db2 = event.target.result;
|
|
2765
|
-
if (!db2.objectStoreNames.contains(METADATA_STORE_NAME)) {
|
|
2766
|
-
db2.createObjectStore(METADATA_STORE_NAME, { keyPath: "tableName" });
|
|
2368
|
+
}
|
|
2369
|
+
async drainPendingMessages() {
|
|
2370
|
+
while (!this.syncInProgress && this.pendingMessages.length > 0) {
|
|
2371
|
+
const messages = this.pendingMessages;
|
|
2372
|
+
this.pendingMessages = [];
|
|
2373
|
+
for (const message of messages) {
|
|
2374
|
+
await this.handleBroadcastMessage(message);
|
|
2375
|
+
if (this.syncInProgress) {
|
|
2376
|
+
break;
|
|
2767
2377
|
}
|
|
2768
|
-
}
|
|
2378
|
+
}
|
|
2769
2379
|
}
|
|
2770
|
-
if (!db.objectStoreNames.contains(tableName)) {
|
|
2771
|
-
options.onMigrationProgress?.(`Object store ${tableName} does not exist, creating...`, 0);
|
|
2772
|
-
db.close();
|
|
2773
|
-
return await createNewDatabase(tableName, primaryKey, expectedIndexes, options, autoIncrement);
|
|
2774
|
-
}
|
|
2775
|
-
const transaction = db.transaction(tableName, "readonly");
|
|
2776
|
-
const store = transaction.objectStore(tableName);
|
|
2777
|
-
const diff = compareSchemas(store, primaryKey, expectedIndexes);
|
|
2778
|
-
await new Promise((resolve) => {
|
|
2779
|
-
transaction.oncomplete = () => resolve();
|
|
2780
|
-
transaction.onerror = () => resolve();
|
|
2781
|
-
});
|
|
2782
|
-
const needsMigration = diff.indexesToAdd.length > 0 || diff.indexesToRemove.length > 0 || diff.indexesToModify.length > 0 || diff.needsObjectStoreRecreation;
|
|
2783
|
-
if (!needsMigration) {
|
|
2784
|
-
options.onMigrationProgress?.(`Schema for ${tableName} is up to date`, 1);
|
|
2785
|
-
const snapshot2 = {
|
|
2786
|
-
version: db.version,
|
|
2787
|
-
primaryKey,
|
|
2788
|
-
indexes: expectedIndexes,
|
|
2789
|
-
timestamp: Date.now()
|
|
2790
|
-
};
|
|
2791
|
-
await saveSchemaMetadata(db, tableName, snapshot2);
|
|
2792
|
-
return db;
|
|
2793
|
-
}
|
|
2794
|
-
if (diff.needsObjectStoreRecreation) {
|
|
2795
|
-
options.onMigrationProgress?.(`Schema change requires object store recreation for ${tableName}`, 0);
|
|
2796
|
-
db = await performDestructiveMigration(db, tableName, primaryKey, expectedIndexes, options, autoIncrement);
|
|
2797
|
-
} else {
|
|
2798
|
-
options.onMigrationProgress?.(`Performing incremental migration for ${tableName}`, 0);
|
|
2799
|
-
db = await performIncrementalMigration(db, tableName, diff, options);
|
|
2800
|
-
}
|
|
2801
|
-
const snapshot = {
|
|
2802
|
-
version: db.version,
|
|
2803
|
-
primaryKey,
|
|
2804
|
-
indexes: expectedIndexes,
|
|
2805
|
-
timestamp: Date.now()
|
|
2806
|
-
};
|
|
2807
|
-
await saveSchemaMetadata(db, tableName, snapshot);
|
|
2808
|
-
return db;
|
|
2809
|
-
} catch (err) {
|
|
2810
|
-
options.onMigrationWarning?.(`Migration failed for ${tableName}: ${err}`, err);
|
|
2811
|
-
throw err;
|
|
2812
|
-
}
|
|
2813
|
-
}
|
|
2814
|
-
async function dropIndexedDbTable(tableName) {
|
|
2815
|
-
return deleteIndexedDbTable(tableName);
|
|
2816
|
-
}
|
|
2817
|
-
|
|
2818
|
-
// src/tabular/IndexedDbTabularStorage.ts
|
|
2819
|
-
var IDB_TABULAR_REPOSITORY = createServiceToken12("storage.tabularRepository.indexedDb");
|
|
2820
|
-
function compareEntitiesForChange(a, b) {
|
|
2821
|
-
const au = a?.updated_at;
|
|
2822
|
-
const bu = b?.updated_at;
|
|
2823
|
-
if (typeof au === "string" && typeof bu === "string") {
|
|
2824
|
-
return au === bu;
|
|
2825
|
-
}
|
|
2826
|
-
return deepEqual2(a, b);
|
|
2827
|
-
}
|
|
2828
|
-
|
|
2829
|
-
class IndexedDbTabularStorage extends BaseTabularStorage {
|
|
2830
|
-
table;
|
|
2831
|
-
db;
|
|
2832
|
-
setupPromise = null;
|
|
2833
|
-
migrationOptions;
|
|
2834
|
-
hybridManager = null;
|
|
2835
|
-
hybridOptions;
|
|
2836
|
-
cursorSafeIndexes;
|
|
2837
|
-
constructor(table = "tabular_store", schema, primaryKeyNames, indexes = [], migrationOptions = {}, clientProvidedKeys = "if-missing") {
|
|
2838
|
-
super(schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
2839
|
-
this.table = table;
|
|
2840
|
-
this.migrationOptions = migrationOptions;
|
|
2841
|
-
this.hybridOptions = {
|
|
2842
|
-
useBroadcastChannel: migrationOptions.useBroadcastChannel ?? true,
|
|
2843
|
-
backupPollingIntervalMs: migrationOptions.backupPollingIntervalMs ?? 5000
|
|
2844
|
-
};
|
|
2845
|
-
}
|
|
2846
|
-
async getDb() {
|
|
2847
|
-
if (this.db)
|
|
2848
|
-
return this.db;
|
|
2849
|
-
await this.setupDatabase();
|
|
2850
|
-
return this.db;
|
|
2851
2380
|
}
|
|
2852
|
-
|
|
2853
|
-
if (this.
|
|
2854
|
-
return;
|
|
2855
|
-
if (this.setupPromise) {
|
|
2856
|
-
await this.setupPromise;
|
|
2381
|
+
syncFromOtherTabs() {
|
|
2382
|
+
if (!this.channel)
|
|
2857
2383
|
return;
|
|
2858
|
-
|
|
2859
|
-
this.
|
|
2860
|
-
|
|
2861
|
-
this.
|
|
2862
|
-
|
|
2863
|
-
|
|
2864
|
-
|
|
2865
|
-
|
|
2866
|
-
async performSetup() {
|
|
2867
|
-
const pkColumns = super.primaryKeyColumns();
|
|
2868
|
-
const expectedIndexes = [];
|
|
2869
|
-
for (const spec of this.indexes) {
|
|
2870
|
-
const columns = spec;
|
|
2871
|
-
if (columns.length <= pkColumns.length) {
|
|
2872
|
-
const isPkPrefix = columns.every((col, idx) => col === pkColumns[idx]);
|
|
2873
|
-
if (isPkPrefix)
|
|
2874
|
-
continue;
|
|
2384
|
+
this.syncInProgress = true;
|
|
2385
|
+
this.channel.postMessage({ type: "SYNC_REQUEST" });
|
|
2386
|
+
setTimeout(() => {
|
|
2387
|
+
if (this.syncInProgress) {
|
|
2388
|
+
this.syncInProgress = false;
|
|
2389
|
+
this.drainPendingMessages().catch((error) => {
|
|
2390
|
+
console.error("Failed to drain pending messages after sync timeout", error);
|
|
2391
|
+
});
|
|
2875
2392
|
}
|
|
2876
|
-
|
|
2877
|
-
const indexName = columnNames.join("_");
|
|
2878
|
-
expectedIndexes.push({
|
|
2879
|
-
name: indexName,
|
|
2880
|
-
keyPath: columnNames.length === 1 ? columnNames[0] : columnNames,
|
|
2881
|
-
options: { unique: false }
|
|
2882
|
-
});
|
|
2883
|
-
}
|
|
2884
|
-
const primaryKey = pkColumns.length === 1 ? pkColumns[0] : pkColumns;
|
|
2885
|
-
const useAutoIncrement = this.hasAutoGeneratedKey() && this.autoGeneratedKeyStrategy === "autoincrement" && pkColumns.length === 1;
|
|
2886
|
-
return await ensureIndexedDbTable(this.table, primaryKey, expectedIndexes, this.migrationOptions, useAutoIncrement);
|
|
2393
|
+
}, SYNC_TIMEOUT);
|
|
2887
2394
|
}
|
|
2888
|
-
|
|
2889
|
-
if (
|
|
2890
|
-
return
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
}
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
if (this.hasAutoGeneratedKey() && this.autoGeneratedKeyName) {
|
|
2898
|
-
const keyName = String(this.autoGeneratedKeyName);
|
|
2899
|
-
const clientProvidedValue = record[keyName];
|
|
2900
|
-
const hasClientValue = clientProvidedValue !== undefined && clientProvidedValue !== null;
|
|
2901
|
-
if (this.autoGeneratedKeyStrategy === "uuid") {
|
|
2902
|
-
let shouldGenerate = false;
|
|
2903
|
-
if (this.clientProvidedKeys === "never") {
|
|
2904
|
-
shouldGenerate = true;
|
|
2905
|
-
} else if (this.clientProvidedKeys === "always") {
|
|
2906
|
-
if (!hasClientValue) {
|
|
2907
|
-
throw new Error(`Auto-generated key "${keyName}" is required when clientProvidedKeys is "always"`);
|
|
2908
|
-
}
|
|
2909
|
-
shouldGenerate = false;
|
|
2910
|
-
} else {
|
|
2911
|
-
shouldGenerate = !hasClientValue;
|
|
2912
|
-
}
|
|
2913
|
-
if (shouldGenerate) {
|
|
2914
|
-
const generatedValue = this.generateKeyValue(keyName, "uuid");
|
|
2915
|
-
recordToStore = { ...record, [keyName]: generatedValue };
|
|
2916
|
-
}
|
|
2917
|
-
} else if (this.autoGeneratedKeyStrategy === "autoincrement") {
|
|
2918
|
-
if (this.clientProvidedKeys === "always" && !hasClientValue) {
|
|
2919
|
-
throw new Error(`Auto-generated key "${keyName}" is required when clientProvidedKeys is "always"`);
|
|
2920
|
-
}
|
|
2921
|
-
if (this.clientProvidedKeys === "never") {
|
|
2922
|
-
const { [keyName]: _, ...rest } = record;
|
|
2923
|
-
recordToStore = rest;
|
|
2924
|
-
}
|
|
2925
|
-
}
|
|
2395
|
+
async copyDataFromArray(entities) {
|
|
2396
|
+
if (entities.length === 0)
|
|
2397
|
+
return;
|
|
2398
|
+
await this.inMemoryRepo.deleteAll();
|
|
2399
|
+
await this.inMemoryRepo.putBulk(entities);
|
|
2400
|
+
}
|
|
2401
|
+
broadcast(message) {
|
|
2402
|
+
if (this.channel) {
|
|
2403
|
+
this.channel.postMessage(message);
|
|
2926
2404
|
}
|
|
2927
|
-
return new Promise((resolve, reject) => {
|
|
2928
|
-
const transaction = db.transaction(this.table, "readwrite");
|
|
2929
|
-
const store = transaction.objectStore(this.table);
|
|
2930
|
-
const request = store.put(recordToStore);
|
|
2931
|
-
request.onerror = () => {
|
|
2932
|
-
reject(request.error);
|
|
2933
|
-
};
|
|
2934
|
-
request.onsuccess = () => {
|
|
2935
|
-
if (this.hasAutoGeneratedKey() && this.autoGeneratedKeyName && this.autoGeneratedKeyStrategy === "autoincrement") {
|
|
2936
|
-
const keyName = String(this.autoGeneratedKeyName);
|
|
2937
|
-
if (recordToStore[keyName] === undefined) {
|
|
2938
|
-
recordToStore = { ...recordToStore, [keyName]: request.result };
|
|
2939
|
-
}
|
|
2940
|
-
}
|
|
2941
|
-
this.events.emit("put", recordToStore);
|
|
2942
|
-
resolve(recordToStore);
|
|
2943
|
-
};
|
|
2944
|
-
transaction.oncomplete = () => {
|
|
2945
|
-
this.hybridManager?.notifyLocalChange();
|
|
2946
|
-
};
|
|
2947
|
-
});
|
|
2948
2405
|
}
|
|
2949
|
-
async
|
|
2950
|
-
|
|
2406
|
+
async setupDatabase() {
|
|
2407
|
+
if (this.isInitialized)
|
|
2408
|
+
return;
|
|
2409
|
+
this.isInitialized = true;
|
|
2410
|
+
await this.syncFromOtherTabs();
|
|
2951
2411
|
}
|
|
2952
|
-
|
|
2953
|
-
|
|
2412
|
+
async put(value) {
|
|
2413
|
+
const result = await this.inMemoryRepo.put(value);
|
|
2414
|
+
this.broadcast({ type: "PUT", entity: result });
|
|
2415
|
+
return result;
|
|
2954
2416
|
}
|
|
2955
|
-
|
|
2956
|
-
const
|
|
2957
|
-
|
|
2417
|
+
async putBulk(values) {
|
|
2418
|
+
const result = await this.inMemoryRepo.putBulk(values);
|
|
2419
|
+
this.broadcast({ type: "PUT_BULK", entities: result });
|
|
2420
|
+
return result;
|
|
2958
2421
|
}
|
|
2959
2422
|
async get(key) {
|
|
2960
|
-
|
|
2961
|
-
return new Promise((resolve, reject) => {
|
|
2962
|
-
const transaction = db.transaction(this.table, "readonly");
|
|
2963
|
-
const store = transaction.objectStore(this.table);
|
|
2964
|
-
const request = store.get(this.getIndexedKey(key));
|
|
2965
|
-
request.onerror = () => reject(request.error);
|
|
2966
|
-
request.onsuccess = () => {
|
|
2967
|
-
if (!request.result) {
|
|
2968
|
-
this.events.emit("get", key, undefined);
|
|
2969
|
-
resolve(undefined);
|
|
2970
|
-
return;
|
|
2971
|
-
}
|
|
2972
|
-
this.events.emit("get", key, request.result);
|
|
2973
|
-
resolve(request.result);
|
|
2974
|
-
};
|
|
2975
|
-
});
|
|
2976
|
-
}
|
|
2977
|
-
async getAll(options) {
|
|
2978
|
-
this.validateGetAllOptions(options);
|
|
2979
|
-
const db = await this.getDb();
|
|
2980
|
-
const transaction = db.transaction(this.table, "readonly");
|
|
2981
|
-
const store = transaction.objectStore(this.table);
|
|
2982
|
-
const request = store.getAll();
|
|
2983
|
-
return new Promise((resolve, reject) => {
|
|
2984
|
-
request.onerror = () => reject(request.error);
|
|
2985
|
-
request.onsuccess = () => {
|
|
2986
|
-
let values = request.result;
|
|
2987
|
-
if (values.length === 0) {
|
|
2988
|
-
resolve(undefined);
|
|
2989
|
-
return;
|
|
2990
|
-
}
|
|
2991
|
-
if (options?.orderBy && options.orderBy.length > 0) {
|
|
2992
|
-
values.sort((a, b) => {
|
|
2993
|
-
for (const { column, direction } of options.orderBy) {
|
|
2994
|
-
const aVal = a[column];
|
|
2995
|
-
const bVal = b[column];
|
|
2996
|
-
if (aVal == null && bVal == null)
|
|
2997
|
-
continue;
|
|
2998
|
-
if (aVal == null)
|
|
2999
|
-
return direction === "ASC" ? -1 : 1;
|
|
3000
|
-
if (bVal == null)
|
|
3001
|
-
return direction === "ASC" ? 1 : -1;
|
|
3002
|
-
if (aVal < bVal)
|
|
3003
|
-
return direction === "ASC" ? -1 : 1;
|
|
3004
|
-
if (aVal > bVal)
|
|
3005
|
-
return direction === "ASC" ? 1 : -1;
|
|
3006
|
-
}
|
|
3007
|
-
return 0;
|
|
3008
|
-
});
|
|
3009
|
-
}
|
|
3010
|
-
if (options?.offset !== undefined) {
|
|
3011
|
-
values = values.slice(options.offset);
|
|
3012
|
-
}
|
|
3013
|
-
if (options?.limit !== undefined) {
|
|
3014
|
-
values = values.slice(0, options.limit);
|
|
3015
|
-
}
|
|
3016
|
-
resolve(values.length > 0 ? values : undefined);
|
|
3017
|
-
};
|
|
3018
|
-
});
|
|
2423
|
+
return await this.inMemoryRepo.get(key);
|
|
3019
2424
|
}
|
|
3020
|
-
async delete(
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
const store = transaction.objectStore(this.table);
|
|
3025
|
-
const request = store.delete(this.getIndexedKey(key));
|
|
3026
|
-
request.onerror = () => reject(request.error);
|
|
3027
|
-
request.onsuccess = () => {
|
|
3028
|
-
this.events.emit("delete", key);
|
|
3029
|
-
resolve();
|
|
3030
|
-
};
|
|
3031
|
-
transaction.oncomplete = () => {
|
|
3032
|
-
this.hybridManager?.notifyLocalChange();
|
|
3033
|
-
};
|
|
3034
|
-
});
|
|
2425
|
+
async delete(value) {
|
|
2426
|
+
await this.inMemoryRepo.delete(value);
|
|
2427
|
+
const { key } = this.separateKeyValueFromCombined(value);
|
|
2428
|
+
this.broadcast({ type: "DELETE", key });
|
|
3035
2429
|
}
|
|
3036
2430
|
async deleteAll() {
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
const transaction = db.transaction(this.table, "readwrite");
|
|
3040
|
-
const store = transaction.objectStore(this.table);
|
|
3041
|
-
const request = store.clear();
|
|
3042
|
-
request.onerror = () => reject(request.error);
|
|
3043
|
-
request.onsuccess = () => {
|
|
3044
|
-
this.events.emit("clearall");
|
|
3045
|
-
resolve();
|
|
3046
|
-
};
|
|
3047
|
-
transaction.oncomplete = () => {
|
|
3048
|
-
this.hybridManager?.notifyLocalChange();
|
|
3049
|
-
};
|
|
3050
|
-
});
|
|
3051
|
-
}
|
|
3052
|
-
async size() {
|
|
3053
|
-
const db = await this.getDb();
|
|
3054
|
-
return new Promise((resolve, reject) => {
|
|
3055
|
-
const transaction = db.transaction(this.table, "readonly");
|
|
3056
|
-
const store = transaction.objectStore(this.table);
|
|
3057
|
-
const request = store.count();
|
|
3058
|
-
request.onerror = () => reject(request.error);
|
|
3059
|
-
request.onsuccess = () => resolve(request.result);
|
|
3060
|
-
});
|
|
3061
|
-
}
|
|
3062
|
-
getCursorSafeIndexes() {
|
|
3063
|
-
if (this.cursorSafeIndexes)
|
|
3064
|
-
return this.cursorSafeIndexes;
|
|
3065
|
-
const required = new Set(this.schema.required ?? []);
|
|
3066
|
-
this.cursorSafeIndexes = this.indexes.filter((columns) => columns.every((column) => required.has(String(column))));
|
|
3067
|
-
return this.cursorSafeIndexes;
|
|
2431
|
+
await this.inMemoryRepo.deleteAll();
|
|
2432
|
+
this.broadcast({ type: "DELETE_ALL" });
|
|
3068
2433
|
}
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
if (criteriaColumns.length === 0)
|
|
3072
|
-
return;
|
|
3073
|
-
let best;
|
|
3074
|
-
for (const indexColumns of this.getCursorSafeIndexes()) {
|
|
3075
|
-
const prefixValues = [];
|
|
3076
|
-
for (const column of indexColumns) {
|
|
3077
|
-
const value = this.getEqualityCriterionValue(criteria, column);
|
|
3078
|
-
if (value === undefined)
|
|
3079
|
-
break;
|
|
3080
|
-
prefixValues.push(value);
|
|
3081
|
-
}
|
|
3082
|
-
if (prefixValues.length === 0)
|
|
3083
|
-
continue;
|
|
3084
|
-
const indexedPrefix = indexColumns.slice(0, prefixValues.length);
|
|
3085
|
-
const coversCriteria = criteriaColumns.every((column) => indexedPrefix.includes(column));
|
|
3086
|
-
const better = !best || coversCriteria && !best.coversCriteria || coversCriteria === best.coversCriteria && prefixValues.length > best.prefixValues.length;
|
|
3087
|
-
if (better) {
|
|
3088
|
-
best = {
|
|
3089
|
-
indexName: indexColumns.map((column) => String(column)).join("_"),
|
|
3090
|
-
prefixValues,
|
|
3091
|
-
fullMatch: prefixValues.length === indexColumns.length,
|
|
3092
|
-
coversCriteria
|
|
3093
|
-
};
|
|
3094
|
-
}
|
|
3095
|
-
}
|
|
3096
|
-
if (!best)
|
|
3097
|
-
return;
|
|
3098
|
-
const range = best.fullMatch ? IDBKeyRange.only(best.prefixValues.length === 1 ? best.prefixValues[0] : best.prefixValues) : IDBKeyRange.bound(best.prefixValues, [...best.prefixValues, []]);
|
|
3099
|
-
return {
|
|
3100
|
-
source: store.index(best.indexName),
|
|
3101
|
-
range,
|
|
3102
|
-
coversCriteria: best.coversCriteria
|
|
3103
|
-
};
|
|
2434
|
+
async getAll(options) {
|
|
2435
|
+
return await this.inMemoryRepo.getAll(options);
|
|
3104
2436
|
}
|
|
3105
|
-
async
|
|
3106
|
-
|
|
3107
|
-
return await this.size();
|
|
3108
|
-
}
|
|
3109
|
-
this.validateQueryParams(criteria);
|
|
3110
|
-
const db = await this.getDb();
|
|
3111
|
-
return new Promise((resolve, reject) => {
|
|
3112
|
-
const transaction = db.transaction(this.table, "readonly");
|
|
3113
|
-
const store = transaction.objectStore(this.table);
|
|
3114
|
-
const plan = this.createIndexedRange(store, criteria);
|
|
3115
|
-
if (plan?.coversCriteria) {
|
|
3116
|
-
const request2 = plan.source.count(plan.range);
|
|
3117
|
-
request2.onerror = () => reject(request2.error);
|
|
3118
|
-
request2.onsuccess = () => resolve(request2.result);
|
|
3119
|
-
return;
|
|
3120
|
-
}
|
|
3121
|
-
const source = plan?.source ?? store;
|
|
3122
|
-
const range = plan?.range;
|
|
3123
|
-
let count = 0;
|
|
3124
|
-
const request = source.openCursor(range);
|
|
3125
|
-
request.onerror = () => reject(request.error);
|
|
3126
|
-
request.onsuccess = () => {
|
|
3127
|
-
const cursor = request.result;
|
|
3128
|
-
if (!cursor) {
|
|
3129
|
-
resolve(count);
|
|
3130
|
-
return;
|
|
3131
|
-
}
|
|
3132
|
-
if (this.matchesCriteria(cursor.value, criteria)) {
|
|
3133
|
-
count += 1;
|
|
3134
|
-
}
|
|
3135
|
-
cursor.continue();
|
|
3136
|
-
};
|
|
3137
|
-
});
|
|
2437
|
+
async size() {
|
|
2438
|
+
return await this.inMemoryRepo.size();
|
|
3138
2439
|
}
|
|
3139
2440
|
async getBulk(offset, limit) {
|
|
3140
|
-
|
|
3141
|
-
throw new RangeError(`offset must be non-negative, got ${offset}`);
|
|
3142
|
-
}
|
|
3143
|
-
if (limit <= 0) {
|
|
3144
|
-
return;
|
|
3145
|
-
}
|
|
3146
|
-
const db = await this.getDb();
|
|
3147
|
-
return new Promise((resolve, reject) => {
|
|
3148
|
-
const transaction = db.transaction(this.table, "readonly");
|
|
3149
|
-
const store = transaction.objectStore(this.table);
|
|
3150
|
-
const request = store.openCursor();
|
|
3151
|
-
const entities = [];
|
|
3152
|
-
let skipped = false;
|
|
3153
|
-
request.onerror = () => reject(request.error);
|
|
3154
|
-
request.onsuccess = () => {
|
|
3155
|
-
const cursor = request.result;
|
|
3156
|
-
if (cursor) {
|
|
3157
|
-
if (!skipped && offset > 0) {
|
|
3158
|
-
skipped = true;
|
|
3159
|
-
cursor.advance(offset);
|
|
3160
|
-
return;
|
|
3161
|
-
}
|
|
3162
|
-
entities.push(cursor.value);
|
|
3163
|
-
if (entities.length === limit) {
|
|
3164
|
-
resolve(entities);
|
|
3165
|
-
return;
|
|
3166
|
-
}
|
|
3167
|
-
cursor.continue();
|
|
3168
|
-
} else {
|
|
3169
|
-
resolve(entities.length > 0 ? entities : undefined);
|
|
3170
|
-
}
|
|
3171
|
-
};
|
|
3172
|
-
});
|
|
3173
|
-
}
|
|
3174
|
-
matchesCriteria(record, criteria) {
|
|
3175
|
-
for (const column of Object.keys(criteria)) {
|
|
3176
|
-
const criterion = criteria[column];
|
|
3177
|
-
const recordValue = record[column];
|
|
3178
|
-
let operator = "=";
|
|
3179
|
-
let value;
|
|
3180
|
-
if (isSearchCondition(criterion)) {
|
|
3181
|
-
operator = criterion.operator;
|
|
3182
|
-
value = criterion.value;
|
|
3183
|
-
} else {
|
|
3184
|
-
value = criterion;
|
|
3185
|
-
}
|
|
3186
|
-
if (operator !== "=" && (recordValue === null || recordValue === undefined)) {
|
|
3187
|
-
return false;
|
|
3188
|
-
}
|
|
3189
|
-
switch (operator) {
|
|
3190
|
-
case "=":
|
|
3191
|
-
if (recordValue !== value)
|
|
3192
|
-
return false;
|
|
3193
|
-
break;
|
|
3194
|
-
case "<":
|
|
3195
|
-
if (!(recordValue < value))
|
|
3196
|
-
return false;
|
|
3197
|
-
break;
|
|
3198
|
-
case "<=":
|
|
3199
|
-
if (!(recordValue <= value))
|
|
3200
|
-
return false;
|
|
3201
|
-
break;
|
|
3202
|
-
case ">":
|
|
3203
|
-
if (!(recordValue > value))
|
|
3204
|
-
return false;
|
|
3205
|
-
break;
|
|
3206
|
-
case ">=":
|
|
3207
|
-
if (!(recordValue >= value))
|
|
3208
|
-
return false;
|
|
3209
|
-
break;
|
|
3210
|
-
default:
|
|
3211
|
-
return false;
|
|
3212
|
-
}
|
|
3213
|
-
}
|
|
3214
|
-
return true;
|
|
3215
|
-
}
|
|
3216
|
-
async deleteSearch(criteria) {
|
|
3217
|
-
const criteriaKeys = Object.keys(criteria);
|
|
3218
|
-
if (criteriaKeys.length === 0) {
|
|
3219
|
-
return;
|
|
3220
|
-
}
|
|
3221
|
-
const db = await this.getDb();
|
|
3222
|
-
return new Promise(async (resolve, reject) => {
|
|
3223
|
-
try {
|
|
3224
|
-
const transaction = db.transaction(this.table, "readwrite");
|
|
3225
|
-
const store = transaction.objectStore(this.table);
|
|
3226
|
-
transaction.oncomplete = () => {
|
|
3227
|
-
this.events.emit("delete", criteriaKeys[0]);
|
|
3228
|
-
this.hybridManager?.notifyLocalChange();
|
|
3229
|
-
resolve();
|
|
3230
|
-
};
|
|
3231
|
-
transaction.onerror = () => {
|
|
3232
|
-
reject(transaction.error);
|
|
3233
|
-
};
|
|
3234
|
-
const getAllRequest = store.getAll();
|
|
3235
|
-
getAllRequest.onsuccess = () => {
|
|
3236
|
-
const allRecords = getAllRequest.result;
|
|
3237
|
-
const recordsToDelete = allRecords.filter((record) => this.matchesCriteria(record, criteria));
|
|
3238
|
-
if (recordsToDelete.length === 0) {
|
|
3239
|
-
return;
|
|
3240
|
-
}
|
|
3241
|
-
for (const record of recordsToDelete) {
|
|
3242
|
-
const primaryKey = this.primaryKeyColumns().reduce((key, col) => {
|
|
3243
|
-
key[col] = record[col];
|
|
3244
|
-
return key;
|
|
3245
|
-
}, {});
|
|
3246
|
-
const request = store.delete(this.getIndexedKey(primaryKey));
|
|
3247
|
-
request.onerror = () => {
|
|
3248
|
-
console.error("Error deleting record:", request.error);
|
|
3249
|
-
};
|
|
3250
|
-
}
|
|
3251
|
-
};
|
|
3252
|
-
getAllRequest.onerror = () => {
|
|
3253
|
-
reject(getAllRequest.error);
|
|
3254
|
-
};
|
|
3255
|
-
} catch (error) {
|
|
3256
|
-
reject(error);
|
|
3257
|
-
}
|
|
3258
|
-
});
|
|
3259
|
-
}
|
|
3260
|
-
getEqualityCriterionValue(criteria, column) {
|
|
3261
|
-
const criterion = criteria[column];
|
|
3262
|
-
if (criterion === undefined)
|
|
3263
|
-
return;
|
|
3264
|
-
if (isSearchCondition(criterion)) {
|
|
3265
|
-
return criterion.operator === "=" ? criterion.value : undefined;
|
|
3266
|
-
}
|
|
3267
|
-
return criterion;
|
|
3268
|
-
}
|
|
3269
|
-
compareByOrder(a, b, options) {
|
|
3270
|
-
if (!options?.orderBy)
|
|
3271
|
-
return 0;
|
|
3272
|
-
for (const { column, direction } of options.orderBy) {
|
|
3273
|
-
const aVal = a[column];
|
|
3274
|
-
const bVal = b[column];
|
|
3275
|
-
if (aVal == null && bVal == null)
|
|
3276
|
-
continue;
|
|
3277
|
-
if (aVal == null)
|
|
3278
|
-
return direction === "ASC" ? -1 : 1;
|
|
3279
|
-
if (bVal == null)
|
|
3280
|
-
return direction === "ASC" ? 1 : -1;
|
|
3281
|
-
if (aVal < bVal)
|
|
3282
|
-
return direction === "ASC" ? -1 : 1;
|
|
3283
|
-
if (aVal > bVal)
|
|
3284
|
-
return direction === "ASC" ? 1 : -1;
|
|
3285
|
-
}
|
|
3286
|
-
return 0;
|
|
3287
|
-
}
|
|
3288
|
-
createIndexedQuery(store, criteria, options) {
|
|
3289
|
-
const orderBy = options?.orderBy ?? [];
|
|
3290
|
-
let best;
|
|
3291
|
-
for (const indexColumns of this.getCursorSafeIndexes()) {
|
|
3292
|
-
const prefixValues = [];
|
|
3293
|
-
for (const column of indexColumns) {
|
|
3294
|
-
const value = this.getEqualityCriterionValue(criteria, column);
|
|
3295
|
-
if (value === undefined)
|
|
3296
|
-
break;
|
|
3297
|
-
prefixValues.push(value);
|
|
3298
|
-
}
|
|
3299
|
-
if (prefixValues.length === 0)
|
|
3300
|
-
continue;
|
|
3301
|
-
const remainingColumns = indexColumns.slice(prefixValues.length);
|
|
3302
|
-
let redundantOrderPrefixLength = 0;
|
|
3303
|
-
while (redundantOrderPrefixLength < orderBy.length && redundantOrderPrefixLength < prefixValues.length && orderBy[redundantOrderPrefixLength]?.column === indexColumns[redundantOrderPrefixLength]) {
|
|
3304
|
-
redundantOrderPrefixLength++;
|
|
3305
|
-
}
|
|
3306
|
-
const normalizedOrderBy = orderBy.slice(redundantOrderPrefixLength);
|
|
3307
|
-
const satisfiesOrder = normalizedOrderBy.length === 0 || normalizedOrderBy.length <= remainingColumns.length && normalizedOrderBy.every((order, index) => order.column === remainingColumns[index]) && orderBy.every((order) => order.direction === orderBy[0]?.direction);
|
|
3308
|
-
if (!satisfiesOrder && best)
|
|
3309
|
-
continue;
|
|
3310
|
-
if (!best || satisfiesOrder && !best.satisfiesOrder || prefixValues.length > best.prefixValues.length) {
|
|
3311
|
-
best = {
|
|
3312
|
-
indexName: indexColumns.map((column) => String(column)).join("_"),
|
|
3313
|
-
prefixValues,
|
|
3314
|
-
fullMatch: prefixValues.length === indexColumns.length,
|
|
3315
|
-
satisfiesOrder,
|
|
3316
|
-
direction: orderBy[0]?.direction === "DESC" ? "prev" : "next"
|
|
3317
|
-
};
|
|
3318
|
-
}
|
|
3319
|
-
}
|
|
3320
|
-
const appliedLimit = Boolean(best?.satisfiesOrder && options?.limit !== undefined);
|
|
3321
|
-
const appliedOffset = Boolean(best?.satisfiesOrder && options?.offset !== undefined);
|
|
3322
|
-
if (!best) {
|
|
3323
|
-
return {
|
|
3324
|
-
source: store,
|
|
3325
|
-
range: undefined,
|
|
3326
|
-
direction: orderBy[0]?.direction === "DESC" ? "prev" : "next",
|
|
3327
|
-
satisfiesOrder: false,
|
|
3328
|
-
appliedLimit: false,
|
|
3329
|
-
appliedOffset: false,
|
|
3330
|
-
skipRemaining: 0
|
|
3331
|
-
};
|
|
3332
|
-
}
|
|
3333
|
-
const source = store.index(best.indexName);
|
|
3334
|
-
const keyRange = best.fullMatch ? IDBKeyRange.only(best.prefixValues.length === 1 ? best.prefixValues[0] : best.prefixValues) : IDBKeyRange.bound(best.prefixValues, [...best.prefixValues, []]);
|
|
3335
|
-
return {
|
|
3336
|
-
source,
|
|
3337
|
-
range: keyRange,
|
|
3338
|
-
direction: best.direction,
|
|
3339
|
-
satisfiesOrder: best.satisfiesOrder,
|
|
3340
|
-
appliedLimit,
|
|
3341
|
-
appliedOffset,
|
|
3342
|
-
skipRemaining: appliedOffset ? options?.offset ?? 0 : 0
|
|
3343
|
-
};
|
|
2441
|
+
return await this.inMemoryRepo.getBulk(offset, limit);
|
|
3344
2442
|
}
|
|
3345
2443
|
async query(criteria, options) {
|
|
3346
|
-
this.
|
|
3347
|
-
const db = await this.getDb();
|
|
3348
|
-
return new Promise((resolve, reject) => {
|
|
3349
|
-
const transaction = db.transaction(this.table, "readonly");
|
|
3350
|
-
const store = transaction.objectStore(this.table);
|
|
3351
|
-
const indexedQuery = this.createIndexedQuery(store, criteria, options);
|
|
3352
|
-
const results = [];
|
|
3353
|
-
const request = indexedQuery.source.openCursor(indexedQuery.range, indexedQuery.direction);
|
|
3354
|
-
request.onsuccess = () => {
|
|
3355
|
-
const cursor = request.result;
|
|
3356
|
-
if (!cursor) {
|
|
3357
|
-
let finalResults = results;
|
|
3358
|
-
if (!indexedQuery.satisfiesOrder && options?.orderBy && options.orderBy.length > 0) {
|
|
3359
|
-
finalResults = [...finalResults].sort((a, b) => this.compareByOrder(a, b, options));
|
|
3360
|
-
}
|
|
3361
|
-
if (!indexedQuery.appliedOffset && options?.offset !== undefined) {
|
|
3362
|
-
finalResults = finalResults.slice(options.offset);
|
|
3363
|
-
}
|
|
3364
|
-
if (!indexedQuery.appliedLimit && options?.limit !== undefined) {
|
|
3365
|
-
finalResults = finalResults.slice(0, options.limit);
|
|
3366
|
-
}
|
|
3367
|
-
const result = finalResults.length > 0 ? finalResults : undefined;
|
|
3368
|
-
this.events.emit("query", criteria, result);
|
|
3369
|
-
resolve(result);
|
|
3370
|
-
return;
|
|
3371
|
-
}
|
|
3372
|
-
const record = cursor.value;
|
|
3373
|
-
if (this.matchesCriteria(record, criteria)) {
|
|
3374
|
-
if (indexedQuery.skipRemaining > 0) {
|
|
3375
|
-
indexedQuery.skipRemaining -= 1;
|
|
3376
|
-
} else {
|
|
3377
|
-
results.push(record);
|
|
3378
|
-
if (indexedQuery.appliedLimit && results.length === options?.limit) {
|
|
3379
|
-
const result = results.length > 0 ? results : undefined;
|
|
3380
|
-
this.events.emit("query", criteria, result);
|
|
3381
|
-
resolve(result);
|
|
3382
|
-
return;
|
|
3383
|
-
}
|
|
3384
|
-
}
|
|
3385
|
-
}
|
|
3386
|
-
cursor.continue();
|
|
3387
|
-
};
|
|
3388
|
-
request.onerror = () => reject(request.error);
|
|
3389
|
-
});
|
|
2444
|
+
return await this.inMemoryRepo.query(criteria, options);
|
|
3390
2445
|
}
|
|
3391
2446
|
async queryIndex(criteria, options) {
|
|
3392
|
-
this.
|
|
3393
|
-
this.validateQueryParams(criteria, options);
|
|
3394
|
-
const registered = this.indexes.map((cols) => {
|
|
3395
|
-
const cs = Array.isArray(cols) ? cols : [cols];
|
|
3396
|
-
return { name: cs.join("_"), keyPath: cs };
|
|
3397
|
-
});
|
|
3398
|
-
const picked = pickCoveringIndex({
|
|
3399
|
-
table: this.table,
|
|
3400
|
-
indexes: registered,
|
|
3401
|
-
criteriaColumns: Object.keys(criteria),
|
|
3402
|
-
orderByColumns: (options.orderBy ?? []).map((o) => ({
|
|
3403
|
-
column: String(o.column),
|
|
3404
|
-
direction: o.direction
|
|
3405
|
-
})),
|
|
3406
|
-
selectColumns: options.select.map(String),
|
|
3407
|
-
primaryKeyColumns: this.primaryKeyColumns().map(String)
|
|
3408
|
-
});
|
|
3409
|
-
const db = await this.getDb();
|
|
3410
|
-
return new Promise((resolve, reject) => {
|
|
3411
|
-
const tx = db.transaction(this.table, "readonly");
|
|
3412
|
-
const store = tx.objectStore(this.table);
|
|
3413
|
-
const idx = store.index(picked.name);
|
|
3414
|
-
const prefix = [];
|
|
3415
|
-
for (const col of picked.keyPath) {
|
|
3416
|
-
const c = criteria[col];
|
|
3417
|
-
if (c === undefined && !(col in criteria))
|
|
3418
|
-
break;
|
|
3419
|
-
if (isSearchCondition(c)) {
|
|
3420
|
-
if (c.operator !== "=")
|
|
3421
|
-
break;
|
|
3422
|
-
prefix.push(c.value);
|
|
3423
|
-
} else {
|
|
3424
|
-
prefix.push(c);
|
|
3425
|
-
}
|
|
3426
|
-
}
|
|
3427
|
-
const range = prefix.length === 0 ? undefined : prefix.length === picked.keyPath.length ? IDBKeyRange.only(prefix.length === 1 ? prefix[0] : prefix) : IDBKeyRange.bound(prefix, [...prefix, []]);
|
|
3428
|
-
const direction = picked.reverseDirection ? "prev" : "next";
|
|
3429
|
-
const request = idx.openKeyCursor(range, direction);
|
|
3430
|
-
const out = [];
|
|
3431
|
-
let toSkip = options.offset ?? 0;
|
|
3432
|
-
const keyPathPositions = new Map;
|
|
3433
|
-
picked.keyPath.forEach((col, i) => keyPathPositions.set(col, i));
|
|
3434
|
-
const pkCols = this.primaryKeyColumns().map(String);
|
|
3435
|
-
const pkPositions = new Map;
|
|
3436
|
-
pkCols.forEach((col, i) => pkPositions.set(col, i));
|
|
3437
|
-
request.onsuccess = () => {
|
|
3438
|
-
const cursor = request.result;
|
|
3439
|
-
if (!cursor) {
|
|
3440
|
-
resolve(out);
|
|
3441
|
-
return;
|
|
3442
|
-
}
|
|
3443
|
-
const key = cursor.key;
|
|
3444
|
-
const row = {};
|
|
3445
|
-
for (const col of options.select) {
|
|
3446
|
-
const colStr = String(col);
|
|
3447
|
-
const pos = keyPathPositions.get(colStr);
|
|
3448
|
-
if (pos !== undefined) {
|
|
3449
|
-
row[colStr] = Array.isArray(key) ? key[pos] : key;
|
|
3450
|
-
} else {
|
|
3451
|
-
if (pkCols.length === 1 && colStr === pkCols[0]) {
|
|
3452
|
-
row[colStr] = cursor.primaryKey;
|
|
3453
|
-
} else {
|
|
3454
|
-
const pkPos = pkPositions.get(colStr);
|
|
3455
|
-
if (pkPos !== undefined) {
|
|
3456
|
-
row[colStr] = Array.isArray(cursor.primaryKey) ? cursor.primaryKey[pkPos] : cursor.primaryKey;
|
|
3457
|
-
}
|
|
3458
|
-
}
|
|
3459
|
-
}
|
|
3460
|
-
}
|
|
3461
|
-
let matches = true;
|
|
3462
|
-
for (const [col, crit] of Object.entries(criteria)) {
|
|
3463
|
-
const pos = keyPathPositions.get(col);
|
|
3464
|
-
if (pos === undefined)
|
|
3465
|
-
continue;
|
|
3466
|
-
if (pos < prefix.length)
|
|
3467
|
-
continue;
|
|
3468
|
-
const valFromKey = Array.isArray(key) ? key[pos] : key;
|
|
3469
|
-
const op = isSearchCondition(crit) ? crit.operator : "=";
|
|
3470
|
-
const val = isSearchCondition(crit) ? crit.value : crit;
|
|
3471
|
-
if (!compareWithOperator(valFromKey, op, val)) {
|
|
3472
|
-
matches = false;
|
|
3473
|
-
break;
|
|
3474
|
-
}
|
|
3475
|
-
}
|
|
3476
|
-
if (matches) {
|
|
3477
|
-
if (toSkip > 0) {
|
|
3478
|
-
toSkip -= 1;
|
|
3479
|
-
} else {
|
|
3480
|
-
out.push(row);
|
|
3481
|
-
if (options.limit !== undefined && out.length >= options.limit) {
|
|
3482
|
-
resolve(out);
|
|
3483
|
-
return;
|
|
3484
|
-
}
|
|
3485
|
-
}
|
|
3486
|
-
}
|
|
3487
|
-
cursor.continue();
|
|
3488
|
-
};
|
|
3489
|
-
request.onerror = () => reject(request.error);
|
|
3490
|
-
});
|
|
2447
|
+
return await this.inMemoryRepo.queryIndex(criteria, options);
|
|
3491
2448
|
}
|
|
3492
|
-
|
|
3493
|
-
|
|
3494
|
-
|
|
3495
|
-
|
|
3496
|
-
|
|
3497
|
-
|
|
3498
|
-
for (const entity of entities) {
|
|
3499
|
-
const { key } = this.separateKeyValueFromCombined(entity);
|
|
3500
|
-
const fingerprint = await makeFingerprint5(key);
|
|
3501
|
-
map.set(fingerprint, entity);
|
|
3502
|
-
}
|
|
3503
|
-
return map;
|
|
3504
|
-
}, compareEntitiesForChange, {
|
|
3505
|
-
insert: (item) => ({ type: "INSERT", new: item }),
|
|
3506
|
-
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
3507
|
-
delete: (item) => ({ type: "DELETE", old: item })
|
|
3508
|
-
}, {
|
|
3509
|
-
defaultIntervalMs: 1000,
|
|
3510
|
-
useBroadcastChannel: this.hybridOptions.useBroadcastChannel,
|
|
3511
|
-
backupPollingIntervalMs: this.hybridOptions.backupPollingIntervalMs
|
|
3512
|
-
});
|
|
3513
|
-
}
|
|
3514
|
-
return this.hybridManager;
|
|
2449
|
+
async deleteSearch(criteria) {
|
|
2450
|
+
await this.inMemoryRepo.deleteSearch(criteria);
|
|
2451
|
+
this.broadcast({
|
|
2452
|
+
type: "DELETE_SEARCH",
|
|
2453
|
+
criteria
|
|
2454
|
+
});
|
|
3515
2455
|
}
|
|
3516
2456
|
subscribeToChanges(callback, options) {
|
|
3517
|
-
|
|
3518
|
-
const manager = this.getHybridManager();
|
|
3519
|
-
return manager.subscribe(callback, { intervalMs });
|
|
2457
|
+
return this.inMemoryRepo.subscribeToChanges(callback, options);
|
|
3520
2458
|
}
|
|
3521
2459
|
destroy() {
|
|
3522
|
-
if (this.
|
|
3523
|
-
this.
|
|
3524
|
-
this.
|
|
3525
|
-
}
|
|
3526
|
-
this.db?.close();
|
|
3527
|
-
}
|
|
3528
|
-
}
|
|
3529
|
-
function compareWithOperator(a, op, b) {
|
|
3530
|
-
const av = a;
|
|
3531
|
-
const bv = b;
|
|
3532
|
-
switch (op) {
|
|
3533
|
-
case "=":
|
|
3534
|
-
return av === bv;
|
|
3535
|
-
case "<":
|
|
3536
|
-
return av !== null && av !== undefined && av < bv;
|
|
3537
|
-
case "<=":
|
|
3538
|
-
return av !== null && av !== undefined && av <= bv;
|
|
3539
|
-
case ">":
|
|
3540
|
-
return av !== null && av !== undefined && av > bv;
|
|
3541
|
-
case ">=":
|
|
3542
|
-
return av !== null && av !== undefined && av >= bv;
|
|
3543
|
-
}
|
|
3544
|
-
}
|
|
3545
|
-
// src/tabular/SharedInMemoryTabularStorage.ts
|
|
3546
|
-
import { createServiceToken as createServiceToken13 } from "@workglow/util";
|
|
3547
|
-
var SHARED_IN_MEMORY_TABULAR_REPOSITORY = createServiceToken13("storage.tabularRepository.sharedInMemory");
|
|
3548
|
-
var SYNC_TIMEOUT = 1000;
|
|
3549
|
-
var MAX_PENDING_MESSAGES = 1000;
|
|
3550
|
-
|
|
3551
|
-
class SharedInMemoryTabularStorage extends BaseTabularStorage {
|
|
3552
|
-
channel = null;
|
|
3553
|
-
channelName;
|
|
3554
|
-
inMemoryRepo;
|
|
3555
|
-
isInitialized = false;
|
|
3556
|
-
syncInProgress = false;
|
|
3557
|
-
pendingMessages = [];
|
|
3558
|
-
constructor(channelName = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
|
|
3559
|
-
super(schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
3560
|
-
this.channelName = channelName;
|
|
3561
|
-
this.inMemoryRepo = new InMemoryTabularStorage(schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
3562
|
-
this.setupEventForwarding();
|
|
3563
|
-
this.initializeBroadcastChannel();
|
|
3564
|
-
}
|
|
3565
|
-
isBroadcastChannelAvailable() {
|
|
3566
|
-
return typeof BroadcastChannel !== "undefined";
|
|
3567
|
-
}
|
|
3568
|
-
initializeBroadcastChannel() {
|
|
3569
|
-
if (!this.isBroadcastChannelAvailable()) {
|
|
3570
|
-
console.warn("BroadcastChannel is not available. Tab synchronization will not work.");
|
|
3571
|
-
return;
|
|
3572
|
-
}
|
|
3573
|
-
try {
|
|
3574
|
-
this.channel = new BroadcastChannel(this.channelName);
|
|
3575
|
-
this.channel.onmessage = (event) => {
|
|
3576
|
-
this.handleBroadcastMessage(event.data);
|
|
3577
|
-
};
|
|
3578
|
-
this.syncFromOtherTabs();
|
|
3579
|
-
} catch (error) {
|
|
3580
|
-
console.error("Failed to initialize BroadcastChannel:", error);
|
|
3581
|
-
}
|
|
3582
|
-
}
|
|
3583
|
-
setupEventForwarding() {
|
|
3584
|
-
this.inMemoryRepo.on("put", (entity) => {
|
|
3585
|
-
this.events.emit("put", entity);
|
|
3586
|
-
});
|
|
3587
|
-
this.inMemoryRepo.on("get", (key, entity) => {
|
|
3588
|
-
this.events.emit("get", key, entity);
|
|
3589
|
-
});
|
|
3590
|
-
this.inMemoryRepo.on("query", (key, entities) => {
|
|
3591
|
-
this.events.emit("query", key, entities);
|
|
3592
|
-
});
|
|
3593
|
-
this.inMemoryRepo.on("delete", (key) => {
|
|
3594
|
-
this.events.emit("delete", key);
|
|
3595
|
-
});
|
|
3596
|
-
this.inMemoryRepo.on("clearall", () => {
|
|
3597
|
-
this.events.emit("clearall");
|
|
3598
|
-
});
|
|
3599
|
-
}
|
|
3600
|
-
async handleBroadcastMessage(message) {
|
|
3601
|
-
if (this.syncInProgress && message.type !== "SYNC_RESPONSE") {
|
|
3602
|
-
if (this.pendingMessages.length < MAX_PENDING_MESSAGES) {
|
|
3603
|
-
this.pendingMessages.push(message);
|
|
3604
|
-
}
|
|
3605
|
-
return;
|
|
3606
|
-
}
|
|
3607
|
-
switch (message.type) {
|
|
3608
|
-
case "SYNC_REQUEST":
|
|
3609
|
-
const all = await this.inMemoryRepo.getAll();
|
|
3610
|
-
if (this.channel && all) {
|
|
3611
|
-
this.channel.postMessage({
|
|
3612
|
-
type: "SYNC_RESPONSE",
|
|
3613
|
-
data: all
|
|
3614
|
-
});
|
|
3615
|
-
}
|
|
3616
|
-
break;
|
|
3617
|
-
case "SYNC_RESPONSE":
|
|
3618
|
-
if (message.data && Array.isArray(message.data)) {
|
|
3619
|
-
await this.copyDataFromArray(message.data);
|
|
3620
|
-
}
|
|
3621
|
-
this.syncInProgress = false;
|
|
3622
|
-
await this.drainPendingMessages();
|
|
3623
|
-
break;
|
|
3624
|
-
case "PUT":
|
|
3625
|
-
await this.inMemoryRepo.put(message.entity);
|
|
3626
|
-
break;
|
|
3627
|
-
case "PUT_BULK":
|
|
3628
|
-
await this.inMemoryRepo.putBulk(message.entities);
|
|
3629
|
-
break;
|
|
3630
|
-
case "DELETE":
|
|
3631
|
-
await this.inMemoryRepo.delete(message.key);
|
|
3632
|
-
break;
|
|
3633
|
-
case "DELETE_ALL":
|
|
3634
|
-
await this.inMemoryRepo.deleteAll();
|
|
3635
|
-
break;
|
|
3636
|
-
case "DELETE_SEARCH":
|
|
3637
|
-
await this.inMemoryRepo.deleteSearch(message.criteria);
|
|
3638
|
-
break;
|
|
3639
|
-
}
|
|
3640
|
-
}
|
|
3641
|
-
async drainPendingMessages() {
|
|
3642
|
-
while (!this.syncInProgress && this.pendingMessages.length > 0) {
|
|
3643
|
-
const messages = this.pendingMessages;
|
|
3644
|
-
this.pendingMessages = [];
|
|
3645
|
-
for (const message of messages) {
|
|
3646
|
-
await this.handleBroadcastMessage(message);
|
|
3647
|
-
if (this.syncInProgress) {
|
|
3648
|
-
break;
|
|
3649
|
-
}
|
|
3650
|
-
}
|
|
3651
|
-
}
|
|
3652
|
-
}
|
|
3653
|
-
syncFromOtherTabs() {
|
|
3654
|
-
if (!this.channel)
|
|
3655
|
-
return;
|
|
3656
|
-
this.syncInProgress = true;
|
|
3657
|
-
this.channel.postMessage({ type: "SYNC_REQUEST" });
|
|
3658
|
-
setTimeout(() => {
|
|
3659
|
-
if (this.syncInProgress) {
|
|
3660
|
-
this.syncInProgress = false;
|
|
3661
|
-
this.drainPendingMessages().catch((error) => {
|
|
3662
|
-
console.error("Failed to drain pending messages after sync timeout", error);
|
|
3663
|
-
});
|
|
3664
|
-
}
|
|
3665
|
-
}, SYNC_TIMEOUT);
|
|
3666
|
-
}
|
|
3667
|
-
async copyDataFromArray(entities) {
|
|
3668
|
-
if (entities.length === 0)
|
|
3669
|
-
return;
|
|
3670
|
-
await this.inMemoryRepo.deleteAll();
|
|
3671
|
-
await this.inMemoryRepo.putBulk(entities);
|
|
3672
|
-
}
|
|
3673
|
-
broadcast(message) {
|
|
3674
|
-
if (this.channel) {
|
|
3675
|
-
this.channel.postMessage(message);
|
|
3676
|
-
}
|
|
3677
|
-
}
|
|
3678
|
-
async setupDatabase() {
|
|
3679
|
-
if (this.isInitialized)
|
|
3680
|
-
return;
|
|
3681
|
-
this.isInitialized = true;
|
|
3682
|
-
await this.syncFromOtherTabs();
|
|
3683
|
-
}
|
|
3684
|
-
async put(value) {
|
|
3685
|
-
const result = await this.inMemoryRepo.put(value);
|
|
3686
|
-
this.broadcast({ type: "PUT", entity: result });
|
|
3687
|
-
return result;
|
|
3688
|
-
}
|
|
3689
|
-
async putBulk(values) {
|
|
3690
|
-
const result = await this.inMemoryRepo.putBulk(values);
|
|
3691
|
-
this.broadcast({ type: "PUT_BULK", entities: result });
|
|
3692
|
-
return result;
|
|
3693
|
-
}
|
|
3694
|
-
async get(key) {
|
|
3695
|
-
return await this.inMemoryRepo.get(key);
|
|
3696
|
-
}
|
|
3697
|
-
async delete(value) {
|
|
3698
|
-
await this.inMemoryRepo.delete(value);
|
|
3699
|
-
const { key } = this.separateKeyValueFromCombined(value);
|
|
3700
|
-
this.broadcast({ type: "DELETE", key });
|
|
3701
|
-
}
|
|
3702
|
-
async deleteAll() {
|
|
3703
|
-
await this.inMemoryRepo.deleteAll();
|
|
3704
|
-
this.broadcast({ type: "DELETE_ALL" });
|
|
3705
|
-
}
|
|
3706
|
-
async getAll(options) {
|
|
3707
|
-
return await this.inMemoryRepo.getAll(options);
|
|
3708
|
-
}
|
|
3709
|
-
async size() {
|
|
3710
|
-
return await this.inMemoryRepo.size();
|
|
3711
|
-
}
|
|
3712
|
-
async getBulk(offset, limit) {
|
|
3713
|
-
return await this.inMemoryRepo.getBulk(offset, limit);
|
|
3714
|
-
}
|
|
3715
|
-
async query(criteria, options) {
|
|
3716
|
-
return await this.inMemoryRepo.query(criteria, options);
|
|
3717
|
-
}
|
|
3718
|
-
async queryIndex(criteria, options) {
|
|
3719
|
-
return await this.inMemoryRepo.queryIndex(criteria, options);
|
|
3720
|
-
}
|
|
3721
|
-
async deleteSearch(criteria) {
|
|
3722
|
-
await this.inMemoryRepo.deleteSearch(criteria);
|
|
3723
|
-
this.broadcast({
|
|
3724
|
-
type: "DELETE_SEARCH",
|
|
3725
|
-
criteria
|
|
3726
|
-
});
|
|
3727
|
-
}
|
|
3728
|
-
subscribeToChanges(callback, options) {
|
|
3729
|
-
return this.inMemoryRepo.subscribeToChanges(callback, options);
|
|
3730
|
-
}
|
|
3731
|
-
destroy() {
|
|
3732
|
-
if (this.channel) {
|
|
3733
|
-
this.channel.close();
|
|
3734
|
-
this.channel = null;
|
|
2460
|
+
if (this.channel) {
|
|
2461
|
+
this.channel.close();
|
|
2462
|
+
this.channel = null;
|
|
3735
2463
|
}
|
|
3736
2464
|
this.inMemoryRepo.destroy();
|
|
3737
2465
|
}
|
|
3738
2466
|
}
|
|
3739
|
-
// src/tabular/SupabaseTabularStorage.ts
|
|
3740
|
-
import { createServiceToken as createServiceToken14 } from "@workglow/util";
|
|
3741
|
-
|
|
3742
|
-
// src/tabular/BaseSqlTabularStorage.ts
|
|
3743
|
-
class BaseSqlTabularStorage extends BaseTabularStorage {
|
|
3744
|
-
table;
|
|
3745
|
-
_pkColsCache = new Map;
|
|
3746
|
-
_valColsCache = new Map;
|
|
3747
|
-
_pkColListCache = new Map;
|
|
3748
|
-
_valColListCache = new Map;
|
|
3749
|
-
constructor(table = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
|
|
3750
|
-
super(schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
3751
|
-
this.table = table;
|
|
3752
|
-
this.validateTableAndSchema();
|
|
3753
|
-
}
|
|
3754
|
-
constructPrimaryKeyColumns($delimiter = "") {
|
|
3755
|
-
let cached = this._pkColsCache.get($delimiter);
|
|
3756
|
-
if (cached === undefined) {
|
|
3757
|
-
cached = Object.entries(this.primaryKeySchema.properties).map(([key, typeDef]) => {
|
|
3758
|
-
const sqlType = this.mapTypeToSQL(typeDef);
|
|
3759
|
-
return `${$delimiter}${key}${$delimiter} ${sqlType} NOT NULL`;
|
|
3760
|
-
}).join(", ");
|
|
3761
|
-
this._pkColsCache.set($delimiter, cached);
|
|
3762
|
-
}
|
|
3763
|
-
return cached;
|
|
3764
|
-
}
|
|
3765
|
-
constructValueColumns($delimiter = "") {
|
|
3766
|
-
let cached = this._valColsCache.get($delimiter);
|
|
3767
|
-
if (cached === undefined) {
|
|
3768
|
-
const requiredSet = new Set(this.valueSchema.required ?? []);
|
|
3769
|
-
const cols = Object.entries(this.valueSchema.properties).map(([key, typeDef]) => {
|
|
3770
|
-
const sqlType = this.mapTypeToSQL(typeDef);
|
|
3771
|
-
const isRequired = requiredSet.has(key);
|
|
3772
|
-
const nullable = !isRequired || this.isNullable(typeDef);
|
|
3773
|
-
return `${$delimiter}${key}${$delimiter} ${sqlType}${nullable ? " NULL" : " NOT NULL"}`;
|
|
3774
|
-
}).join(", ");
|
|
3775
|
-
cached = cols.length > 0 ? `, ${cols}` : "";
|
|
3776
|
-
this._valColsCache.set($delimiter, cached);
|
|
3777
|
-
}
|
|
3778
|
-
return cached;
|
|
3779
|
-
}
|
|
3780
|
-
isNullable(typeDef) {
|
|
3781
|
-
if (typeof typeDef === "boolean")
|
|
3782
|
-
return typeDef;
|
|
3783
|
-
if (typeDef.type === "null") {
|
|
3784
|
-
return true;
|
|
3785
|
-
}
|
|
3786
|
-
if (Array.isArray(typeDef.type)) {
|
|
3787
|
-
return typeDef.type.includes("null");
|
|
3788
|
-
}
|
|
3789
|
-
if (typeDef.anyOf && Array.isArray(typeDef.anyOf)) {
|
|
3790
|
-
return typeDef.anyOf.some((type) => type.type === "null");
|
|
3791
|
-
}
|
|
3792
|
-
if (typeDef.oneOf && Array.isArray(typeDef.oneOf)) {
|
|
3793
|
-
return typeDef.oneOf.some((type) => type.type === "null");
|
|
3794
|
-
}
|
|
3795
|
-
return false;
|
|
3796
|
-
}
|
|
3797
|
-
primaryKeyColumnList($delimiter = "") {
|
|
3798
|
-
let cached = this._pkColListCache.get($delimiter);
|
|
3799
|
-
if (cached === undefined) {
|
|
3800
|
-
cached = $delimiter + this.primaryKeyColumns().join(`${$delimiter}, ${$delimiter}`) + $delimiter;
|
|
3801
|
-
this._pkColListCache.set($delimiter, cached);
|
|
3802
|
-
}
|
|
3803
|
-
return cached;
|
|
3804
|
-
}
|
|
3805
|
-
valueColumnList($delimiter = "") {
|
|
3806
|
-
let cached = this._valColListCache.get($delimiter);
|
|
3807
|
-
if (cached === undefined) {
|
|
3808
|
-
cached = $delimiter + this.valueColumns().join(`${$delimiter}, ${$delimiter}`) + $delimiter;
|
|
3809
|
-
this._valColListCache.set($delimiter, cached);
|
|
3810
|
-
}
|
|
3811
|
-
return cached;
|
|
3812
|
-
}
|
|
3813
|
-
getNonNullType(typeDef) {
|
|
3814
|
-
if (typeof typeDef === "boolean")
|
|
3815
|
-
return typeDef;
|
|
3816
|
-
if (typeDef.anyOf && Array.isArray(typeDef.anyOf)) {
|
|
3817
|
-
const nonNullType = typeDef.anyOf.find((t) => t.type !== "null");
|
|
3818
|
-
if (nonNullType) {
|
|
3819
|
-
return nonNullType;
|
|
3820
|
-
}
|
|
3821
|
-
}
|
|
3822
|
-
if (typeDef.oneOf && Array.isArray(typeDef.oneOf)) {
|
|
3823
|
-
const nonNullType = typeDef.oneOf.find((t) => t.type !== "null");
|
|
3824
|
-
if (nonNullType) {
|
|
3825
|
-
return nonNullType;
|
|
3826
|
-
}
|
|
3827
|
-
}
|
|
3828
|
-
return typeDef;
|
|
3829
|
-
}
|
|
3830
|
-
getValueAsOrderedArray(value) {
|
|
3831
|
-
const orderedParams = [];
|
|
3832
|
-
const valueAsRecord = value;
|
|
3833
|
-
const requiredSet = new Set(this.valueSchema.required ?? []);
|
|
3834
|
-
for (const key in this.valueSchema.properties) {
|
|
3835
|
-
if (Object.prototype.hasOwnProperty.call(valueAsRecord, key)) {
|
|
3836
|
-
const val = valueAsRecord[key];
|
|
3837
|
-
if (val === undefined && !requiredSet.has(key)) {
|
|
3838
|
-
orderedParams.push(null);
|
|
3839
|
-
} else {
|
|
3840
|
-
orderedParams.push(this.jsToSqlValue(key, val));
|
|
3841
|
-
}
|
|
3842
|
-
} else {
|
|
3843
|
-
if (requiredSet.has(key)) {
|
|
3844
|
-
throw new Error(`Missing required value field: ${key}`);
|
|
3845
|
-
}
|
|
3846
|
-
orderedParams.push(null);
|
|
3847
|
-
}
|
|
3848
|
-
}
|
|
3849
|
-
return orderedParams;
|
|
3850
|
-
}
|
|
3851
|
-
getPrimaryKeyAsOrderedArray(key) {
|
|
3852
|
-
const orderedParams = [];
|
|
3853
|
-
const keyObj = key;
|
|
3854
|
-
for (const k of Object.keys(this.primaryKeySchema.properties)) {
|
|
3855
|
-
if (k in keyObj) {
|
|
3856
|
-
const value = keyObj[k];
|
|
3857
|
-
if (value === null) {
|
|
3858
|
-
throw new Error(`Primary key field ${k} cannot be null`);
|
|
3859
|
-
}
|
|
3860
|
-
orderedParams.push(this.jsToSqlValue(k, value));
|
|
3861
|
-
} else {
|
|
3862
|
-
throw new Error(`Missing required primary key field: ${k}`);
|
|
3863
|
-
}
|
|
3864
|
-
}
|
|
3865
|
-
return orderedParams;
|
|
3866
|
-
}
|
|
3867
|
-
jsToSqlValue(column, value) {
|
|
3868
|
-
const typeDef = this.schema.properties[column];
|
|
3869
|
-
if (!typeDef) {
|
|
3870
|
-
return value;
|
|
3871
|
-
}
|
|
3872
|
-
if (value === null && this.isNullable(typeDef)) {
|
|
3873
|
-
return null;
|
|
3874
|
-
}
|
|
3875
|
-
const actualType = this.getNonNullType(typeDef);
|
|
3876
|
-
if (typeof actualType === "boolean") {
|
|
3877
|
-
return value;
|
|
3878
|
-
}
|
|
3879
|
-
if (actualType.contentEncoding === "blob") {
|
|
3880
|
-
if (value instanceof Uint8Array) {
|
|
3881
|
-
return value;
|
|
3882
|
-
}
|
|
3883
|
-
if (typeof Buffer !== "undefined" && value instanceof Buffer) {
|
|
3884
|
-
return new Uint8Array(value);
|
|
3885
|
-
}
|
|
3886
|
-
if (Array.isArray(value)) {
|
|
3887
|
-
return new Uint8Array(value);
|
|
3888
|
-
}
|
|
3889
|
-
return value;
|
|
3890
|
-
} else if (value instanceof Date) {
|
|
3891
|
-
return value.toISOString();
|
|
3892
|
-
} else {
|
|
3893
|
-
return value;
|
|
3894
|
-
}
|
|
3895
|
-
}
|
|
3896
|
-
sqlToJsValue(column, value) {
|
|
3897
|
-
const typeDef = this.schema.properties[column];
|
|
3898
|
-
if (!typeDef) {
|
|
3899
|
-
return value;
|
|
3900
|
-
}
|
|
3901
|
-
if (value === null && this.isNullable(typeDef)) {
|
|
3902
|
-
return null;
|
|
3903
|
-
}
|
|
3904
|
-
const actualType = this.getNonNullType(typeDef);
|
|
3905
|
-
if (typeof actualType === "boolean") {
|
|
3906
|
-
return value;
|
|
3907
|
-
}
|
|
3908
|
-
if (actualType.contentEncoding === "blob") {
|
|
3909
|
-
if (typeof Buffer !== "undefined" && value instanceof Buffer) {
|
|
3910
|
-
return new Uint8Array(value);
|
|
3911
|
-
}
|
|
3912
|
-
if (value instanceof Uint8Array) {
|
|
3913
|
-
return value;
|
|
3914
|
-
}
|
|
3915
|
-
return value;
|
|
3916
|
-
} else {
|
|
3917
|
-
return value;
|
|
3918
|
-
}
|
|
3919
|
-
}
|
|
3920
|
-
validateTableAndSchema() {
|
|
3921
|
-
if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(this.table)) {
|
|
3922
|
-
throw new Error("Table name must start with a letter and contain only letters, digits, and underscores, got: " + this.table);
|
|
3923
|
-
}
|
|
3924
|
-
const validateSchemaKeys = (schema) => {
|
|
3925
|
-
for (const key in schema.properties) {
|
|
3926
|
-
if (!/^[a-zA-Z][a-zA-Z0-9_]*$/.test(key)) {
|
|
3927
|
-
throw new Error("Schema keys must start with a letter and contain only letters, digits, and underscores, got: " + key);
|
|
3928
|
-
}
|
|
3929
|
-
}
|
|
3930
|
-
};
|
|
3931
|
-
validateSchemaKeys(this.primaryKeySchema);
|
|
3932
|
-
validateSchemaKeys(this.valueSchema);
|
|
3933
|
-
const primaryKeys = new Set(Object.keys(this.primaryKeySchema.properties));
|
|
3934
|
-
const valueKeys = Object.keys(this.valueSchema.properties);
|
|
3935
|
-
const duplicates = valueKeys.filter((key) => primaryKeys.has(key));
|
|
3936
|
-
if (duplicates.length > 0) {
|
|
3937
|
-
throw new Error(`Duplicate keys found in schemas: ${duplicates.join(", ")}`);
|
|
3938
|
-
}
|
|
3939
|
-
}
|
|
3940
|
-
}
|
|
3941
|
-
|
|
3942
|
-
// src/tabular/SupabaseTabularStorage.ts
|
|
3943
|
-
var SUPABASE_TABULAR_REPOSITORY = createServiceToken14("storage.tabularRepository.supabase");
|
|
3944
|
-
|
|
3945
|
-
class SupabaseTabularStorage extends BaseSqlTabularStorage {
|
|
3946
|
-
client;
|
|
3947
|
-
realtimeChannel = null;
|
|
3948
|
-
constructor(client, table = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
|
|
3949
|
-
super(table, schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
3950
|
-
this.client = client;
|
|
3951
|
-
}
|
|
3952
|
-
async setupDatabase() {
|
|
3953
|
-
const sql = `
|
|
3954
|
-
CREATE TABLE IF NOT EXISTS "${this.table}" (
|
|
3955
|
-
${this.constructPrimaryKeyColumns('"')} ${this.constructValueColumns('"')},
|
|
3956
|
-
PRIMARY KEY (${this.primaryKeyColumnList()})
|
|
3957
|
-
)
|
|
3958
|
-
`;
|
|
3959
|
-
const { error } = await this.client.rpc("exec_sql", { query: sql });
|
|
3960
|
-
if (error && !error.message.includes("already exists")) {
|
|
3961
|
-
throw error;
|
|
3962
|
-
}
|
|
3963
|
-
const pkColumns = this.primaryKeyColumns();
|
|
3964
|
-
const createdIndexes = new Set;
|
|
3965
|
-
for (const columns of this.indexes) {
|
|
3966
|
-
if (columns.length <= pkColumns.length) {
|
|
3967
|
-
const isPkPrefix = columns.every((col, idx) => col === pkColumns[idx]);
|
|
3968
|
-
if (isPkPrefix)
|
|
3969
|
-
continue;
|
|
3970
|
-
}
|
|
3971
|
-
const indexName = `${this.table}_${columns.join("_")}`;
|
|
3972
|
-
const columnList = columns.map((col) => `"${String(col)}"`).join(", ");
|
|
3973
|
-
const columnKey = columns.join(",");
|
|
3974
|
-
if (createdIndexes.has(columnKey))
|
|
3975
|
-
continue;
|
|
3976
|
-
const isRedundant = Array.from(createdIndexes).some((existing) => {
|
|
3977
|
-
const existingCols = existing.split(",");
|
|
3978
|
-
return existingCols.length >= columns.length && columns.every((col, idx) => col === existingCols[idx]);
|
|
3979
|
-
});
|
|
3980
|
-
if (!isRedundant) {
|
|
3981
|
-
const indexSql = `CREATE INDEX IF NOT EXISTS "${indexName}" ON "${this.table}" (${columnList})`;
|
|
3982
|
-
const { error: indexError } = await this.client.rpc("exec_sql", { query: indexSql });
|
|
3983
|
-
if (indexError && !indexError.message.includes("already exists")) {
|
|
3984
|
-
console.warn(`Failed to create index ${indexName}:`, indexError);
|
|
3985
|
-
}
|
|
3986
|
-
createdIndexes.add(columnKey);
|
|
3987
|
-
}
|
|
3988
|
-
}
|
|
3989
|
-
}
|
|
3990
|
-
mapTypeToSQL(typeDef) {
|
|
3991
|
-
const actualType = this.getNonNullType(typeDef);
|
|
3992
|
-
if (typeof actualType === "boolean") {
|
|
3993
|
-
return "TEXT /* boolean schema */";
|
|
3994
|
-
}
|
|
3995
|
-
if (actualType.contentEncoding === "blob")
|
|
3996
|
-
return "BYTEA";
|
|
3997
|
-
switch (actualType.type) {
|
|
3998
|
-
case "string":
|
|
3999
|
-
if (actualType.format === "date-time")
|
|
4000
|
-
return "TIMESTAMP";
|
|
4001
|
-
if (actualType.format === "date")
|
|
4002
|
-
return "DATE";
|
|
4003
|
-
if (actualType.format === "email")
|
|
4004
|
-
return "VARCHAR(255)";
|
|
4005
|
-
if (actualType.format === "uri")
|
|
4006
|
-
return "VARCHAR(2048)";
|
|
4007
|
-
if (actualType.format === "uuid")
|
|
4008
|
-
return "UUID";
|
|
4009
|
-
if (typeof actualType.maxLength === "number") {
|
|
4010
|
-
return `VARCHAR(${actualType.maxLength})`;
|
|
4011
|
-
}
|
|
4012
|
-
return "TEXT";
|
|
4013
|
-
case "number":
|
|
4014
|
-
case "integer":
|
|
4015
|
-
if (actualType.multipleOf === 1 || actualType.type === "integer") {
|
|
4016
|
-
if (typeof actualType.minimum === "number") {
|
|
4017
|
-
if (actualType.minimum >= 0) {
|
|
4018
|
-
if (typeof actualType.maximum === "number") {
|
|
4019
|
-
if (actualType.maximum <= 32767)
|
|
4020
|
-
return "SMALLINT";
|
|
4021
|
-
if (actualType.maximum <= 2147483647)
|
|
4022
|
-
return "INTEGER";
|
|
4023
|
-
}
|
|
4024
|
-
return "BIGINT";
|
|
4025
|
-
}
|
|
4026
|
-
}
|
|
4027
|
-
return "INTEGER";
|
|
4028
|
-
}
|
|
4029
|
-
if (actualType.format === "float")
|
|
4030
|
-
return "REAL";
|
|
4031
|
-
if (actualType.format === "double")
|
|
4032
|
-
return "DOUBLE PRECISION";
|
|
4033
|
-
if (typeof actualType.multipleOf === "number") {
|
|
4034
|
-
const decimalPlaces = String(actualType.multipleOf).split(".")[1]?.length || 0;
|
|
4035
|
-
if (decimalPlaces > 0) {
|
|
4036
|
-
return `NUMERIC(38, ${decimalPlaces})`;
|
|
4037
|
-
}
|
|
4038
|
-
}
|
|
4039
|
-
return "NUMERIC";
|
|
4040
|
-
case "boolean":
|
|
4041
|
-
return "BOOLEAN";
|
|
4042
|
-
case "array":
|
|
4043
|
-
if (actualType.items && typeof actualType.items === "object" && !Array.isArray(actualType.items)) {
|
|
4044
|
-
const itemType = this.mapTypeToSQL(actualType.items);
|
|
4045
|
-
const supportedArrayElementTypes = [
|
|
4046
|
-
"TEXT",
|
|
4047
|
-
"VARCHAR",
|
|
4048
|
-
"CHAR",
|
|
4049
|
-
"INTEGER",
|
|
4050
|
-
"SMALLINT",
|
|
4051
|
-
"BIGINT",
|
|
4052
|
-
"REAL",
|
|
4053
|
-
"DOUBLE PRECISION",
|
|
4054
|
-
"NUMERIC",
|
|
4055
|
-
"BOOLEAN",
|
|
4056
|
-
"UUID",
|
|
4057
|
-
"DATE",
|
|
4058
|
-
"TIMESTAMP"
|
|
4059
|
-
];
|
|
4060
|
-
const isSupported = supportedArrayElementTypes.some((type) => itemType === type || itemType.startsWith(type + "(") && type !== "VARCHAR");
|
|
4061
|
-
if (isSupported) {
|
|
4062
|
-
return `${itemType}[]`;
|
|
4063
|
-
} else {
|
|
4064
|
-
return "JSONB /* complex array */";
|
|
4065
|
-
}
|
|
4066
|
-
}
|
|
4067
|
-
return "JSONB /* generic array */";
|
|
4068
|
-
case "object":
|
|
4069
|
-
return "JSONB /* object */";
|
|
4070
|
-
default:
|
|
4071
|
-
return "TEXT /* unknown type */";
|
|
4072
|
-
}
|
|
4073
|
-
}
|
|
4074
|
-
constructPrimaryKeyColumns($delimiter = "") {
|
|
4075
|
-
const cols = Object.entries(this.primaryKeySchema.properties).map(([key, typeDef]) => {
|
|
4076
|
-
if (this.isAutoGeneratedKey(key)) {
|
|
4077
|
-
if (this.autoGeneratedKeyStrategy === "autoincrement") {
|
|
4078
|
-
const sqlType2 = this.mapTypeToSQL(typeDef);
|
|
4079
|
-
const isSmallInt = sqlType2.includes("SMALLINT");
|
|
4080
|
-
const isBigInt = sqlType2.includes("BIGINT");
|
|
4081
|
-
const serialType = isBigInt ? "BIGSERIAL" : isSmallInt ? "SMALLSERIAL" : "SERIAL";
|
|
4082
|
-
return `${$delimiter}${key}${$delimiter} ${serialType}`;
|
|
4083
|
-
} else if (this.autoGeneratedKeyStrategy === "uuid") {
|
|
4084
|
-
return `${$delimiter}${key}${$delimiter} UUID DEFAULT gen_random_uuid()`;
|
|
4085
|
-
}
|
|
4086
|
-
}
|
|
4087
|
-
const sqlType = this.mapTypeToSQL(typeDef);
|
|
4088
|
-
let constraints = "NOT NULL";
|
|
4089
|
-
if (this.shouldBeUnsigned(typeDef)) {
|
|
4090
|
-
constraints += ` CHECK (${$delimiter}${key}${$delimiter} >= 0)`;
|
|
4091
|
-
}
|
|
4092
|
-
return `${$delimiter}${key}${$delimiter} ${sqlType} ${constraints}`;
|
|
4093
|
-
}).join(", ");
|
|
4094
|
-
return cols;
|
|
4095
|
-
}
|
|
4096
|
-
constructValueColumns($delimiter = "") {
|
|
4097
|
-
const delimiter = $delimiter || '"';
|
|
4098
|
-
const requiredSet = new Set(this.valueSchema.required ?? []);
|
|
4099
|
-
const cols = Object.entries(this.valueSchema.properties).map(([key, typeDef]) => {
|
|
4100
|
-
const sqlType = this.mapTypeToSQL(typeDef);
|
|
4101
|
-
const isRequired = requiredSet.has(key);
|
|
4102
|
-
const nullable = !isRequired || this.isNullable(typeDef);
|
|
4103
|
-
let constraints = nullable ? "NULL" : "NOT NULL";
|
|
4104
|
-
if (this.shouldBeUnsigned(typeDef)) {
|
|
4105
|
-
constraints += ` CHECK (${delimiter}${key}${delimiter} >= 0)`;
|
|
4106
|
-
}
|
|
4107
|
-
return `${delimiter}${key}${delimiter} ${sqlType} ${constraints}`;
|
|
4108
|
-
}).join(", ");
|
|
4109
|
-
if (cols.length > 0) {
|
|
4110
|
-
return `, ${cols}`;
|
|
4111
|
-
} else {
|
|
4112
|
-
return "";
|
|
4113
|
-
}
|
|
4114
|
-
}
|
|
4115
|
-
sqlToJsValue(column, value) {
|
|
4116
|
-
const typeDef = this.schema.properties[column];
|
|
4117
|
-
if (typeDef) {
|
|
4118
|
-
if (value === null && this.isNullable(typeDef)) {
|
|
4119
|
-
return null;
|
|
4120
|
-
}
|
|
4121
|
-
const actualType = this.getNonNullType(typeDef);
|
|
4122
|
-
if (typeof actualType !== "boolean" && (actualType.type === "number" || actualType.type === "integer")) {
|
|
4123
|
-
if (typeof value === "number")
|
|
4124
|
-
return value;
|
|
4125
|
-
if (typeof value === "string") {
|
|
4126
|
-
const parsed = Number(value);
|
|
4127
|
-
if (!isNaN(parsed))
|
|
4128
|
-
return parsed;
|
|
4129
|
-
}
|
|
4130
|
-
}
|
|
4131
|
-
}
|
|
4132
|
-
return super.sqlToJsValue(column, value);
|
|
4133
|
-
}
|
|
4134
|
-
shouldBeUnsigned(typeDef) {
|
|
4135
|
-
const actualType = this.getNonNullType(typeDef);
|
|
4136
|
-
if (typeof actualType === "boolean") {
|
|
4137
|
-
return false;
|
|
4138
|
-
}
|
|
4139
|
-
if ((actualType.type === "number" || actualType.type === "integer") && typeof actualType.minimum === "number" && actualType.minimum >= 0) {
|
|
4140
|
-
return true;
|
|
4141
|
-
}
|
|
4142
|
-
return false;
|
|
4143
|
-
}
|
|
4144
|
-
async put(entity) {
|
|
4145
|
-
let entityToInsert = { ...entity };
|
|
4146
|
-
if (this.hasAutoGeneratedKey() && this.autoGeneratedKeyName) {
|
|
4147
|
-
const keyName = String(this.autoGeneratedKeyName);
|
|
4148
|
-
const entityRecord = entity;
|
|
4149
|
-
const clientProvidedValue = entityRecord[keyName];
|
|
4150
|
-
const hasClientValue = clientProvidedValue !== undefined && clientProvidedValue !== null;
|
|
4151
|
-
let shouldOmitKey = false;
|
|
4152
|
-
if (this.clientProvidedKeys === "never") {
|
|
4153
|
-
shouldOmitKey = true;
|
|
4154
|
-
} else if (this.clientProvidedKeys === "always") {
|
|
4155
|
-
if (!hasClientValue) {
|
|
4156
|
-
throw new Error(`Auto-generated key "${keyName}" is required when clientProvidedKeys is "always"`);
|
|
4157
|
-
}
|
|
4158
|
-
shouldOmitKey = false;
|
|
4159
|
-
} else {
|
|
4160
|
-
shouldOmitKey = !hasClientValue;
|
|
4161
|
-
}
|
|
4162
|
-
if (shouldOmitKey) {
|
|
4163
|
-
delete entityToInsert[keyName];
|
|
4164
|
-
}
|
|
4165
|
-
}
|
|
4166
|
-
const normalizedEntity = { ...entityToInsert };
|
|
4167
|
-
const requiredSet = new Set(this.valueSchema.required ?? []);
|
|
4168
|
-
for (const key in this.valueSchema.properties) {
|
|
4169
|
-
if (!(key in normalizedEntity) || normalizedEntity[key] === undefined) {
|
|
4170
|
-
if (!requiredSet.has(key)) {
|
|
4171
|
-
normalizedEntity[key] = null;
|
|
4172
|
-
}
|
|
4173
|
-
}
|
|
4174
|
-
}
|
|
4175
|
-
const { data, error } = await this.client.from(this.table).upsert(normalizedEntity, { onConflict: this.primaryKeyColumnList() }).select().single();
|
|
4176
|
-
if (error)
|
|
4177
|
-
throw error;
|
|
4178
|
-
const updatedEntity = data;
|
|
4179
|
-
const updatedRecord = updatedEntity;
|
|
4180
|
-
for (const key in this.schema.properties) {
|
|
4181
|
-
updatedRecord[key] = this.sqlToJsValue(key, updatedRecord[key]);
|
|
4182
|
-
}
|
|
4183
|
-
this.events.emit("put", updatedEntity);
|
|
4184
|
-
return updatedEntity;
|
|
4185
|
-
}
|
|
4186
|
-
async putBulk(entities) {
|
|
4187
|
-
if (entities.length === 0)
|
|
4188
|
-
return [];
|
|
4189
|
-
return await Promise.all(entities.map((entity) => this.put(entity)));
|
|
4190
|
-
}
|
|
4191
|
-
async get(key) {
|
|
4192
|
-
let query = this.client.from(this.table).select("*");
|
|
4193
|
-
const keyRecord = key;
|
|
4194
|
-
for (const pkName of this.primaryKeyNames) {
|
|
4195
|
-
query = query.eq(String(pkName), keyRecord[String(pkName)]);
|
|
4196
|
-
}
|
|
4197
|
-
const { data, error } = await query.single();
|
|
4198
|
-
if (error) {
|
|
4199
|
-
if (error.code === "PGRST116") {
|
|
4200
|
-
this.events.emit("get", key, undefined);
|
|
4201
|
-
return;
|
|
4202
|
-
}
|
|
4203
|
-
throw error;
|
|
4204
|
-
}
|
|
4205
|
-
const val = data;
|
|
4206
|
-
if (val) {
|
|
4207
|
-
const valRecord = val;
|
|
4208
|
-
for (const key2 in this.schema.properties) {
|
|
4209
|
-
valRecord[key2] = this.sqlToJsValue(key2, valRecord[key2]);
|
|
4210
|
-
}
|
|
4211
|
-
}
|
|
4212
|
-
this.events.emit("get", key, val);
|
|
4213
|
-
return val;
|
|
4214
|
-
}
|
|
4215
|
-
async delete(value) {
|
|
4216
|
-
const { key } = this.separateKeyValueFromCombined(value);
|
|
4217
|
-
let query = this.client.from(this.table).delete();
|
|
4218
|
-
const deleteKeyRecord = key;
|
|
4219
|
-
for (const pkName of this.primaryKeyNames) {
|
|
4220
|
-
query = query.eq(String(pkName), deleteKeyRecord[String(pkName)]);
|
|
4221
|
-
}
|
|
4222
|
-
const { error } = await query;
|
|
4223
|
-
if (error)
|
|
4224
|
-
throw error;
|
|
4225
|
-
this.events.emit("delete", key);
|
|
4226
|
-
}
|
|
4227
|
-
async getAll(options) {
|
|
4228
|
-
this.validateGetAllOptions(options);
|
|
4229
|
-
let query = this.client.from(this.table).select("*");
|
|
4230
|
-
if (options?.orderBy) {
|
|
4231
|
-
for (const { column, direction } of options.orderBy) {
|
|
4232
|
-
query = query.order(String(column), { ascending: direction === "ASC" });
|
|
4233
|
-
}
|
|
4234
|
-
}
|
|
4235
|
-
if (options?.offset !== undefined || options?.limit !== undefined) {
|
|
4236
|
-
const start = options?.offset ?? 0;
|
|
4237
|
-
if (options?.limit !== undefined) {
|
|
4238
|
-
query = query.range(start, start + options.limit - 1);
|
|
4239
|
-
} else if (options?.offset !== undefined) {
|
|
4240
|
-
query = query.range(start, start + 999999);
|
|
4241
|
-
}
|
|
4242
|
-
}
|
|
4243
|
-
const { data, error } = await query;
|
|
4244
|
-
if (error)
|
|
4245
|
-
throw error;
|
|
4246
|
-
if (data && data.length) {
|
|
4247
|
-
for (const row of data) {
|
|
4248
|
-
const record = row;
|
|
4249
|
-
for (const key in this.schema.properties) {
|
|
4250
|
-
record[key] = this.sqlToJsValue(key, record[key]);
|
|
4251
|
-
}
|
|
4252
|
-
}
|
|
4253
|
-
return data;
|
|
4254
|
-
}
|
|
4255
|
-
return;
|
|
4256
|
-
}
|
|
4257
|
-
async deleteAll() {
|
|
4258
|
-
const firstPkColumn = this.primaryKeyNames[0];
|
|
4259
|
-
const { error } = await this.client.from(this.table).delete().neq(String(firstPkColumn), null);
|
|
4260
|
-
if (error)
|
|
4261
|
-
throw error;
|
|
4262
|
-
this.events.emit("clearall");
|
|
4263
|
-
}
|
|
4264
|
-
async size() {
|
|
4265
|
-
const { count, error } = await this.client.from(this.table).select("*", { count: "exact", head: true });
|
|
4266
|
-
if (error)
|
|
4267
|
-
throw error;
|
|
4268
|
-
return count ?? 0;
|
|
4269
|
-
}
|
|
4270
|
-
async getBulk(offset, limit) {
|
|
4271
|
-
let query = this.client.from(this.table).select("*");
|
|
4272
|
-
for (const pkName of this.primaryKeyNames) {
|
|
4273
|
-
query = query.order(String(pkName));
|
|
4274
|
-
}
|
|
4275
|
-
const { data, error } = await query.range(offset, offset + limit - 1);
|
|
4276
|
-
if (error)
|
|
4277
|
-
throw error;
|
|
4278
|
-
if (!data || data.length === 0) {
|
|
4279
|
-
return;
|
|
4280
|
-
}
|
|
4281
|
-
for (const row of data) {
|
|
4282
|
-
const record = row;
|
|
4283
|
-
for (const key in this.schema.properties) {
|
|
4284
|
-
record[key] = this.sqlToJsValue(key, record[key]);
|
|
4285
|
-
}
|
|
4286
|
-
}
|
|
4287
|
-
return data;
|
|
4288
|
-
}
|
|
4289
|
-
applyCriteriaToFilter(query, criteria) {
|
|
4290
|
-
let q = query;
|
|
4291
|
-
for (const column of Object.keys(criteria)) {
|
|
4292
|
-
const criterion = criteria[column];
|
|
4293
|
-
let operator = "=";
|
|
4294
|
-
let value;
|
|
4295
|
-
if (isSearchCondition(criterion)) {
|
|
4296
|
-
operator = criterion.operator;
|
|
4297
|
-
value = criterion.value;
|
|
4298
|
-
} else {
|
|
4299
|
-
value = criterion;
|
|
4300
|
-
}
|
|
4301
|
-
switch (operator) {
|
|
4302
|
-
case "=":
|
|
4303
|
-
q = q.eq(String(column), value);
|
|
4304
|
-
break;
|
|
4305
|
-
case "<":
|
|
4306
|
-
q = q.lt(String(column), value);
|
|
4307
|
-
break;
|
|
4308
|
-
case "<=":
|
|
4309
|
-
q = q.lte(String(column), value);
|
|
4310
|
-
break;
|
|
4311
|
-
case ">":
|
|
4312
|
-
q = q.gt(String(column), value);
|
|
4313
|
-
break;
|
|
4314
|
-
case ">=":
|
|
4315
|
-
q = q.gte(String(column), value);
|
|
4316
|
-
break;
|
|
4317
|
-
}
|
|
4318
|
-
}
|
|
4319
|
-
return q;
|
|
4320
|
-
}
|
|
4321
|
-
async count(criteria) {
|
|
4322
|
-
if (!criteria || Object.keys(criteria).length === 0) {
|
|
4323
|
-
return await this.size();
|
|
4324
|
-
}
|
|
4325
|
-
this.validateQueryParams(criteria);
|
|
4326
|
-
const query = this.applyCriteriaToFilter(this.client.from(this.table).select("*", { count: "exact", head: true }), criteria);
|
|
4327
|
-
const { count, error } = await query;
|
|
4328
|
-
if (error)
|
|
4329
|
-
throw error;
|
|
4330
|
-
return count ?? 0;
|
|
4331
|
-
}
|
|
4332
|
-
async deleteSearch(criteria) {
|
|
4333
|
-
const criteriaKeys = Object.keys(criteria);
|
|
4334
|
-
if (criteriaKeys.length === 0) {
|
|
4335
|
-
return;
|
|
4336
|
-
}
|
|
4337
|
-
let query = this.client.from(this.table).delete();
|
|
4338
|
-
for (const column of criteriaKeys) {
|
|
4339
|
-
if (!(column in this.schema.properties)) {
|
|
4340
|
-
throw new Error(`Schema must have a ${String(column)} field to use deleteSearch`);
|
|
4341
|
-
}
|
|
4342
|
-
const criterion = criteria[column];
|
|
4343
|
-
let operator = "=";
|
|
4344
|
-
let value;
|
|
4345
|
-
if (isSearchCondition(criterion)) {
|
|
4346
|
-
operator = criterion.operator;
|
|
4347
|
-
value = criterion.value;
|
|
4348
|
-
} else {
|
|
4349
|
-
value = criterion;
|
|
4350
|
-
}
|
|
4351
|
-
switch (operator) {
|
|
4352
|
-
case "=":
|
|
4353
|
-
query = query.eq(String(column), value);
|
|
4354
|
-
break;
|
|
4355
|
-
case "<":
|
|
4356
|
-
query = query.lt(String(column), value);
|
|
4357
|
-
break;
|
|
4358
|
-
case "<=":
|
|
4359
|
-
query = query.lte(String(column), value);
|
|
4360
|
-
break;
|
|
4361
|
-
case ">":
|
|
4362
|
-
query = query.gt(String(column), value);
|
|
4363
|
-
break;
|
|
4364
|
-
case ">=":
|
|
4365
|
-
query = query.gte(String(column), value);
|
|
4366
|
-
break;
|
|
4367
|
-
}
|
|
4368
|
-
}
|
|
4369
|
-
const { error } = await query;
|
|
4370
|
-
if (error)
|
|
4371
|
-
throw error;
|
|
4372
|
-
this.events.emit("delete", criteriaKeys[0]);
|
|
4373
|
-
}
|
|
4374
|
-
async query(criteria, options) {
|
|
4375
|
-
this.validateQueryParams(criteria, options);
|
|
4376
|
-
let query = this.applyCriteriaToFilter(this.client.from(this.table).select("*"), criteria);
|
|
4377
|
-
if (options?.orderBy) {
|
|
4378
|
-
for (const { column, direction } of options.orderBy) {
|
|
4379
|
-
query = query.order(String(column), { ascending: direction === "ASC" });
|
|
4380
|
-
}
|
|
4381
|
-
}
|
|
4382
|
-
if (options?.offset !== undefined || options?.limit !== undefined) {
|
|
4383
|
-
const start = options?.offset ?? 0;
|
|
4384
|
-
if (options?.limit !== undefined) {
|
|
4385
|
-
query = query.range(start, start + options.limit - 1);
|
|
4386
|
-
} else if (options?.offset !== undefined) {
|
|
4387
|
-
query = query.range(start, start + 999999);
|
|
4388
|
-
}
|
|
4389
|
-
} else if (options?.limit !== undefined) {
|
|
4390
|
-
query = query.limit(options.limit);
|
|
4391
|
-
}
|
|
4392
|
-
const { data, error } = await query;
|
|
4393
|
-
if (error)
|
|
4394
|
-
throw error;
|
|
4395
|
-
if (data && data.length > 0) {
|
|
4396
|
-
for (const row of data) {
|
|
4397
|
-
const record = row;
|
|
4398
|
-
for (const key in this.schema.properties) {
|
|
4399
|
-
record[key] = this.sqlToJsValue(key, record[key]);
|
|
4400
|
-
}
|
|
4401
|
-
}
|
|
4402
|
-
this.events.emit("query", criteria, data);
|
|
4403
|
-
return data;
|
|
4404
|
-
}
|
|
4405
|
-
this.events.emit("query", criteria, undefined);
|
|
4406
|
-
return;
|
|
4407
|
-
}
|
|
4408
|
-
async queryIndex(criteria, options) {
|
|
4409
|
-
this.validateSelect(options);
|
|
4410
|
-
this.validateQueryParams(criteria, options);
|
|
4411
|
-
const registered = this.indexes.map((cols, i) => {
|
|
4412
|
-
const cs = Array.isArray(cols) ? cols : [cols];
|
|
4413
|
-
return { name: `idx_${i}`, keyPath: cs };
|
|
4414
|
-
});
|
|
4415
|
-
pickCoveringIndex({
|
|
4416
|
-
table: this.table,
|
|
4417
|
-
indexes: registered,
|
|
4418
|
-
criteriaColumns: Object.keys(criteria),
|
|
4419
|
-
orderByColumns: (options.orderBy ?? []).map((o) => ({
|
|
4420
|
-
column: String(o.column),
|
|
4421
|
-
direction: o.direction
|
|
4422
|
-
})),
|
|
4423
|
-
selectColumns: options.select.map(String),
|
|
4424
|
-
primaryKeyColumns: this.primaryKeyNames.map(String)
|
|
4425
|
-
});
|
|
4426
|
-
const colList = options.select.map(String).join(",");
|
|
4427
|
-
let q = this.applyCriteriaToFilter(this.client.from(this.table).select(colList), criteria);
|
|
4428
|
-
if (options.orderBy) {
|
|
4429
|
-
for (const { column, direction } of options.orderBy) {
|
|
4430
|
-
q = q.order(String(column), { ascending: direction === "ASC" });
|
|
4431
|
-
}
|
|
4432
|
-
}
|
|
4433
|
-
if (options.offset !== undefined && options.limit === undefined) {
|
|
4434
|
-
throw new StorageValidationError("queryIndex with offset requires limit (no implicit cap)");
|
|
4435
|
-
}
|
|
4436
|
-
if (options.offset !== undefined || options.limit !== undefined) {
|
|
4437
|
-
const start = options.offset ?? 0;
|
|
4438
|
-
if (options.limit !== undefined) {
|
|
4439
|
-
q = q.range(start, start + options.limit - 1);
|
|
4440
|
-
}
|
|
4441
|
-
}
|
|
4442
|
-
const { data, error } = await q;
|
|
4443
|
-
if (error)
|
|
4444
|
-
throw error;
|
|
4445
|
-
if (!data)
|
|
4446
|
-
return [];
|
|
4447
|
-
const rows = data;
|
|
4448
|
-
const sel = new Set(options.select.map(String));
|
|
4449
|
-
for (const row of rows) {
|
|
4450
|
-
for (const key of Object.keys(row)) {
|
|
4451
|
-
if (sel.has(key)) {
|
|
4452
|
-
row[key] = this.sqlToJsValue(key, row[key]);
|
|
4453
|
-
}
|
|
4454
|
-
}
|
|
4455
|
-
}
|
|
4456
|
-
return rows;
|
|
4457
|
-
}
|
|
4458
|
-
convertRealtimeRow(row) {
|
|
4459
|
-
const entity = { ...row };
|
|
4460
|
-
const record = entity;
|
|
4461
|
-
for (const key in this.schema.properties) {
|
|
4462
|
-
record[key] = this.sqlToJsValue(key, row[key]);
|
|
4463
|
-
}
|
|
4464
|
-
return entity;
|
|
4465
|
-
}
|
|
4466
|
-
subscribeToChanges(callback, options) {
|
|
4467
|
-
const channelName = `tabular-${this.table}-${Date.now()}`;
|
|
4468
|
-
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
4469
|
-
event: "*",
|
|
4470
|
-
schema: "public",
|
|
4471
|
-
table: this.table
|
|
4472
|
-
}, (payload) => {
|
|
4473
|
-
const change = {
|
|
4474
|
-
type: payload.eventType.toUpperCase(),
|
|
4475
|
-
old: payload.old && Object.keys(payload.old).length > 0 ? this.convertRealtimeRow(payload.old) : undefined,
|
|
4476
|
-
new: payload.new && Object.keys(payload.new).length > 0 ? this.convertRealtimeRow(payload.new) : undefined
|
|
4477
|
-
};
|
|
4478
|
-
callback(change);
|
|
4479
|
-
}).subscribe();
|
|
4480
|
-
return () => {
|
|
4481
|
-
if (this.realtimeChannel) {
|
|
4482
|
-
this.client.removeChannel(this.realtimeChannel);
|
|
4483
|
-
this.realtimeChannel = null;
|
|
4484
|
-
}
|
|
4485
|
-
};
|
|
4486
|
-
}
|
|
4487
|
-
destroy() {
|
|
4488
|
-
if (this.realtimeChannel) {
|
|
4489
|
-
this.client.removeChannel(this.realtimeChannel);
|
|
4490
|
-
this.realtimeChannel = null;
|
|
4491
|
-
}
|
|
4492
|
-
}
|
|
4493
|
-
}
|
|
4494
|
-
// src/kv/IndexedDbKvStorage.ts
|
|
4495
|
-
import { createServiceToken as createServiceToken15 } from "@workglow/util";
|
|
4496
|
-
var IDB_KV_REPOSITORY = createServiceToken15("storage.kvRepository.indexedDb");
|
|
4497
|
-
|
|
4498
|
-
class IndexedDbKvStorage extends KvViaTabularStorage {
|
|
4499
|
-
dbName;
|
|
4500
|
-
tabularRepository;
|
|
4501
|
-
constructor(dbName, keySchema = { type: "string" }, valueSchema = {}) {
|
|
4502
|
-
super(keySchema, valueSchema);
|
|
4503
|
-
this.dbName = dbName;
|
|
4504
|
-
this.tabularRepository = new IndexedDbTabularStorage(dbName, DefaultKeyValueSchema, DefaultKeyValueKey);
|
|
4505
|
-
}
|
|
4506
|
-
}
|
|
4507
|
-
// src/kv/SupabaseKvStorage.ts
|
|
4508
|
-
import { createServiceToken as createServiceToken16 } from "@workglow/util";
|
|
4509
|
-
var SUPABASE_KV_REPOSITORY = createServiceToken16("storage.kvRepository.supabase");
|
|
4510
|
-
|
|
4511
|
-
class SupabaseKvStorage extends KvViaTabularStorage {
|
|
4512
|
-
client;
|
|
4513
|
-
tableName;
|
|
4514
|
-
tabularRepository;
|
|
4515
|
-
constructor(client, tableName, keySchema = { type: "string" }, valueSchema = {}, tabularRepository) {
|
|
4516
|
-
super(keySchema, valueSchema);
|
|
4517
|
-
this.client = client;
|
|
4518
|
-
this.tableName = tableName;
|
|
4519
|
-
this.tabularRepository = tabularRepository ?? new SupabaseTabularStorage(client, tableName, DefaultKeyValueSchema, DefaultKeyValueKey);
|
|
4520
|
-
}
|
|
4521
|
-
}
|
|
4522
|
-
// src/queue/IndexedDbQueueStorage.ts
|
|
4523
|
-
import { createServiceToken as createServiceToken17, deepEqual as deepEqual3, makeFingerprint as makeFingerprint6, uuid4 as uuid45 } from "@workglow/util";
|
|
4524
|
-
var INDEXED_DB_QUEUE_STORAGE = createServiceToken17("jobqueue.storage.indexedDb");
|
|
4525
|
-
|
|
4526
|
-
class IndexedDbQueueStorage {
|
|
4527
|
-
queueName;
|
|
4528
|
-
scope = "process";
|
|
4529
|
-
db;
|
|
4530
|
-
tableName;
|
|
4531
|
-
migrationOptions;
|
|
4532
|
-
prefixes;
|
|
4533
|
-
prefixValues;
|
|
4534
|
-
hybridManager = null;
|
|
4535
|
-
hybridOptions;
|
|
4536
|
-
constructor(queueName, options = {}) {
|
|
4537
|
-
this.queueName = queueName;
|
|
4538
|
-
this.migrationOptions = options;
|
|
4539
|
-
this.prefixes = options.prefixes ?? [];
|
|
4540
|
-
this.prefixValues = options.prefixValues ?? {};
|
|
4541
|
-
this.hybridOptions = {
|
|
4542
|
-
useBroadcastChannel: options.useBroadcastChannel ?? true,
|
|
4543
|
-
backupPollingIntervalMs: options.backupPollingIntervalMs ?? 5000
|
|
4544
|
-
};
|
|
4545
|
-
if (this.prefixes.length > 0) {
|
|
4546
|
-
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
4547
|
-
this.tableName = `jobs_${prefixNames}`;
|
|
4548
|
-
} else {
|
|
4549
|
-
this.tableName = "jobs";
|
|
4550
|
-
}
|
|
4551
|
-
}
|
|
4552
|
-
getPrefixColumnNames() {
|
|
4553
|
-
return this.prefixes.map((p) => p.name);
|
|
4554
|
-
}
|
|
4555
|
-
matchesPrefixes(job) {
|
|
4556
|
-
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
4557
|
-
if (job[key] !== value) {
|
|
4558
|
-
return false;
|
|
4559
|
-
}
|
|
4560
|
-
}
|
|
4561
|
-
return true;
|
|
4562
|
-
}
|
|
4563
|
-
getPrefixKeyValues() {
|
|
4564
|
-
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
4565
|
-
}
|
|
4566
|
-
async getDb() {
|
|
4567
|
-
if (this.db)
|
|
4568
|
-
return this.db;
|
|
4569
|
-
await this.setupDatabase();
|
|
4570
|
-
return this.db;
|
|
4571
|
-
}
|
|
4572
|
-
async setupDatabase() {
|
|
4573
|
-
const prefixColumnNames = this.getPrefixColumnNames();
|
|
4574
|
-
const buildKeyPath = (basePath) => {
|
|
4575
|
-
return [...prefixColumnNames, ...basePath];
|
|
4576
|
-
};
|
|
4577
|
-
const expectedIndexes = [
|
|
4578
|
-
{
|
|
4579
|
-
name: "queue_status",
|
|
4580
|
-
keyPath: buildKeyPath(["queue", "status"]),
|
|
4581
|
-
options: { unique: false }
|
|
4582
|
-
},
|
|
4583
|
-
{
|
|
4584
|
-
name: "queue_status_run_after",
|
|
4585
|
-
keyPath: buildKeyPath(["queue", "status", "run_after"]),
|
|
4586
|
-
options: { unique: false }
|
|
4587
|
-
},
|
|
4588
|
-
{
|
|
4589
|
-
name: "queue_job_run_id",
|
|
4590
|
-
keyPath: buildKeyPath(["queue", "job_run_id"]),
|
|
4591
|
-
options: { unique: false }
|
|
4592
|
-
},
|
|
4593
|
-
{
|
|
4594
|
-
name: "queue_fingerprint_status",
|
|
4595
|
-
keyPath: buildKeyPath(["queue", "fingerprint", "status"]),
|
|
4596
|
-
options: { unique: false }
|
|
4597
|
-
}
|
|
4598
|
-
];
|
|
4599
|
-
this.db = await ensureIndexedDbTable(this.tableName, "id", expectedIndexes, this.migrationOptions);
|
|
4600
|
-
}
|
|
4601
|
-
async add(job) {
|
|
4602
|
-
const db = await this.getDb();
|
|
4603
|
-
const now = new Date().toISOString();
|
|
4604
|
-
const jobWithPrefixes = job;
|
|
4605
|
-
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid45();
|
|
4606
|
-
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid45();
|
|
4607
|
-
jobWithPrefixes.queue = this.queueName;
|
|
4608
|
-
jobWithPrefixes.fingerprint = await makeFingerprint6(jobWithPrefixes.input);
|
|
4609
|
-
jobWithPrefixes.status = JobStatus.PENDING;
|
|
4610
|
-
jobWithPrefixes.progress = 0;
|
|
4611
|
-
jobWithPrefixes.progress_message = "";
|
|
4612
|
-
jobWithPrefixes.progress_details = null;
|
|
4613
|
-
jobWithPrefixes.created_at = now;
|
|
4614
|
-
jobWithPrefixes.run_after = now;
|
|
4615
|
-
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
4616
|
-
jobWithPrefixes[key] = value;
|
|
4617
|
-
}
|
|
4618
|
-
const tx = db.transaction(this.tableName, "readwrite");
|
|
4619
|
-
const store = tx.objectStore(this.tableName);
|
|
4620
|
-
return new Promise((resolve, reject) => {
|
|
4621
|
-
const request = store.add(jobWithPrefixes);
|
|
4622
|
-
tx.oncomplete = () => {
|
|
4623
|
-
this.hybridManager?.notifyLocalChange();
|
|
4624
|
-
resolve(jobWithPrefixes.id);
|
|
4625
|
-
};
|
|
4626
|
-
tx.onerror = () => reject(tx.error);
|
|
4627
|
-
request.onerror = () => reject(request.error);
|
|
4628
|
-
});
|
|
4629
|
-
}
|
|
4630
|
-
async get(id) {
|
|
4631
|
-
const db = await this.getDb();
|
|
4632
|
-
const tx = db.transaction(this.tableName, "readonly");
|
|
4633
|
-
const store = tx.objectStore(this.tableName);
|
|
4634
|
-
const request = store.get(id);
|
|
4635
|
-
return new Promise((resolve, reject) => {
|
|
4636
|
-
request.onsuccess = () => {
|
|
4637
|
-
const job = request.result;
|
|
4638
|
-
if (job && job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
4639
|
-
resolve(job);
|
|
4640
|
-
} else {
|
|
4641
|
-
resolve(undefined);
|
|
4642
|
-
}
|
|
4643
|
-
};
|
|
4644
|
-
request.onerror = () => reject(request.error);
|
|
4645
|
-
tx.onerror = () => reject(tx.error);
|
|
4646
|
-
});
|
|
4647
|
-
}
|
|
4648
|
-
async peek(status = JobStatus.PENDING, num = 100) {
|
|
4649
|
-
const db = await this.getDb();
|
|
4650
|
-
const tx = db.transaction(this.tableName, "readonly");
|
|
4651
|
-
const store = tx.objectStore(this.tableName);
|
|
4652
|
-
const index = store.index("queue_status_run_after");
|
|
4653
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
4654
|
-
return new Promise((resolve, reject) => {
|
|
4655
|
-
const ret = new Map;
|
|
4656
|
-
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, status, ""], [...prefixKeyValues, this.queueName, status, ""]);
|
|
4657
|
-
const cursorRequest = index.openCursor(keyRange);
|
|
4658
|
-
const handleCursor = (e) => {
|
|
4659
|
-
const cursor = e.target.result;
|
|
4660
|
-
if (!cursor || ret.size >= num) {
|
|
4661
|
-
resolve(Array.from(ret.values()));
|
|
4662
|
-
return;
|
|
4663
|
-
}
|
|
4664
|
-
const job = cursor.value;
|
|
4665
|
-
if (this.matchesPrefixes(job)) {
|
|
4666
|
-
ret.set(cursor.value.id, cursor.value);
|
|
4667
|
-
}
|
|
4668
|
-
cursor.continue();
|
|
4669
|
-
};
|
|
4670
|
-
cursorRequest.onsuccess = handleCursor;
|
|
4671
|
-
cursorRequest.onerror = () => reject(cursorRequest.error);
|
|
4672
|
-
tx.onerror = () => reject(tx.error);
|
|
4673
|
-
});
|
|
4674
|
-
}
|
|
4675
|
-
async next(workerId) {
|
|
4676
|
-
const db = await this.getDb();
|
|
4677
|
-
const tx = db.transaction(this.tableName, "readwrite");
|
|
4678
|
-
const store = tx.objectStore(this.tableName);
|
|
4679
|
-
const index = store.index("queue_status_run_after");
|
|
4680
|
-
const now = new Date().toISOString();
|
|
4681
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
4682
|
-
const claimToken = workerId;
|
|
4683
|
-
const jobToReturn = await new Promise((resolve, reject) => {
|
|
4684
|
-
const cursorRequest = index.openCursor(IDBKeyRange.bound([...prefixKeyValues, this.queueName, JobStatus.PENDING, ""], [...prefixKeyValues, this.queueName, JobStatus.PENDING, now], false, false));
|
|
4685
|
-
let claimedJob;
|
|
4686
|
-
let cursorStopped = false;
|
|
4687
|
-
cursorRequest.onsuccess = (e) => {
|
|
4688
|
-
const cursor = e.target.result;
|
|
4689
|
-
if (!cursor) {
|
|
4690
|
-
return;
|
|
4691
|
-
}
|
|
4692
|
-
if (cursorStopped) {
|
|
4693
|
-
return;
|
|
4694
|
-
}
|
|
4695
|
-
const job = cursor.value;
|
|
4696
|
-
if (job.queue !== this.queueName || job.status !== JobStatus.PENDING || !this.matchesPrefixes(job)) {
|
|
4697
|
-
cursor.continue();
|
|
4698
|
-
return;
|
|
4699
|
-
}
|
|
4700
|
-
job.status = JobStatus.PROCESSING;
|
|
4701
|
-
job.last_ran_at = now;
|
|
4702
|
-
job.worker_id = claimToken;
|
|
4703
|
-
try {
|
|
4704
|
-
const updateRequest = store.put(job);
|
|
4705
|
-
updateRequest.onsuccess = () => {
|
|
4706
|
-
claimedJob = job;
|
|
4707
|
-
cursorStopped = true;
|
|
4708
|
-
};
|
|
4709
|
-
updateRequest.onerror = (err) => {
|
|
4710
|
-
console.error("Failed to update job status:", err);
|
|
4711
|
-
cursor.continue();
|
|
4712
|
-
};
|
|
4713
|
-
} catch (err) {
|
|
4714
|
-
console.error("Error updating job:", err);
|
|
4715
|
-
cursor.continue();
|
|
4716
|
-
}
|
|
4717
|
-
};
|
|
4718
|
-
cursorRequest.onerror = () => reject(cursorRequest.error);
|
|
4719
|
-
tx.oncomplete = () => {
|
|
4720
|
-
if (claimedJob) {
|
|
4721
|
-
this.hybridManager?.notifyLocalChange();
|
|
4722
|
-
}
|
|
4723
|
-
resolve(claimedJob);
|
|
4724
|
-
};
|
|
4725
|
-
tx.onerror = () => reject(tx.error);
|
|
4726
|
-
});
|
|
4727
|
-
if (!jobToReturn) {
|
|
4728
|
-
return;
|
|
4729
|
-
}
|
|
4730
|
-
const verifiedJob = await this.get(jobToReturn.id);
|
|
4731
|
-
if (!verifiedJob) {
|
|
4732
|
-
return;
|
|
4733
|
-
}
|
|
4734
|
-
if (verifiedJob.worker_id !== claimToken) {
|
|
4735
|
-
return;
|
|
4736
|
-
}
|
|
4737
|
-
if (verifiedJob.status !== JobStatus.PROCESSING) {
|
|
4738
|
-
return;
|
|
4739
|
-
}
|
|
4740
|
-
return verifiedJob;
|
|
4741
|
-
}
|
|
4742
|
-
async size(status = JobStatus.PENDING) {
|
|
4743
|
-
const db = await this.getDb();
|
|
4744
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
4745
|
-
return new Promise((resolve, reject) => {
|
|
4746
|
-
const tx = db.transaction(this.tableName, "readonly");
|
|
4747
|
-
const store = tx.objectStore(this.tableName);
|
|
4748
|
-
const index = store.index("queue_status");
|
|
4749
|
-
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
4750
|
-
const request = index.count(keyRange);
|
|
4751
|
-
request.onsuccess = () => resolve(request.result);
|
|
4752
|
-
request.onerror = () => reject(request.error);
|
|
4753
|
-
tx.onerror = () => reject(tx.error);
|
|
4754
|
-
});
|
|
4755
|
-
}
|
|
4756
|
-
async complete(job) {
|
|
4757
|
-
const db = await this.getDb();
|
|
4758
|
-
const tx = db.transaction(this.tableName, "readwrite");
|
|
4759
|
-
const store = tx.objectStore(this.tableName);
|
|
4760
|
-
return new Promise((resolve, reject) => {
|
|
4761
|
-
const getReq = store.get(job.id);
|
|
4762
|
-
getReq.onsuccess = () => {
|
|
4763
|
-
const existing = getReq.result;
|
|
4764
|
-
if (!existing || existing.queue !== this.queueName || !this.matchesPrefixes(existing)) {
|
|
4765
|
-
reject(new Error(`Job ${job.id} not found or does not belong to queue ${this.queueName}`));
|
|
4766
|
-
return;
|
|
4767
|
-
}
|
|
4768
|
-
const currentAttempts = existing.run_attempts ?? 0;
|
|
4769
|
-
job.run_attempts = currentAttempts + 1;
|
|
4770
|
-
job.queue = this.queueName;
|
|
4771
|
-
const jobWithPrefixes = job;
|
|
4772
|
-
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
4773
|
-
jobWithPrefixes[key] = value;
|
|
4774
|
-
}
|
|
4775
|
-
const putReq = store.put(jobWithPrefixes);
|
|
4776
|
-
putReq.onsuccess = () => {};
|
|
4777
|
-
putReq.onerror = () => reject(putReq.error);
|
|
4778
|
-
};
|
|
4779
|
-
getReq.onerror = () => reject(getReq.error);
|
|
4780
|
-
tx.oncomplete = () => {
|
|
4781
|
-
this.hybridManager?.notifyLocalChange();
|
|
4782
|
-
resolve();
|
|
4783
|
-
};
|
|
4784
|
-
tx.onerror = () => reject(tx.error);
|
|
4785
|
-
});
|
|
4786
|
-
}
|
|
4787
|
-
async release(id) {
|
|
4788
|
-
const job = await this.get(id);
|
|
4789
|
-
if (!job)
|
|
4790
|
-
return;
|
|
4791
|
-
job.status = JobStatus.PENDING;
|
|
4792
|
-
job.worker_id = null;
|
|
4793
|
-
job.progress = 0;
|
|
4794
|
-
job.progress_message = "";
|
|
4795
|
-
job.progress_details = null;
|
|
4796
|
-
await this.put(job);
|
|
4797
|
-
}
|
|
4798
|
-
async abort(id) {
|
|
4799
|
-
const job = await this.get(id);
|
|
4800
|
-
if (!job)
|
|
4801
|
-
return;
|
|
4802
|
-
job.status = JobStatus.ABORTING;
|
|
4803
|
-
await this.complete(job);
|
|
4804
|
-
}
|
|
4805
|
-
async getByRunId(job_run_id) {
|
|
4806
|
-
const db = await this.getDb();
|
|
4807
|
-
const tx = db.transaction(this.tableName, "readonly");
|
|
4808
|
-
const store = tx.objectStore(this.tableName);
|
|
4809
|
-
const index = store.index("queue_job_run_id");
|
|
4810
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
4811
|
-
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, job_run_id]);
|
|
4812
|
-
const request = index.getAll(keyRange);
|
|
4813
|
-
return new Promise((resolve, reject) => {
|
|
4814
|
-
request.onsuccess = () => {
|
|
4815
|
-
const results = (request.result || []).filter((job) => this.matchesPrefixes(job));
|
|
4816
|
-
resolve(results);
|
|
4817
|
-
};
|
|
4818
|
-
request.onerror = () => reject(request.error);
|
|
4819
|
-
tx.onerror = () => reject(tx.error);
|
|
4820
|
-
});
|
|
4821
|
-
}
|
|
4822
|
-
async deleteAll() {
|
|
4823
|
-
const db = await this.getDb();
|
|
4824
|
-
const tx = db.transaction(this.tableName, "readwrite");
|
|
4825
|
-
const store = tx.objectStore(this.tableName);
|
|
4826
|
-
const index = store.index("queue_status");
|
|
4827
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
4828
|
-
return new Promise((resolve, reject) => {
|
|
4829
|
-
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, ""]);
|
|
4830
|
-
const request = index.openCursor(keyRange);
|
|
4831
|
-
request.onsuccess = (event) => {
|
|
4832
|
-
const cursor = event.target.result;
|
|
4833
|
-
if (cursor) {
|
|
4834
|
-
const job = cursor.value;
|
|
4835
|
-
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
4836
|
-
const deleteRequest = cursor.delete();
|
|
4837
|
-
deleteRequest.onsuccess = () => {
|
|
4838
|
-
cursor.continue();
|
|
4839
|
-
};
|
|
4840
|
-
deleteRequest.onerror = () => {
|
|
4841
|
-
cursor.continue();
|
|
4842
|
-
};
|
|
4843
|
-
} else {
|
|
4844
|
-
cursor.continue();
|
|
4845
|
-
}
|
|
4846
|
-
}
|
|
4847
|
-
};
|
|
4848
|
-
tx.oncomplete = () => {
|
|
4849
|
-
this.hybridManager?.notifyLocalChange();
|
|
4850
|
-
resolve();
|
|
4851
|
-
};
|
|
4852
|
-
tx.onerror = () => reject(tx.error);
|
|
4853
|
-
request.onerror = () => reject(request.error);
|
|
4854
|
-
});
|
|
4855
|
-
}
|
|
4856
|
-
async outputForInput(input) {
|
|
4857
|
-
const fingerprint = await makeFingerprint6(input);
|
|
4858
|
-
const db = await this.getDb();
|
|
4859
|
-
const tx = db.transaction(this.tableName, "readonly");
|
|
4860
|
-
const store = tx.objectStore(this.tableName);
|
|
4861
|
-
const index = store.index("queue_fingerprint_status");
|
|
4862
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
4863
|
-
const request = index.get([
|
|
4864
|
-
...prefixKeyValues,
|
|
4865
|
-
this.queueName,
|
|
4866
|
-
fingerprint,
|
|
4867
|
-
JobStatus.COMPLETED
|
|
4868
|
-
]);
|
|
4869
|
-
return new Promise((resolve, reject) => {
|
|
4870
|
-
request.onsuccess = () => {
|
|
4871
|
-
const job = request.result;
|
|
4872
|
-
if (job && this.matchesPrefixes(job)) {
|
|
4873
|
-
resolve(job.output ?? null);
|
|
4874
|
-
} else {
|
|
4875
|
-
resolve(null);
|
|
4876
|
-
}
|
|
4877
|
-
};
|
|
4878
|
-
request.onerror = () => reject(request.error);
|
|
4879
|
-
tx.onerror = () => reject(tx.error);
|
|
4880
|
-
});
|
|
4881
|
-
}
|
|
4882
|
-
async saveProgress(id, progress, message, details) {
|
|
4883
|
-
const job = await this.get(id);
|
|
4884
|
-
if (!job)
|
|
4885
|
-
throw new Error(`Job ${id} not found`);
|
|
4886
|
-
job.progress = progress;
|
|
4887
|
-
job.progress_message = message;
|
|
4888
|
-
job.progress_details = details;
|
|
4889
|
-
await this.put(job);
|
|
4890
|
-
}
|
|
4891
|
-
async put(job) {
|
|
4892
|
-
const db = await this.getDb();
|
|
4893
|
-
const tx = db.transaction(this.tableName, "readwrite");
|
|
4894
|
-
const store = tx.objectStore(this.tableName);
|
|
4895
|
-
job.queue = this.queueName;
|
|
4896
|
-
const jobWithPrefixes = job;
|
|
4897
|
-
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
4898
|
-
jobWithPrefixes[key] = value;
|
|
4899
|
-
}
|
|
4900
|
-
return new Promise((resolve, reject) => {
|
|
4901
|
-
const putReq = store.put(jobWithPrefixes);
|
|
4902
|
-
putReq.onerror = () => reject(putReq.error);
|
|
4903
|
-
tx.oncomplete = () => {
|
|
4904
|
-
this.hybridManager?.notifyLocalChange();
|
|
4905
|
-
resolve();
|
|
4906
|
-
};
|
|
4907
|
-
tx.onerror = () => reject(tx.error);
|
|
4908
|
-
});
|
|
4909
|
-
}
|
|
4910
|
-
async delete(id) {
|
|
4911
|
-
const job = await this.get(id);
|
|
4912
|
-
if (!job)
|
|
4913
|
-
return;
|
|
4914
|
-
const db = await this.getDb();
|
|
4915
|
-
const tx = db.transaction(this.tableName, "readwrite");
|
|
4916
|
-
const store = tx.objectStore(this.tableName);
|
|
4917
|
-
const request = store.delete(id);
|
|
4918
|
-
return new Promise((resolve, reject) => {
|
|
4919
|
-
request.onsuccess = () => resolve();
|
|
4920
|
-
request.onerror = () => reject(request.error);
|
|
4921
|
-
tx.oncomplete = () => {
|
|
4922
|
-
this.hybridManager?.notifyLocalChange();
|
|
4923
|
-
};
|
|
4924
|
-
tx.onerror = () => reject(tx.error);
|
|
4925
|
-
});
|
|
4926
|
-
}
|
|
4927
|
-
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
4928
|
-
const db = await this.getDb();
|
|
4929
|
-
const tx = db.transaction(this.tableName, "readwrite");
|
|
4930
|
-
const store = tx.objectStore(this.tableName);
|
|
4931
|
-
const index = store.index("queue_status");
|
|
4932
|
-
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
4933
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
4934
|
-
const keyRange = IDBKeyRange.only([...prefixKeyValues, this.queueName, status]);
|
|
4935
|
-
return new Promise((resolve, reject) => {
|
|
4936
|
-
const request = index.openCursor(keyRange);
|
|
4937
|
-
request.onsuccess = (event) => {
|
|
4938
|
-
const cursor = event.target.result;
|
|
4939
|
-
if (cursor) {
|
|
4940
|
-
const job = cursor.value;
|
|
4941
|
-
if (job.queue === this.queueName && this.matchesPrefixes(job) && job.status === status && job.completed_at && job.completed_at <= cutoffDate) {
|
|
4942
|
-
cursor.delete();
|
|
4943
|
-
}
|
|
4944
|
-
cursor.continue();
|
|
4945
|
-
}
|
|
4946
|
-
};
|
|
4947
|
-
tx.oncomplete = () => {
|
|
4948
|
-
this.hybridManager?.notifyLocalChange();
|
|
4949
|
-
resolve();
|
|
4950
|
-
};
|
|
4951
|
-
tx.onerror = () => reject(tx.error);
|
|
4952
|
-
request.onerror = () => reject(request.error);
|
|
4953
|
-
});
|
|
4954
|
-
}
|
|
4955
|
-
async getAllJobs() {
|
|
4956
|
-
const db = await this.getDb();
|
|
4957
|
-
const tx = db.transaction(this.tableName, "readonly");
|
|
4958
|
-
const store = tx.objectStore(this.tableName);
|
|
4959
|
-
const index = store.index("queue_status");
|
|
4960
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
4961
|
-
return new Promise((resolve, reject) => {
|
|
4962
|
-
const jobs = [];
|
|
4963
|
-
const keyRange = IDBKeyRange.bound([...prefixKeyValues, this.queueName, ""], [...prefixKeyValues, this.queueName, ""]);
|
|
4964
|
-
const request = index.openCursor(keyRange);
|
|
4965
|
-
request.onsuccess = (event) => {
|
|
4966
|
-
const cursor = event.target.result;
|
|
4967
|
-
if (cursor) {
|
|
4968
|
-
const job = cursor.value;
|
|
4969
|
-
if (job.queue === this.queueName && this.matchesPrefixes(job)) {
|
|
4970
|
-
jobs.push(job);
|
|
4971
|
-
}
|
|
4972
|
-
cursor.continue();
|
|
4973
|
-
}
|
|
4974
|
-
};
|
|
4975
|
-
tx.oncomplete = () => resolve(jobs);
|
|
4976
|
-
tx.onerror = () => reject(tx.error);
|
|
4977
|
-
request.onerror = () => reject(request.error);
|
|
4978
|
-
});
|
|
4979
|
-
}
|
|
4980
|
-
async getAllJobsWithFilter(prefixFilter) {
|
|
4981
|
-
const db = await this.getDb();
|
|
4982
|
-
const tx = db.transaction(this.tableName, "readonly");
|
|
4983
|
-
const store = tx.objectStore(this.tableName);
|
|
4984
|
-
return new Promise((resolve, reject) => {
|
|
4985
|
-
const jobs = [];
|
|
4986
|
-
const request = store.openCursor();
|
|
4987
|
-
request.onsuccess = (event) => {
|
|
4988
|
-
const cursor = event.target.result;
|
|
4989
|
-
if (cursor) {
|
|
4990
|
-
const job = cursor.value;
|
|
4991
|
-
if (job.queue !== this.queueName) {
|
|
4992
|
-
cursor.continue();
|
|
4993
|
-
return;
|
|
4994
|
-
}
|
|
4995
|
-
if (Object.keys(prefixFilter).length === 0) {
|
|
4996
|
-
jobs.push(job);
|
|
4997
|
-
} else {
|
|
4998
|
-
let matches = true;
|
|
4999
|
-
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
5000
|
-
if (job[key] !== value) {
|
|
5001
|
-
matches = false;
|
|
5002
|
-
break;
|
|
5003
|
-
}
|
|
5004
|
-
}
|
|
5005
|
-
if (matches) {
|
|
5006
|
-
jobs.push(job);
|
|
5007
|
-
}
|
|
5008
|
-
}
|
|
5009
|
-
cursor.continue();
|
|
5010
|
-
}
|
|
5011
|
-
};
|
|
5012
|
-
tx.oncomplete = () => resolve(jobs);
|
|
5013
|
-
tx.onerror = () => reject(tx.error);
|
|
5014
|
-
request.onerror = () => reject(request.error);
|
|
5015
|
-
});
|
|
5016
|
-
}
|
|
5017
|
-
isCustomPrefixFilter(prefixFilter) {
|
|
5018
|
-
if (prefixFilter === undefined) {
|
|
5019
|
-
return false;
|
|
5020
|
-
}
|
|
5021
|
-
if (Object.keys(prefixFilter).length === 0) {
|
|
5022
|
-
return true;
|
|
5023
|
-
}
|
|
5024
|
-
const instanceKeys = Object.keys(this.prefixValues);
|
|
5025
|
-
const filterKeys = Object.keys(prefixFilter);
|
|
5026
|
-
if (instanceKeys.length !== filterKeys.length) {
|
|
5027
|
-
return true;
|
|
5028
|
-
}
|
|
5029
|
-
for (const key of instanceKeys) {
|
|
5030
|
-
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
5031
|
-
return true;
|
|
5032
|
-
}
|
|
5033
|
-
}
|
|
5034
|
-
return false;
|
|
5035
|
-
}
|
|
5036
|
-
getHybridManager() {
|
|
5037
|
-
if (!this.hybridManager) {
|
|
5038
|
-
const channelName = `indexeddb-queue-${this.tableName}-${this.queueName}`;
|
|
5039
|
-
this.hybridManager = new HybridSubscriptionManager(channelName, async () => {
|
|
5040
|
-
const jobs = await this.getAllJobs();
|
|
5041
|
-
return new Map(jobs.map((j) => [j.id, j]));
|
|
5042
|
-
}, (a, b) => deepEqual3(a, b), {
|
|
5043
|
-
insert: (item) => ({ type: "INSERT", new: item }),
|
|
5044
|
-
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
5045
|
-
delete: (item) => ({ type: "DELETE", old: item })
|
|
5046
|
-
}, {
|
|
5047
|
-
defaultIntervalMs: 1000,
|
|
5048
|
-
useBroadcastChannel: this.hybridOptions.useBroadcastChannel,
|
|
5049
|
-
backupPollingIntervalMs: this.hybridOptions.backupPollingIntervalMs
|
|
5050
|
-
});
|
|
5051
|
-
}
|
|
5052
|
-
return this.hybridManager;
|
|
5053
|
-
}
|
|
5054
|
-
subscribeWithCustomPrefixFilter(callback, prefixFilter, intervalMs) {
|
|
5055
|
-
let lastKnownJobs = new Map;
|
|
5056
|
-
let cancelled = false;
|
|
5057
|
-
const poll = async () => {
|
|
5058
|
-
if (cancelled)
|
|
5059
|
-
return;
|
|
5060
|
-
try {
|
|
5061
|
-
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
5062
|
-
if (cancelled)
|
|
5063
|
-
return;
|
|
5064
|
-
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
5065
|
-
for (const [id, job] of currentMap) {
|
|
5066
|
-
const old = lastKnownJobs.get(id);
|
|
5067
|
-
if (!old) {
|
|
5068
|
-
callback({ type: "INSERT", new: job });
|
|
5069
|
-
} else if (!deepEqual3(old, job)) {
|
|
5070
|
-
callback({ type: "UPDATE", old, new: job });
|
|
5071
|
-
}
|
|
5072
|
-
}
|
|
5073
|
-
for (const [id, job] of lastKnownJobs) {
|
|
5074
|
-
if (!currentMap.has(id)) {
|
|
5075
|
-
callback({ type: "DELETE", old: job });
|
|
5076
|
-
}
|
|
5077
|
-
}
|
|
5078
|
-
lastKnownJobs = currentMap;
|
|
5079
|
-
} catch {}
|
|
5080
|
-
};
|
|
5081
|
-
const intervalId = setInterval(poll, intervalMs);
|
|
5082
|
-
poll();
|
|
5083
|
-
return () => {
|
|
5084
|
-
cancelled = true;
|
|
5085
|
-
clearInterval(intervalId);
|
|
5086
|
-
};
|
|
5087
|
-
}
|
|
5088
|
-
subscribeToChanges(callback, options) {
|
|
5089
|
-
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
5090
|
-
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
5091
|
-
return this.subscribeWithCustomPrefixFilter(callback, options.prefixFilter, intervalMs);
|
|
5092
|
-
}
|
|
5093
|
-
const manager = this.getHybridManager();
|
|
5094
|
-
return manager.subscribe(callback, { intervalMs });
|
|
5095
|
-
}
|
|
5096
|
-
destroy() {
|
|
5097
|
-
if (this.hybridManager) {
|
|
5098
|
-
this.hybridManager.destroy();
|
|
5099
|
-
this.hybridManager = null;
|
|
5100
|
-
}
|
|
5101
|
-
}
|
|
5102
|
-
}
|
|
5103
|
-
// src/queue/SupabaseQueueStorage.ts
|
|
5104
|
-
import { createServiceToken as createServiceToken18, deepEqual as deepEqual4, makeFingerprint as makeFingerprint7, uuid4 as uuid46 } from "@workglow/util";
|
|
5105
|
-
var SUPABASE_QUEUE_STORAGE = createServiceToken18("jobqueue.storage.supabase");
|
|
5106
|
-
|
|
5107
|
-
class SupabaseQueueStorage {
|
|
5108
|
-
queueName;
|
|
5109
|
-
scope = "cluster";
|
|
5110
|
-
client;
|
|
5111
|
-
prefixes;
|
|
5112
|
-
prefixValues;
|
|
5113
|
-
tableName;
|
|
5114
|
-
realtimeChannel = null;
|
|
5115
|
-
pollingManager = null;
|
|
5116
|
-
constructor(client, queueName, options) {
|
|
5117
|
-
this.queueName = queueName;
|
|
5118
|
-
this.client = client;
|
|
5119
|
-
this.prefixes = options?.prefixes ?? [];
|
|
5120
|
-
this.prefixValues = options?.prefixValues ?? {};
|
|
5121
|
-
if (this.prefixes.length > 0) {
|
|
5122
|
-
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
5123
|
-
this.tableName = `job_queue_${prefixNames}`;
|
|
5124
|
-
} else {
|
|
5125
|
-
this.tableName = "job_queue";
|
|
5126
|
-
}
|
|
5127
|
-
}
|
|
5128
|
-
getPrefixColumnType(type) {
|
|
5129
|
-
return type === "uuid" ? "UUID" : "INTEGER";
|
|
5130
|
-
}
|
|
5131
|
-
buildPrefixColumnsSql() {
|
|
5132
|
-
if (this.prefixes.length === 0)
|
|
5133
|
-
return "";
|
|
5134
|
-
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
5135
|
-
`) + `,
|
|
5136
|
-
`;
|
|
5137
|
-
}
|
|
5138
|
-
getPrefixColumnNames() {
|
|
5139
|
-
return this.prefixes.map((p) => p.name);
|
|
5140
|
-
}
|
|
5141
|
-
applyPrefixFilters(query) {
|
|
5142
|
-
let result = query;
|
|
5143
|
-
for (const prefix of this.prefixes) {
|
|
5144
|
-
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
5145
|
-
}
|
|
5146
|
-
return result;
|
|
5147
|
-
}
|
|
5148
|
-
getPrefixInsertValues() {
|
|
5149
|
-
const values = {};
|
|
5150
|
-
for (const prefix of this.prefixes) {
|
|
5151
|
-
values[prefix.name] = this.prefixValues[prefix.name];
|
|
5152
|
-
}
|
|
5153
|
-
return values;
|
|
5154
|
-
}
|
|
5155
|
-
buildPrefixWhereSql() {
|
|
5156
|
-
if (this.prefixes.length === 0) {
|
|
5157
|
-
return "";
|
|
5158
|
-
}
|
|
5159
|
-
const conditions = this.prefixes.map((p) => {
|
|
5160
|
-
const value = this.prefixValues[p.name];
|
|
5161
|
-
if (p.type === "uuid") {
|
|
5162
|
-
const validated = this.validateSqlValue(String(value), `prefix "${p.name}"`);
|
|
5163
|
-
return `${p.name} = '${this.escapeSqlString(validated)}'`;
|
|
5164
|
-
}
|
|
5165
|
-
const numValue = Number(value ?? 0);
|
|
5166
|
-
if (!Number.isFinite(numValue)) {
|
|
5167
|
-
throw new Error(`Invalid numeric prefix value for "${p.name}": ${value}`);
|
|
5168
|
-
}
|
|
5169
|
-
return `${p.name} = ${numValue}`;
|
|
5170
|
-
}).join(" AND ");
|
|
5171
|
-
return " AND " + conditions;
|
|
5172
|
-
}
|
|
5173
|
-
static SAFE_SQL_VALUE_RE = /^[a-zA-Z0-9_\-.:]+$/;
|
|
5174
|
-
validateSqlValue(value, context) {
|
|
5175
|
-
if (!SupabaseQueueStorage.SAFE_SQL_VALUE_RE.test(value)) {
|
|
5176
|
-
throw new Error(`Unsafe value for ${context}: "${value}". Values must match /^[a-zA-Z0-9_\\-.:]+$/.`);
|
|
5177
|
-
}
|
|
5178
|
-
return value;
|
|
5179
|
-
}
|
|
5180
|
-
escapeSqlString(value) {
|
|
5181
|
-
return value.replace(/'/g, "''");
|
|
5182
|
-
}
|
|
5183
|
-
async setupDatabase() {
|
|
5184
|
-
const createTypeSql = `CREATE TYPE job_status AS ENUM (${Object.values(JobStatus).map((v) => `'${v}'`).join(",")})`;
|
|
5185
|
-
const { error: typeError } = await this.client.rpc("exec_sql", { query: createTypeSql });
|
|
5186
|
-
if (typeError && typeError.code !== "42710") {
|
|
5187
|
-
throw typeError;
|
|
5188
|
-
}
|
|
5189
|
-
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
5190
|
-
const prefixColumnNames = this.getPrefixColumnNames();
|
|
5191
|
-
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
5192
|
-
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
5193
|
-
const createTableSql = `
|
|
5194
|
-
CREATE TABLE IF NOT EXISTS ${this.tableName} (
|
|
5195
|
-
id SERIAL NOT NULL,
|
|
5196
|
-
${prefixColumnsSql}fingerprint text NOT NULL,
|
|
5197
|
-
queue text NOT NULL,
|
|
5198
|
-
job_run_id text NOT NULL,
|
|
5199
|
-
status job_status NOT NULL default 'PENDING',
|
|
5200
|
-
input jsonb NOT NULL,
|
|
5201
|
-
output jsonb,
|
|
5202
|
-
run_attempts integer default 0,
|
|
5203
|
-
max_retries integer default 20,
|
|
5204
|
-
run_after timestamp with time zone DEFAULT now(),
|
|
5205
|
-
last_ran_at timestamp with time zone,
|
|
5206
|
-
created_at timestamp with time zone DEFAULT now(),
|
|
5207
|
-
deadline_at timestamp with time zone,
|
|
5208
|
-
completed_at timestamp with time zone,
|
|
5209
|
-
error text,
|
|
5210
|
-
error_code text,
|
|
5211
|
-
progress real DEFAULT 0,
|
|
5212
|
-
progress_message text DEFAULT '',
|
|
5213
|
-
progress_details jsonb,
|
|
5214
|
-
worker_id text
|
|
5215
|
-
)`;
|
|
5216
|
-
const { error: tableError } = await this.client.rpc("exec_sql", { query: createTableSql });
|
|
5217
|
-
if (tableError) {
|
|
5218
|
-
if (tableError.code !== "42P07") {
|
|
5219
|
-
throw tableError;
|
|
5220
|
-
}
|
|
5221
|
-
}
|
|
5222
|
-
const indexes = [
|
|
5223
|
-
`CREATE INDEX IF NOT EXISTS job_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}id, status, run_after)`,
|
|
5224
|
-
`CREATE INDEX IF NOT EXISTS job_queue_fetcher${indexSuffix}_idx ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`,
|
|
5225
|
-
`CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`
|
|
5226
|
-
];
|
|
5227
|
-
for (const indexSql of indexes) {
|
|
5228
|
-
await this.client.rpc("exec_sql", { query: indexSql });
|
|
5229
|
-
}
|
|
5230
|
-
}
|
|
5231
|
-
async add(job) {
|
|
5232
|
-
const now = new Date().toISOString();
|
|
5233
|
-
job.queue = this.queueName;
|
|
5234
|
-
job.job_run_id = job.job_run_id ?? uuid46();
|
|
5235
|
-
job.fingerprint = await makeFingerprint7(job.input);
|
|
5236
|
-
job.status = JobStatus.PENDING;
|
|
5237
|
-
job.progress = 0;
|
|
5238
|
-
job.progress_message = "";
|
|
5239
|
-
job.progress_details = null;
|
|
5240
|
-
job.created_at = now;
|
|
5241
|
-
job.run_after = now;
|
|
5242
|
-
const prefixInsertValues = this.getPrefixInsertValues();
|
|
5243
|
-
const { data, error } = await this.client.from(this.tableName).insert({
|
|
5244
|
-
...prefixInsertValues,
|
|
5245
|
-
queue: job.queue,
|
|
5246
|
-
fingerprint: job.fingerprint,
|
|
5247
|
-
input: job.input,
|
|
5248
|
-
run_after: job.run_after,
|
|
5249
|
-
created_at: job.created_at,
|
|
5250
|
-
deadline_at: job.deadline_at,
|
|
5251
|
-
max_retries: job.max_retries,
|
|
5252
|
-
job_run_id: job.job_run_id,
|
|
5253
|
-
progress: job.progress,
|
|
5254
|
-
progress_message: job.progress_message,
|
|
5255
|
-
progress_details: job.progress_details
|
|
5256
|
-
}).select("id").single();
|
|
5257
|
-
if (error)
|
|
5258
|
-
throw error;
|
|
5259
|
-
if (!data)
|
|
5260
|
-
throw new Error("Failed to add to queue");
|
|
5261
|
-
job.id = data.id;
|
|
5262
|
-
return job.id;
|
|
5263
|
-
}
|
|
5264
|
-
async get(id) {
|
|
5265
|
-
let query = this.client.from(this.tableName).select("*").eq("id", id).eq("queue", this.queueName);
|
|
5266
|
-
query = this.applyPrefixFilters(query);
|
|
5267
|
-
const { data, error } = await query.single();
|
|
5268
|
-
if (error) {
|
|
5269
|
-
if (error.code === "PGRST116")
|
|
5270
|
-
return;
|
|
5271
|
-
throw error;
|
|
5272
|
-
}
|
|
5273
|
-
return data;
|
|
5274
|
-
}
|
|
5275
|
-
async peek(status = JobStatus.PENDING, num = 100) {
|
|
5276
|
-
num = Number(num) || 100;
|
|
5277
|
-
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName).eq("status", status);
|
|
5278
|
-
query = this.applyPrefixFilters(query);
|
|
5279
|
-
const { data, error } = await query.order("run_after", { ascending: true }).limit(num);
|
|
5280
|
-
if (error)
|
|
5281
|
-
throw error;
|
|
5282
|
-
return data ?? [];
|
|
5283
|
-
}
|
|
5284
|
-
async next(workerId) {
|
|
5285
|
-
const prefixConditions = this.buildPrefixWhereSql();
|
|
5286
|
-
const validatedQueueName = this.validateSqlValue(this.queueName, "queueName");
|
|
5287
|
-
const validatedWorkerId = this.validateSqlValue(workerId, "workerId");
|
|
5288
|
-
const escapedQueueName = this.escapeSqlString(validatedQueueName);
|
|
5289
|
-
const escapedWorkerId = this.escapeSqlString(validatedWorkerId);
|
|
5290
|
-
const sql = `
|
|
5291
|
-
UPDATE ${this.tableName}
|
|
5292
|
-
SET status = '${JobStatus.PROCESSING}', last_ran_at = NOW() AT TIME ZONE 'UTC', worker_id = '${escapedWorkerId}'
|
|
5293
|
-
WHERE id = (
|
|
5294
|
-
SELECT id
|
|
5295
|
-
FROM ${this.tableName}
|
|
5296
|
-
WHERE queue = '${escapedQueueName}'
|
|
5297
|
-
AND status = '${JobStatus.PENDING}'
|
|
5298
|
-
${prefixConditions}
|
|
5299
|
-
AND run_after <= NOW() AT TIME ZONE 'UTC'
|
|
5300
|
-
ORDER BY run_after ASC
|
|
5301
|
-
FOR UPDATE SKIP LOCKED
|
|
5302
|
-
LIMIT 1
|
|
5303
|
-
)
|
|
5304
|
-
RETURNING *`;
|
|
5305
|
-
const { data, error } = await this.client.rpc("exec_sql", { query: sql });
|
|
5306
|
-
if (error)
|
|
5307
|
-
throw error;
|
|
5308
|
-
if (!data || !Array.isArray(data) || data.length === 0) {
|
|
5309
|
-
return;
|
|
5310
|
-
}
|
|
5311
|
-
return data[0];
|
|
5312
|
-
}
|
|
5313
|
-
async size(status = JobStatus.PENDING) {
|
|
5314
|
-
let query = this.client.from(this.tableName).select("*", { count: "exact", head: true }).eq("queue", this.queueName).eq("status", status);
|
|
5315
|
-
query = this.applyPrefixFilters(query);
|
|
5316
|
-
const { count, error } = await query;
|
|
5317
|
-
if (error)
|
|
5318
|
-
throw error;
|
|
5319
|
-
return count ?? 0;
|
|
5320
|
-
}
|
|
5321
|
-
async getAllJobs() {
|
|
5322
|
-
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
5323
|
-
query = this.applyPrefixFilters(query);
|
|
5324
|
-
const { data, error } = await query;
|
|
5325
|
-
if (error)
|
|
5326
|
-
throw error;
|
|
5327
|
-
return data ?? [];
|
|
5328
|
-
}
|
|
5329
|
-
async complete(jobDetails) {
|
|
5330
|
-
const now = new Date().toISOString();
|
|
5331
|
-
if (jobDetails.status === JobStatus.DISABLED) {
|
|
5332
|
-
let query2 = this.client.from(this.tableName).update({
|
|
5333
|
-
status: jobDetails.status,
|
|
5334
|
-
progress: 100,
|
|
5335
|
-
progress_message: "",
|
|
5336
|
-
progress_details: null,
|
|
5337
|
-
completed_at: now,
|
|
5338
|
-
last_ran_at: now
|
|
5339
|
-
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
5340
|
-
query2 = this.applyPrefixFilters(query2);
|
|
5341
|
-
const { error: error2 } = await query2;
|
|
5342
|
-
if (error2)
|
|
5343
|
-
throw error2;
|
|
5344
|
-
return;
|
|
5345
|
-
}
|
|
5346
|
-
let getQuery = this.client.from(this.tableName).select("run_attempts, max_retries").eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
5347
|
-
getQuery = this.applyPrefixFilters(getQuery);
|
|
5348
|
-
const { data: current, error: getError } = await getQuery.single();
|
|
5349
|
-
if (getError)
|
|
5350
|
-
throw getError;
|
|
5351
|
-
const currentAttempts = current?.run_attempts ?? 0;
|
|
5352
|
-
const maxRetries = current?.max_retries ?? jobDetails.max_retries ?? 10;
|
|
5353
|
-
const nextAttempts = currentAttempts + 1;
|
|
5354
|
-
if (jobDetails.status === JobStatus.PENDING) {
|
|
5355
|
-
if (nextAttempts > maxRetries) {
|
|
5356
|
-
let failQuery = this.client.from(this.tableName).update({
|
|
5357
|
-
status: JobStatus.FAILED,
|
|
5358
|
-
error: "Max retries reached",
|
|
5359
|
-
error_code: "MAX_RETRIES_REACHED",
|
|
5360
|
-
progress: 100,
|
|
5361
|
-
progress_message: "",
|
|
5362
|
-
progress_details: null,
|
|
5363
|
-
completed_at: now,
|
|
5364
|
-
last_ran_at: now
|
|
5365
|
-
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
5366
|
-
failQuery = this.applyPrefixFilters(failQuery);
|
|
5367
|
-
const { error: failError } = await failQuery;
|
|
5368
|
-
if (failError)
|
|
5369
|
-
throw failError;
|
|
5370
|
-
return;
|
|
5371
|
-
}
|
|
5372
|
-
let query2 = this.client.from(this.tableName).update({
|
|
5373
|
-
error: jobDetails.error ?? null,
|
|
5374
|
-
error_code: jobDetails.error_code ?? null,
|
|
5375
|
-
status: jobDetails.status,
|
|
5376
|
-
run_after: jobDetails.run_after,
|
|
5377
|
-
progress: 0,
|
|
5378
|
-
progress_message: "",
|
|
5379
|
-
progress_details: null,
|
|
5380
|
-
run_attempts: nextAttempts,
|
|
5381
|
-
last_ran_at: now
|
|
5382
|
-
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
5383
|
-
query2 = this.applyPrefixFilters(query2);
|
|
5384
|
-
const { error: error2 } = await query2;
|
|
5385
|
-
if (error2)
|
|
5386
|
-
throw error2;
|
|
5387
|
-
return;
|
|
5388
|
-
}
|
|
5389
|
-
if (jobDetails.status === JobStatus.COMPLETED || jobDetails.status === JobStatus.FAILED) {
|
|
5390
|
-
let query2 = this.client.from(this.tableName).update({
|
|
5391
|
-
output: jobDetails.output ?? null,
|
|
5392
|
-
error: jobDetails.error ?? null,
|
|
5393
|
-
error_code: jobDetails.error_code ?? null,
|
|
5394
|
-
status: jobDetails.status,
|
|
5395
|
-
progress: 100,
|
|
5396
|
-
progress_message: "",
|
|
5397
|
-
progress_details: null,
|
|
5398
|
-
run_attempts: nextAttempts,
|
|
5399
|
-
completed_at: now,
|
|
5400
|
-
last_ran_at: now
|
|
5401
|
-
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
5402
|
-
query2 = this.applyPrefixFilters(query2);
|
|
5403
|
-
const { error: error2 } = await query2;
|
|
5404
|
-
if (error2)
|
|
5405
|
-
throw error2;
|
|
5406
|
-
return;
|
|
5407
|
-
}
|
|
5408
|
-
let query = this.client.from(this.tableName).update({
|
|
5409
|
-
status: jobDetails.status,
|
|
5410
|
-
output: jobDetails.output ?? null,
|
|
5411
|
-
error: jobDetails.error ?? null,
|
|
5412
|
-
error_code: jobDetails.error_code ?? null,
|
|
5413
|
-
run_after: jobDetails.run_after ?? null,
|
|
5414
|
-
run_attempts: nextAttempts,
|
|
5415
|
-
last_ran_at: now
|
|
5416
|
-
}).eq("id", jobDetails.id).eq("queue", this.queueName);
|
|
5417
|
-
query = this.applyPrefixFilters(query);
|
|
5418
|
-
const { error } = await query;
|
|
5419
|
-
if (error)
|
|
5420
|
-
throw error;
|
|
5421
|
-
}
|
|
5422
|
-
async release(jobId) {
|
|
5423
|
-
let query = this.client.from(this.tableName).update({
|
|
5424
|
-
status: JobStatus.PENDING,
|
|
5425
|
-
worker_id: null,
|
|
5426
|
-
progress: 0,
|
|
5427
|
-
progress_message: "",
|
|
5428
|
-
progress_details: null
|
|
5429
|
-
}).eq("id", jobId).eq("queue", this.queueName);
|
|
5430
|
-
query = this.applyPrefixFilters(query);
|
|
5431
|
-
const { error } = await query;
|
|
5432
|
-
if (error)
|
|
5433
|
-
throw error;
|
|
5434
|
-
}
|
|
5435
|
-
async deleteAll() {
|
|
5436
|
-
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName);
|
|
5437
|
-
query = this.applyPrefixFilters(query);
|
|
5438
|
-
const { error } = await query;
|
|
5439
|
-
if (error)
|
|
5440
|
-
throw error;
|
|
5441
|
-
}
|
|
5442
|
-
async outputForInput(input) {
|
|
5443
|
-
const fingerprint = await makeFingerprint7(input);
|
|
5444
|
-
let query = this.client.from(this.tableName).select("output").eq("fingerprint", fingerprint).eq("queue", this.queueName).eq("status", JobStatus.COMPLETED);
|
|
5445
|
-
query = this.applyPrefixFilters(query);
|
|
5446
|
-
const { data, error } = await query.single();
|
|
5447
|
-
if (error) {
|
|
5448
|
-
if (error.code === "PGRST116")
|
|
5449
|
-
return null;
|
|
5450
|
-
throw error;
|
|
5451
|
-
}
|
|
5452
|
-
return data?.output ?? null;
|
|
5453
|
-
}
|
|
5454
|
-
async abort(jobId) {
|
|
5455
|
-
let query = this.client.from(this.tableName).update({ status: JobStatus.ABORTING }).eq("id", jobId).eq("queue", this.queueName);
|
|
5456
|
-
query = this.applyPrefixFilters(query);
|
|
5457
|
-
const { error } = await query;
|
|
5458
|
-
if (error)
|
|
5459
|
-
throw error;
|
|
5460
|
-
}
|
|
5461
|
-
async getByRunId(job_run_id) {
|
|
5462
|
-
let query = this.client.from(this.tableName).select("*").eq("job_run_id", job_run_id).eq("queue", this.queueName);
|
|
5463
|
-
query = this.applyPrefixFilters(query);
|
|
5464
|
-
const { data, error } = await query;
|
|
5465
|
-
if (error)
|
|
5466
|
-
throw error;
|
|
5467
|
-
return data ?? [];
|
|
5468
|
-
}
|
|
5469
|
-
async saveProgress(jobId, progress, message, details) {
|
|
5470
|
-
let query = this.client.from(this.tableName).update({
|
|
5471
|
-
progress,
|
|
5472
|
-
progress_message: message,
|
|
5473
|
-
progress_details: details
|
|
5474
|
-
}).eq("id", jobId).eq("queue", this.queueName);
|
|
5475
|
-
query = this.applyPrefixFilters(query);
|
|
5476
|
-
const { error } = await query;
|
|
5477
|
-
if (error)
|
|
5478
|
-
throw error;
|
|
5479
|
-
}
|
|
5480
|
-
async delete(jobId) {
|
|
5481
|
-
let query = this.client.from(this.tableName).delete().eq("id", jobId).eq("queue", this.queueName);
|
|
5482
|
-
query = this.applyPrefixFilters(query);
|
|
5483
|
-
const { error } = await query;
|
|
5484
|
-
if (error)
|
|
5485
|
-
throw error;
|
|
5486
|
-
}
|
|
5487
|
-
async deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
5488
|
-
const cutoffDate = new Date(Date.now() - olderThanMs).toISOString();
|
|
5489
|
-
let query = this.client.from(this.tableName).delete().eq("queue", this.queueName).eq("status", status).not("completed_at", "is", null).lte("completed_at", cutoffDate);
|
|
5490
|
-
query = this.applyPrefixFilters(query);
|
|
5491
|
-
const { error } = await query;
|
|
5492
|
-
if (error)
|
|
5493
|
-
throw error;
|
|
5494
|
-
}
|
|
5495
|
-
matchesPrefixFilter(job, prefixFilter) {
|
|
5496
|
-
if (!job)
|
|
5497
|
-
return false;
|
|
5498
|
-
if (job.queue !== this.queueName) {
|
|
5499
|
-
return false;
|
|
5500
|
-
}
|
|
5501
|
-
if (prefixFilter && Object.keys(prefixFilter).length === 0) {
|
|
5502
|
-
return true;
|
|
5503
|
-
}
|
|
5504
|
-
const filterValues = prefixFilter ?? this.prefixValues;
|
|
5505
|
-
if (Object.keys(filterValues).length === 0) {
|
|
5506
|
-
return true;
|
|
5507
|
-
}
|
|
5508
|
-
for (const [key, value] of Object.entries(filterValues)) {
|
|
5509
|
-
if (job[key] !== value) {
|
|
5510
|
-
return false;
|
|
5511
|
-
}
|
|
5512
|
-
}
|
|
5513
|
-
return true;
|
|
5514
|
-
}
|
|
5515
|
-
isCustomPrefixFilter(prefixFilter) {
|
|
5516
|
-
if (prefixFilter === undefined) {
|
|
5517
|
-
return false;
|
|
5518
|
-
}
|
|
5519
|
-
if (Object.keys(prefixFilter).length === 0) {
|
|
5520
|
-
return true;
|
|
5521
|
-
}
|
|
5522
|
-
const instanceKeys = Object.keys(this.prefixValues);
|
|
5523
|
-
const filterKeys = Object.keys(prefixFilter);
|
|
5524
|
-
if (instanceKeys.length !== filterKeys.length) {
|
|
5525
|
-
return true;
|
|
5526
|
-
}
|
|
5527
|
-
for (const key of instanceKeys) {
|
|
5528
|
-
if (this.prefixValues[key] !== prefixFilter[key]) {
|
|
5529
|
-
return true;
|
|
5530
|
-
}
|
|
5531
|
-
}
|
|
5532
|
-
return false;
|
|
5533
|
-
}
|
|
5534
|
-
async getAllJobsWithFilter(prefixFilter) {
|
|
5535
|
-
let query = this.client.from(this.tableName).select("*").eq("queue", this.queueName);
|
|
5536
|
-
for (const [key, value] of Object.entries(prefixFilter)) {
|
|
5537
|
-
query = query.eq(key, value);
|
|
5538
|
-
}
|
|
5539
|
-
const { data, error } = await query;
|
|
5540
|
-
if (error)
|
|
5541
|
-
throw error;
|
|
5542
|
-
return data ?? [];
|
|
5543
|
-
}
|
|
5544
|
-
subscribeToChanges(callback, options) {
|
|
5545
|
-
return this.subscribeToChangesWithRealtime(callback, options?.prefixFilter);
|
|
5546
|
-
}
|
|
5547
|
-
subscribeToChangesWithRealtime(callback, prefixFilter) {
|
|
5548
|
-
const channelName = `queue-${this.tableName}-${this.queueName}-${Date.now()}`;
|
|
5549
|
-
this.realtimeChannel = this.client.channel(channelName).on("postgres_changes", {
|
|
5550
|
-
event: "*",
|
|
5551
|
-
schema: "public",
|
|
5552
|
-
table: this.tableName,
|
|
5553
|
-
filter: `queue=eq.${this.queueName}`
|
|
5554
|
-
}, (payload) => {
|
|
5555
|
-
const newJob = payload.new;
|
|
5556
|
-
const oldJob = payload.old;
|
|
5557
|
-
const newMatches = this.matchesPrefixFilter(newJob, prefixFilter);
|
|
5558
|
-
const oldMatches = this.matchesPrefixFilter(oldJob, prefixFilter);
|
|
5559
|
-
if (!newMatches && !oldMatches) {
|
|
5560
|
-
return;
|
|
5561
|
-
}
|
|
5562
|
-
callback({
|
|
5563
|
-
type: payload.eventType.toUpperCase(),
|
|
5564
|
-
old: oldJob && Object.keys(oldJob).length > 0 ? oldJob : undefined,
|
|
5565
|
-
new: newJob && Object.keys(newJob).length > 0 ? newJob : undefined
|
|
5566
|
-
});
|
|
5567
|
-
}).subscribe();
|
|
5568
|
-
return () => {
|
|
5569
|
-
if (this.realtimeChannel) {
|
|
5570
|
-
this.client.removeChannel(this.realtimeChannel);
|
|
5571
|
-
this.realtimeChannel = null;
|
|
5572
|
-
}
|
|
5573
|
-
};
|
|
5574
|
-
}
|
|
5575
|
-
getPollingManager() {
|
|
5576
|
-
if (!this.pollingManager) {
|
|
5577
|
-
this.pollingManager = new PollingSubscriptionManager(async () => {
|
|
5578
|
-
const jobs = await this.getAllJobs();
|
|
5579
|
-
return new Map(jobs.map((j) => [j.id, j]));
|
|
5580
|
-
}, (a, b) => deepEqual4(a, b), {
|
|
5581
|
-
insert: (item) => ({ type: "INSERT", new: item }),
|
|
5582
|
-
update: (oldItem, newItem) => ({ type: "UPDATE", old: oldItem, new: newItem }),
|
|
5583
|
-
delete: (item) => ({ type: "DELETE", old: item })
|
|
5584
|
-
});
|
|
5585
|
-
}
|
|
5586
|
-
return this.pollingManager;
|
|
5587
|
-
}
|
|
5588
|
-
subscribeWithCustomPrefixFilterPolling(callback, prefixFilter, intervalMs) {
|
|
5589
|
-
let lastKnownJobs = new Map;
|
|
5590
|
-
let cancelled = false;
|
|
5591
|
-
const poll = async () => {
|
|
5592
|
-
if (cancelled)
|
|
5593
|
-
return;
|
|
5594
|
-
try {
|
|
5595
|
-
const currentJobs = await this.getAllJobsWithFilter(prefixFilter);
|
|
5596
|
-
if (cancelled)
|
|
5597
|
-
return;
|
|
5598
|
-
const currentMap = new Map(currentJobs.map((j) => [j.id, j]));
|
|
5599
|
-
for (const [id, job] of currentMap) {
|
|
5600
|
-
const old = lastKnownJobs.get(id);
|
|
5601
|
-
if (!old) {
|
|
5602
|
-
callback({ type: "INSERT", new: job });
|
|
5603
|
-
} else if (!deepEqual4(old, job)) {
|
|
5604
|
-
callback({ type: "UPDATE", old, new: job });
|
|
5605
|
-
}
|
|
5606
|
-
}
|
|
5607
|
-
for (const [id, job] of lastKnownJobs) {
|
|
5608
|
-
if (!currentMap.has(id)) {
|
|
5609
|
-
callback({ type: "DELETE", old: job });
|
|
5610
|
-
}
|
|
5611
|
-
}
|
|
5612
|
-
lastKnownJobs = currentMap;
|
|
5613
|
-
} catch {}
|
|
5614
|
-
};
|
|
5615
|
-
const intervalId = setInterval(poll, intervalMs);
|
|
5616
|
-
poll();
|
|
5617
|
-
return () => {
|
|
5618
|
-
cancelled = true;
|
|
5619
|
-
clearInterval(intervalId);
|
|
5620
|
-
};
|
|
5621
|
-
}
|
|
5622
|
-
subscribeToChangesWithPolling(callback, options) {
|
|
5623
|
-
const intervalMs = options?.pollingIntervalMs ?? 1000;
|
|
5624
|
-
if (this.isCustomPrefixFilter(options?.prefixFilter)) {
|
|
5625
|
-
return this.subscribeWithCustomPrefixFilterPolling(callback, options.prefixFilter, intervalMs);
|
|
5626
|
-
}
|
|
5627
|
-
const manager = this.getPollingManager();
|
|
5628
|
-
return manager.subscribe(callback, { intervalMs });
|
|
5629
|
-
}
|
|
5630
|
-
}
|
|
5631
|
-
// src/queue-limiter/IndexedDbRateLimiterStorage.ts
|
|
5632
|
-
import { createServiceToken as createServiceToken19 } from "@workglow/util";
|
|
5633
|
-
var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken19("ratelimiter.storage.indexedDb");
|
|
5634
|
-
|
|
5635
|
-
class IndexedDbRateLimiterStorage {
|
|
5636
|
-
scope = "process";
|
|
5637
|
-
executionDb;
|
|
5638
|
-
nextAvailableDb;
|
|
5639
|
-
executionTableName;
|
|
5640
|
-
nextAvailableTableName;
|
|
5641
|
-
migrationOptions;
|
|
5642
|
-
prefixes;
|
|
5643
|
-
prefixValues;
|
|
5644
|
-
constructor(options = {}) {
|
|
5645
|
-
this.migrationOptions = options;
|
|
5646
|
-
this.prefixes = options.prefixes ?? [];
|
|
5647
|
-
this.prefixValues = options.prefixValues ?? {};
|
|
5648
|
-
if (this.prefixes.length > 0) {
|
|
5649
|
-
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
5650
|
-
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
5651
|
-
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
5652
|
-
} else {
|
|
5653
|
-
this.executionTableName = "rate_limit_executions";
|
|
5654
|
-
this.nextAvailableTableName = "rate_limit_next_available";
|
|
5655
|
-
}
|
|
5656
|
-
}
|
|
5657
|
-
getPrefixColumnNames() {
|
|
5658
|
-
return this.prefixes.map((p) => p.name);
|
|
5659
|
-
}
|
|
5660
|
-
matchesPrefixes(record) {
|
|
5661
|
-
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5662
|
-
if (record[key] !== value) {
|
|
5663
|
-
return false;
|
|
5664
|
-
}
|
|
5665
|
-
}
|
|
5666
|
-
return true;
|
|
5667
|
-
}
|
|
5668
|
-
getPrefixKeyValues() {
|
|
5669
|
-
return this.prefixes.map((p) => this.prefixValues[p.name]);
|
|
5670
|
-
}
|
|
5671
|
-
async getExecutionDb() {
|
|
5672
|
-
if (this.executionDb)
|
|
5673
|
-
return this.executionDb;
|
|
5674
|
-
await this.setupDatabase();
|
|
5675
|
-
return this.executionDb;
|
|
5676
|
-
}
|
|
5677
|
-
async getNextAvailableDb() {
|
|
5678
|
-
if (this.nextAvailableDb)
|
|
5679
|
-
return this.nextAvailableDb;
|
|
5680
|
-
await this.setupDatabase();
|
|
5681
|
-
return this.nextAvailableDb;
|
|
5682
|
-
}
|
|
5683
|
-
async setupDatabase() {
|
|
5684
|
-
const prefixColumnNames = this.getPrefixColumnNames();
|
|
5685
|
-
const buildKeyPath = (basePath) => {
|
|
5686
|
-
return [...prefixColumnNames, ...basePath];
|
|
5687
|
-
};
|
|
5688
|
-
const executionIndexes = [
|
|
5689
|
-
{
|
|
5690
|
-
name: "queue_executed_at",
|
|
5691
|
-
keyPath: buildKeyPath(["queue_name", "executed_at"]),
|
|
5692
|
-
options: { unique: false }
|
|
5693
|
-
}
|
|
5694
|
-
];
|
|
5695
|
-
this.executionDb = await ensureIndexedDbTable(this.executionTableName, "id", executionIndexes, this.migrationOptions);
|
|
5696
|
-
const nextAvailableIndexes = [
|
|
5697
|
-
{
|
|
5698
|
-
name: "queue_name",
|
|
5699
|
-
keyPath: buildKeyPath(["queue_name"]),
|
|
5700
|
-
options: { unique: true }
|
|
5701
|
-
}
|
|
5702
|
-
];
|
|
5703
|
-
this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
|
|
5704
|
-
}
|
|
5705
|
-
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
5706
|
-
const nextIso = await this.getNextAvailableTime(queueName);
|
|
5707
|
-
if (nextIso && new Date(nextIso).getTime() > Date.now()) {
|
|
5708
|
-
return null;
|
|
5709
|
-
}
|
|
5710
|
-
const execDb = await this.getExecutionDb();
|
|
5711
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5712
|
-
const windowStartIso = new Date(Date.now() - windowMs).toISOString();
|
|
5713
|
-
const execTx = execDb.transaction(this.executionTableName, "readwrite");
|
|
5714
|
-
const execStore = execTx.objectStore(this.executionTableName);
|
|
5715
|
-
const insertedId = crypto.randomUUID();
|
|
5716
|
-
return new Promise((resolve, reject) => {
|
|
5717
|
-
let liveCount = 0;
|
|
5718
|
-
let didInsert = false;
|
|
5719
|
-
const liveRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartIso], [...prefixKeyValues, queueName, ""], true, false);
|
|
5720
|
-
const cursorReq = execStore.index("queue_executed_at").openCursor(liveRange);
|
|
5721
|
-
cursorReq.onsuccess = (event) => {
|
|
5722
|
-
const cursor = event.target.result;
|
|
5723
|
-
if (cursor) {
|
|
5724
|
-
const record2 = cursor.value;
|
|
5725
|
-
if (this.matchesPrefixes(record2)) {
|
|
5726
|
-
liveCount++;
|
|
5727
|
-
}
|
|
5728
|
-
cursor.continue();
|
|
5729
|
-
return;
|
|
5730
|
-
}
|
|
5731
|
-
if (liveCount >= maxExecutions) {
|
|
5732
|
-
execTx.abort();
|
|
5733
|
-
return;
|
|
5734
|
-
}
|
|
5735
|
-
const record = {
|
|
5736
|
-
id: insertedId,
|
|
5737
|
-
queue_name: queueName,
|
|
5738
|
-
executed_at: new Date().toISOString()
|
|
5739
|
-
};
|
|
5740
|
-
for (const [k, v] of Object.entries(this.prefixValues)) {
|
|
5741
|
-
record[k] = v;
|
|
5742
|
-
}
|
|
5743
|
-
const addReq = execStore.add(record);
|
|
5744
|
-
didInsert = true;
|
|
5745
|
-
addReq.onerror = () => {
|
|
5746
|
-
try {
|
|
5747
|
-
execTx.abort();
|
|
5748
|
-
} catch {}
|
|
5749
|
-
reject(addReq.error);
|
|
5750
|
-
};
|
|
5751
|
-
};
|
|
5752
|
-
cursorReq.onerror = () => reject(cursorReq.error);
|
|
5753
|
-
execTx.oncomplete = () => resolve(didInsert ? insertedId : null);
|
|
5754
|
-
execTx.onerror = () => reject(execTx.error);
|
|
5755
|
-
execTx.onabort = () => resolve(null);
|
|
5756
|
-
});
|
|
5757
|
-
}
|
|
5758
|
-
async releaseExecution(queueName, token) {
|
|
5759
|
-
if (token === null || token === undefined)
|
|
5760
|
-
return;
|
|
5761
|
-
const db = await this.getExecutionDb();
|
|
5762
|
-
const tx = db.transaction(this.executionTableName, "readwrite");
|
|
5763
|
-
const store = tx.objectStore(this.executionTableName);
|
|
5764
|
-
return new Promise((resolve, reject) => {
|
|
5765
|
-
const req = store.delete(token);
|
|
5766
|
-
req.onerror = () => reject(req.error);
|
|
5767
|
-
tx.oncomplete = () => resolve();
|
|
5768
|
-
tx.onerror = () => reject(tx.error);
|
|
5769
|
-
});
|
|
5770
|
-
}
|
|
5771
|
-
async recordExecution(queueName) {
|
|
5772
|
-
const db = await this.getExecutionDb();
|
|
5773
|
-
const tx = db.transaction(this.executionTableName, "readwrite");
|
|
5774
|
-
const store = tx.objectStore(this.executionTableName);
|
|
5775
|
-
const record = {
|
|
5776
|
-
id: crypto.randomUUID(),
|
|
5777
|
-
queue_name: queueName,
|
|
5778
|
-
executed_at: new Date().toISOString()
|
|
5779
|
-
};
|
|
5780
|
-
for (const [key, value] of Object.entries(this.prefixValues)) {
|
|
5781
|
-
record[key] = value;
|
|
5782
|
-
}
|
|
5783
|
-
return new Promise((resolve, reject) => {
|
|
5784
|
-
const request = store.add(record);
|
|
5785
|
-
tx.oncomplete = () => resolve();
|
|
5786
|
-
tx.onerror = () => reject(tx.error);
|
|
5787
|
-
request.onerror = () => reject(request.error);
|
|
5788
|
-
});
|
|
5789
|
-
}
|
|
5790
|
-
async getExecutionCount(queueName, windowStartTime) {
|
|
5791
|
-
const db = await this.getExecutionDb();
|
|
5792
|
-
const tx = db.transaction(this.executionTableName, "readonly");
|
|
5793
|
-
const store = tx.objectStore(this.executionTableName);
|
|
5794
|
-
const index = store.index("queue_executed_at");
|
|
5795
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5796
|
-
return new Promise((resolve, reject) => {
|
|
5797
|
-
let count = 0;
|
|
5798
|
-
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartTime], [...prefixKeyValues, queueName, ""], true, false);
|
|
5799
|
-
const request = index.openCursor(keyRange);
|
|
5800
|
-
request.onsuccess = (event) => {
|
|
5801
|
-
const cursor = event.target.result;
|
|
5802
|
-
if (cursor) {
|
|
5803
|
-
const record = cursor.value;
|
|
5804
|
-
if (this.matchesPrefixes(record)) {
|
|
5805
|
-
count++;
|
|
5806
|
-
}
|
|
5807
|
-
cursor.continue();
|
|
5808
|
-
}
|
|
5809
|
-
};
|
|
5810
|
-
tx.oncomplete = () => resolve(count);
|
|
5811
|
-
tx.onerror = () => reject(tx.error);
|
|
5812
|
-
request.onerror = () => reject(request.error);
|
|
5813
|
-
});
|
|
5814
|
-
}
|
|
5815
|
-
async getOldestExecutionAtOffset(queueName, offset) {
|
|
5816
|
-
const db = await this.getExecutionDb();
|
|
5817
|
-
const tx = db.transaction(this.executionTableName, "readonly");
|
|
5818
|
-
const store = tx.objectStore(this.executionTableName);
|
|
5819
|
-
const index = store.index("queue_executed_at");
|
|
5820
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5821
|
-
return new Promise((resolve, reject) => {
|
|
5822
|
-
const executions = [];
|
|
5823
|
-
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, ""]);
|
|
5824
|
-
const request = index.openCursor(keyRange);
|
|
5825
|
-
request.onsuccess = (event) => {
|
|
5826
|
-
const cursor = event.target.result;
|
|
5827
|
-
if (cursor) {
|
|
5828
|
-
const record = cursor.value;
|
|
5829
|
-
if (this.matchesPrefixes(record)) {
|
|
5830
|
-
executions.push(record.executed_at);
|
|
5831
|
-
}
|
|
5832
|
-
cursor.continue();
|
|
5833
|
-
}
|
|
5834
|
-
};
|
|
5835
|
-
tx.oncomplete = () => {
|
|
5836
|
-
executions.sort();
|
|
5837
|
-
resolve(executions[offset]);
|
|
5838
|
-
};
|
|
5839
|
-
tx.onerror = () => reject(tx.error);
|
|
5840
|
-
request.onerror = () => reject(request.error);
|
|
5841
|
-
});
|
|
5842
|
-
}
|
|
5843
|
-
async getNextAvailableTime(queueName) {
|
|
5844
|
-
const db = await this.getNextAvailableDb();
|
|
5845
|
-
const tx = db.transaction(this.nextAvailableTableName, "readonly");
|
|
5846
|
-
const store = tx.objectStore(this.nextAvailableTableName);
|
|
5847
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5848
|
-
const key = [...prefixKeyValues, queueName].join("_");
|
|
5849
|
-
return new Promise((resolve, reject) => {
|
|
5850
|
-
const request = store.get(key);
|
|
5851
|
-
request.onsuccess = () => {
|
|
5852
|
-
const record = request.result;
|
|
5853
|
-
if (record && this.matchesPrefixes(record)) {
|
|
5854
|
-
resolve(record.next_available_at);
|
|
5855
|
-
} else {
|
|
5856
|
-
resolve(undefined);
|
|
5857
|
-
}
|
|
5858
|
-
};
|
|
5859
|
-
request.onerror = () => reject(request.error);
|
|
5860
|
-
tx.onerror = () => reject(tx.error);
|
|
5861
|
-
});
|
|
5862
|
-
}
|
|
5863
|
-
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
5864
|
-
const db = await this.getNextAvailableDb();
|
|
5865
|
-
const tx = db.transaction(this.nextAvailableTableName, "readwrite");
|
|
5866
|
-
const store = tx.objectStore(this.nextAvailableTableName);
|
|
5867
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5868
|
-
const key = [...prefixKeyValues, queueName].join("_");
|
|
5869
|
-
const record = {
|
|
5870
|
-
queue_name: queueName,
|
|
5871
|
-
next_available_at: nextAvailableAt
|
|
5872
|
-
};
|
|
5873
|
-
for (const [k, value] of Object.entries(this.prefixValues)) {
|
|
5874
|
-
record[k] = value;
|
|
5875
|
-
}
|
|
5876
|
-
record[this.getPrefixColumnNames().concat(["queue_name"]).join("_")] = key;
|
|
5877
|
-
return new Promise((resolve, reject) => {
|
|
5878
|
-
const request = store.put(record);
|
|
5879
|
-
tx.oncomplete = () => resolve();
|
|
5880
|
-
tx.onerror = () => reject(tx.error);
|
|
5881
|
-
request.onerror = () => reject(request.error);
|
|
5882
|
-
});
|
|
5883
|
-
}
|
|
5884
|
-
async clear(queueName) {
|
|
5885
|
-
const execDb = await this.getExecutionDb();
|
|
5886
|
-
const execTx = execDb.transaction(this.executionTableName, "readwrite");
|
|
5887
|
-
const execStore = execTx.objectStore(this.executionTableName);
|
|
5888
|
-
const execIndex = execStore.index("queue_executed_at");
|
|
5889
|
-
const prefixKeyValues = this.getPrefixKeyValues();
|
|
5890
|
-
await new Promise((resolve, reject) => {
|
|
5891
|
-
const keyRange = IDBKeyRange.bound([...prefixKeyValues, queueName, ""], [...prefixKeyValues, queueName, ""]);
|
|
5892
|
-
const request = execIndex.openCursor(keyRange);
|
|
5893
|
-
request.onsuccess = (event) => {
|
|
5894
|
-
const cursor = event.target.result;
|
|
5895
|
-
if (cursor) {
|
|
5896
|
-
const record = cursor.value;
|
|
5897
|
-
if (this.matchesPrefixes(record)) {
|
|
5898
|
-
cursor.delete();
|
|
5899
|
-
}
|
|
5900
|
-
cursor.continue();
|
|
5901
|
-
}
|
|
5902
|
-
};
|
|
5903
|
-
execTx.oncomplete = () => resolve();
|
|
5904
|
-
execTx.onerror = () => reject(execTx.error);
|
|
5905
|
-
request.onerror = () => reject(request.error);
|
|
5906
|
-
});
|
|
5907
|
-
const nextDb = await this.getNextAvailableDb();
|
|
5908
|
-
const nextTx = nextDb.transaction(this.nextAvailableTableName, "readwrite");
|
|
5909
|
-
const nextStore = nextTx.objectStore(this.nextAvailableTableName);
|
|
5910
|
-
const key = [...prefixKeyValues, queueName].join("_");
|
|
5911
|
-
await new Promise((resolve, reject) => {
|
|
5912
|
-
const request = nextStore.delete(key);
|
|
5913
|
-
nextTx.oncomplete = () => resolve();
|
|
5914
|
-
nextTx.onerror = () => reject(nextTx.error);
|
|
5915
|
-
request.onerror = () => reject(request.error);
|
|
5916
|
-
});
|
|
5917
|
-
}
|
|
5918
|
-
}
|
|
5919
|
-
// src/queue-limiter/SupabaseRateLimiterStorage.ts
|
|
5920
|
-
import { createServiceToken as createServiceToken20 } from "@workglow/util";
|
|
5921
|
-
var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken20("ratelimiter.storage.supabase");
|
|
5922
|
-
|
|
5923
|
-
class SupabaseRateLimiterStorage {
|
|
5924
|
-
scope = "cluster";
|
|
5925
|
-
client;
|
|
5926
|
-
prefixes;
|
|
5927
|
-
prefixValues;
|
|
5928
|
-
executionTableName;
|
|
5929
|
-
nextAvailableTableName;
|
|
5930
|
-
constructor(client, options) {
|
|
5931
|
-
this.client = client;
|
|
5932
|
-
this.prefixes = options?.prefixes ?? [];
|
|
5933
|
-
this.prefixValues = options?.prefixValues ?? {};
|
|
5934
|
-
if (this.prefixes.length > 0) {
|
|
5935
|
-
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
5936
|
-
this.executionTableName = `rate_limit_executions_${prefixNames}`;
|
|
5937
|
-
this.nextAvailableTableName = `rate_limit_next_available_${prefixNames}`;
|
|
5938
|
-
} else {
|
|
5939
|
-
this.executionTableName = "rate_limit_executions";
|
|
5940
|
-
this.nextAvailableTableName = "rate_limit_next_available";
|
|
5941
|
-
}
|
|
5942
|
-
}
|
|
5943
|
-
getPrefixColumnType(type) {
|
|
5944
|
-
return type === "uuid" ? "UUID" : "INTEGER";
|
|
5945
|
-
}
|
|
5946
|
-
buildPrefixColumnsSql() {
|
|
5947
|
-
if (this.prefixes.length === 0)
|
|
5948
|
-
return "";
|
|
5949
|
-
return this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)} NOT NULL`).join(`,
|
|
5950
|
-
`) + `,
|
|
5951
|
-
`;
|
|
5952
|
-
}
|
|
5953
|
-
getPrefixColumnNames() {
|
|
5954
|
-
return this.prefixes.map((p) => p.name);
|
|
5955
|
-
}
|
|
5956
|
-
applyPrefixFilters(query) {
|
|
5957
|
-
let result = query;
|
|
5958
|
-
for (const prefix of this.prefixes) {
|
|
5959
|
-
result = result.eq(prefix.name, this.prefixValues[prefix.name]);
|
|
5960
|
-
}
|
|
5961
|
-
return result;
|
|
5962
|
-
}
|
|
5963
|
-
getPrefixInsertValues() {
|
|
5964
|
-
const values = {};
|
|
5965
|
-
for (const prefix of this.prefixes) {
|
|
5966
|
-
values[prefix.name] = this.prefixValues[prefix.name];
|
|
5967
|
-
}
|
|
5968
|
-
return values;
|
|
5969
|
-
}
|
|
5970
|
-
async setupDatabase() {
|
|
5971
|
-
const prefixColumnsSql = this.buildPrefixColumnsSql();
|
|
5972
|
-
const prefixColumnNames = this.getPrefixColumnNames();
|
|
5973
|
-
const prefixIndexPrefix = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
5974
|
-
const indexSuffix = prefixColumnNames.length > 0 ? "_" + prefixColumnNames.join("_") : "";
|
|
5975
|
-
const createExecTableSql = `
|
|
5976
|
-
CREATE TABLE IF NOT EXISTS ${this.executionTableName} (
|
|
5977
|
-
id SERIAL PRIMARY KEY,
|
|
5978
|
-
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
5979
|
-
executed_at TIMESTAMP WITH TIME ZONE DEFAULT NOW()
|
|
5980
|
-
)
|
|
5981
|
-
`;
|
|
5982
|
-
const { error: execTableError } = await this.client.rpc("exec_sql", {
|
|
5983
|
-
query: createExecTableSql
|
|
5984
|
-
});
|
|
5985
|
-
if (execTableError && execTableError.code !== "42P07") {
|
|
5986
|
-
throw execTableError;
|
|
5987
|
-
}
|
|
5988
|
-
const createExecIndexSql = `
|
|
5989
|
-
CREATE INDEX IF NOT EXISTS rate_limit_exec_queue${indexSuffix}_idx
|
|
5990
|
-
ON ${this.executionTableName} (${prefixIndexPrefix}queue_name, executed_at)
|
|
5991
|
-
`;
|
|
5992
|
-
await this.client.rpc("exec_sql", { query: createExecIndexSql });
|
|
5993
|
-
const primaryKeyColumns = prefixColumnNames.length > 0 ? `${prefixColumnNames.join(", ")}, queue_name` : "queue_name";
|
|
5994
|
-
const createNextTableSql = `
|
|
5995
|
-
CREATE TABLE IF NOT EXISTS ${this.nextAvailableTableName} (
|
|
5996
|
-
${prefixColumnsSql}queue_name TEXT NOT NULL,
|
|
5997
|
-
next_available_at TIMESTAMP WITH TIME ZONE,
|
|
5998
|
-
PRIMARY KEY (${primaryKeyColumns})
|
|
5999
|
-
)
|
|
6000
|
-
`;
|
|
6001
|
-
const { error: nextTableError } = await this.client.rpc("exec_sql", {
|
|
6002
|
-
query: createNextTableSql
|
|
6003
|
-
});
|
|
6004
|
-
if (nextTableError && nextTableError.code !== "42P07") {
|
|
6005
|
-
throw nextTableError;
|
|
6006
|
-
}
|
|
6007
|
-
const fnName = this.atomicReserveFunctionName();
|
|
6008
|
-
const prefixSig = this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)}`).join(", ");
|
|
6009
|
-
const prefixSigPrefix = prefixSig ? prefixSig + ", " : "";
|
|
6010
|
-
const prefixWhere = this.prefixes.length > 0 ? " AND " + this.prefixes.map((p) => `${p.name} = _${p.name}`).join(" AND ") : "";
|
|
6011
|
-
const prefixInsertCols = this.prefixes.length > 0 ? this.prefixes.map((p) => p.name).join(", ") + ", " : "";
|
|
6012
|
-
const prefixInsertVals = this.prefixes.length > 0 ? this.prefixes.map((p) => `_${p.name}`).join(", ") + ", " : "";
|
|
6013
|
-
const lockKeyParts = [`'${this.executionTableName}'`, ...this.prefixes.map((p) => `_${p.name}::text`), `_queue_name::text`];
|
|
6014
|
-
const lockKeyExpr = `hashtextextended(${lockKeyParts.join(" || '|' || ")}, 0)`;
|
|
6015
|
-
const createFnSql = `
|
|
6016
|
-
CREATE OR REPLACE FUNCTION ${fnName}(
|
|
6017
|
-
${prefixSigPrefix}_queue_name TEXT, _window_start TIMESTAMPTZ, _max_exec INT
|
|
6018
|
-
) RETURNS BIGINT AS $fn$
|
|
6019
|
-
DECLARE
|
|
6020
|
-
_count INT;
|
|
6021
|
-
_next TIMESTAMPTZ;
|
|
6022
|
-
_new_id BIGINT;
|
|
6023
|
-
BEGIN
|
|
6024
|
-
PERFORM pg_advisory_xact_lock(${lockKeyExpr});
|
|
6025
|
-
SELECT COUNT(*) INTO _count FROM ${this.executionTableName}
|
|
6026
|
-
WHERE queue_name = _queue_name AND executed_at > _window_start${prefixWhere};
|
|
6027
|
-
IF _count >= _max_exec THEN RETURN NULL; END IF;
|
|
6028
|
-
SELECT next_available_at INTO _next FROM ${this.nextAvailableTableName}
|
|
6029
|
-
WHERE queue_name = _queue_name${prefixWhere};
|
|
6030
|
-
IF _next IS NOT NULL AND _next > NOW() THEN RETURN NULL; END IF;
|
|
6031
|
-
INSERT INTO ${this.executionTableName} (${prefixInsertCols}queue_name)
|
|
6032
|
-
VALUES (${prefixInsertVals}_queue_name)
|
|
6033
|
-
RETURNING id INTO _new_id;
|
|
6034
|
-
RETURN _new_id;
|
|
6035
|
-
END;
|
|
6036
|
-
$fn$ LANGUAGE plpgsql;
|
|
6037
|
-
`;
|
|
6038
|
-
const { error: fnError } = await this.client.rpc("exec_sql", { query: createFnSql });
|
|
6039
|
-
if (fnError) {
|
|
6040
|
-
throw fnError;
|
|
6041
|
-
}
|
|
6042
|
-
}
|
|
6043
|
-
atomicReserveFunctionName() {
|
|
6044
|
-
return `${this.executionTableName}_try_reserve`.slice(0, 63);
|
|
6045
|
-
}
|
|
6046
|
-
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
6047
|
-
const args = {
|
|
6048
|
-
_queue_name: queueName,
|
|
6049
|
-
_window_start: new Date(Date.now() - windowMs).toISOString(),
|
|
6050
|
-
_max_exec: maxExecutions
|
|
6051
|
-
};
|
|
6052
|
-
for (const p of this.prefixes) {
|
|
6053
|
-
args[`_${p.name}`] = this.prefixValues[p.name];
|
|
6054
|
-
}
|
|
6055
|
-
const { data, error } = await this.client.rpc(this.atomicReserveFunctionName(), args);
|
|
6056
|
-
if (error)
|
|
6057
|
-
throw error;
|
|
6058
|
-
if (data === null || data === undefined)
|
|
6059
|
-
return null;
|
|
6060
|
-
if (Array.isArray(data)) {
|
|
6061
|
-
if (data.length === 0)
|
|
6062
|
-
return null;
|
|
6063
|
-
const first = Object.values(data[0])[0];
|
|
6064
|
-
return first ?? null;
|
|
6065
|
-
}
|
|
6066
|
-
return data;
|
|
6067
|
-
}
|
|
6068
|
-
async releaseExecution(queueName, token) {
|
|
6069
|
-
if (token === null || token === undefined)
|
|
6070
|
-
return;
|
|
6071
|
-
let del = this.client.from(this.executionTableName).delete().eq("id", token).eq("queue_name", queueName);
|
|
6072
|
-
del = this.applyPrefixFilters(del);
|
|
6073
|
-
const { error: delError } = await del;
|
|
6074
|
-
if (delError)
|
|
6075
|
-
throw delError;
|
|
6076
|
-
}
|
|
6077
|
-
async recordExecution(queueName) {
|
|
6078
|
-
const prefixInsertValues = this.getPrefixInsertValues();
|
|
6079
|
-
const { error } = await this.client.from(this.executionTableName).insert({
|
|
6080
|
-
...prefixInsertValues,
|
|
6081
|
-
queue_name: queueName
|
|
6082
|
-
});
|
|
6083
|
-
if (error)
|
|
6084
|
-
throw error;
|
|
6085
|
-
}
|
|
6086
|
-
async getExecutionCount(queueName, windowStartTime) {
|
|
6087
|
-
let query = this.client.from(this.executionTableName).select("*", { count: "exact", head: true }).eq("queue_name", queueName).gt("executed_at", windowStartTime);
|
|
6088
|
-
query = this.applyPrefixFilters(query);
|
|
6089
|
-
const { count, error } = await query;
|
|
6090
|
-
if (error)
|
|
6091
|
-
throw error;
|
|
6092
|
-
return count ?? 0;
|
|
6093
|
-
}
|
|
6094
|
-
async getOldestExecutionAtOffset(queueName, offset) {
|
|
6095
|
-
let query = this.client.from(this.executionTableName).select("executed_at").eq("queue_name", queueName);
|
|
6096
|
-
query = this.applyPrefixFilters(query);
|
|
6097
|
-
const { data, error } = await query.order("executed_at", { ascending: true }).range(offset, offset);
|
|
6098
|
-
if (error)
|
|
6099
|
-
throw error;
|
|
6100
|
-
if (!data || data.length === 0)
|
|
6101
|
-
return;
|
|
6102
|
-
return new Date(data[0].executed_at).toISOString();
|
|
6103
|
-
}
|
|
6104
|
-
async getNextAvailableTime(queueName) {
|
|
6105
|
-
let query = this.client.from(this.nextAvailableTableName).select("next_available_at").eq("queue_name", queueName);
|
|
6106
|
-
query = this.applyPrefixFilters(query);
|
|
6107
|
-
const { data, error } = await query.single();
|
|
6108
|
-
if (error) {
|
|
6109
|
-
if (error.code === "PGRST116")
|
|
6110
|
-
return;
|
|
6111
|
-
throw error;
|
|
6112
|
-
}
|
|
6113
|
-
if (!data?.next_available_at)
|
|
6114
|
-
return;
|
|
6115
|
-
return new Date(data.next_available_at).toISOString();
|
|
6116
|
-
}
|
|
6117
|
-
async setNextAvailableTime(queueName, nextAvailableAt) {
|
|
6118
|
-
const prefixInsertValues = this.getPrefixInsertValues();
|
|
6119
|
-
const { error } = await this.client.from(this.nextAvailableTableName).upsert({
|
|
6120
|
-
...prefixInsertValues,
|
|
6121
|
-
queue_name: queueName,
|
|
6122
|
-
next_available_at: nextAvailableAt
|
|
6123
|
-
}, {
|
|
6124
|
-
onConflict: this.prefixes.length > 0 ? `${this.getPrefixColumnNames().join(",")},queue_name` : "queue_name"
|
|
6125
|
-
});
|
|
6126
|
-
if (error)
|
|
6127
|
-
throw error;
|
|
6128
|
-
}
|
|
6129
|
-
async clear(queueName) {
|
|
6130
|
-
let execQuery = this.client.from(this.executionTableName).delete().eq("queue_name", queueName);
|
|
6131
|
-
execQuery = this.applyPrefixFilters(execQuery);
|
|
6132
|
-
const { error: execError } = await execQuery;
|
|
6133
|
-
if (execError)
|
|
6134
|
-
throw execError;
|
|
6135
|
-
let nextQuery = this.client.from(this.nextAvailableTableName).delete().eq("queue_name", queueName);
|
|
6136
|
-
nextQuery = this.applyPrefixFilters(nextQuery);
|
|
6137
|
-
const { error: nextError } = await nextQuery;
|
|
6138
|
-
if (nextError)
|
|
6139
|
-
throw nextError;
|
|
6140
|
-
}
|
|
6141
|
-
}
|
|
6142
|
-
// src/vector/IndexedDbVectorStorage.ts
|
|
6143
|
-
import { createServiceToken as createServiceToken21 } from "@workglow/util";
|
|
6144
|
-
import { cosineSimilarity as cosineSimilarity2 } from "@workglow/util/schema";
|
|
6145
|
-
var IDB_VECTOR_REPOSITORY = createServiceToken21("storage.vectorRepository.indexedDb");
|
|
6146
|
-
function matchesFilter2(metadata, filter) {
|
|
6147
|
-
for (const [key, value] of Object.entries(filter)) {
|
|
6148
|
-
if (metadata[key] !== value) {
|
|
6149
|
-
return false;
|
|
6150
|
-
}
|
|
6151
|
-
}
|
|
6152
|
-
return true;
|
|
6153
|
-
}
|
|
6154
|
-
function textRelevance2(text, query) {
|
|
6155
|
-
const textLower = text.toLowerCase();
|
|
6156
|
-
const queryLower = query.toLowerCase();
|
|
6157
|
-
const queryWords = queryLower.split(/\s+/).filter((w) => w.length > 0);
|
|
6158
|
-
if (queryWords.length === 0) {
|
|
6159
|
-
return 0;
|
|
6160
|
-
}
|
|
6161
|
-
let matches = 0;
|
|
6162
|
-
for (const word of queryWords) {
|
|
6163
|
-
if (textLower.includes(word)) {
|
|
6164
|
-
matches++;
|
|
6165
|
-
}
|
|
6166
|
-
}
|
|
6167
|
-
return matches / queryWords.length;
|
|
6168
|
-
}
|
|
6169
|
-
|
|
6170
|
-
class IndexedDbVectorStorage extends IndexedDbTabularStorage {
|
|
6171
|
-
vectorDimensions;
|
|
6172
|
-
vectorPropertyName;
|
|
6173
|
-
metadataPropertyName;
|
|
6174
|
-
constructor(table = "vectors", schema, primaryKeyNames, indexes = [], dimensions, _vectorCtor = Float32Array, migrationOptions = {}, clientProvidedKeys = "if-missing") {
|
|
6175
|
-
super(table, schema, primaryKeyNames, indexes, migrationOptions, clientProvidedKeys);
|
|
6176
|
-
this.vectorDimensions = dimensions;
|
|
6177
|
-
const vectorProp = getVectorProperty(schema);
|
|
6178
|
-
if (!vectorProp) {
|
|
6179
|
-
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
6180
|
-
}
|
|
6181
|
-
this.vectorPropertyName = vectorProp;
|
|
6182
|
-
this.metadataPropertyName = getMetadataProperty(schema);
|
|
6183
|
-
}
|
|
6184
|
-
getVectorDimensions() {
|
|
6185
|
-
return this.vectorDimensions;
|
|
6186
|
-
}
|
|
6187
|
-
async similaritySearch(query, options = {}) {
|
|
6188
|
-
const { topK = 10, filter, scoreThreshold = 0 } = options;
|
|
6189
|
-
const results = [];
|
|
6190
|
-
const allEntities = await this.getAll() || [];
|
|
6191
|
-
for (const entity of allEntities) {
|
|
6192
|
-
const vector = entity[this.vectorPropertyName];
|
|
6193
|
-
const metadata = this.metadataPropertyName ? entity[this.metadataPropertyName] : {};
|
|
6194
|
-
if (filter && !matchesFilter2(metadata, filter)) {
|
|
6195
|
-
continue;
|
|
6196
|
-
}
|
|
6197
|
-
const score = cosineSimilarity2(query, vector);
|
|
6198
|
-
if (score < scoreThreshold) {
|
|
6199
|
-
continue;
|
|
6200
|
-
}
|
|
6201
|
-
results.push({
|
|
6202
|
-
...entity,
|
|
6203
|
-
score
|
|
6204
|
-
});
|
|
6205
|
-
}
|
|
6206
|
-
results.sort((a, b) => b.score - a.score);
|
|
6207
|
-
const topResults = results.slice(0, topK);
|
|
6208
|
-
return topResults;
|
|
6209
|
-
}
|
|
6210
|
-
async hybridSearch(query, options) {
|
|
6211
|
-
const { topK = 10, filter, scoreThreshold = 0, textQuery, vectorWeight = 0.7 } = options;
|
|
6212
|
-
if (!textQuery || textQuery.trim().length === 0) {
|
|
6213
|
-
return this.similaritySearch(query, { topK, filter, scoreThreshold });
|
|
6214
|
-
}
|
|
6215
|
-
const results = [];
|
|
6216
|
-
const allEntities = await this.getAll() || [];
|
|
6217
|
-
for (const entity of allEntities) {
|
|
6218
|
-
const vector = entity[this.vectorPropertyName];
|
|
6219
|
-
const metadata = this.metadataPropertyName ? entity[this.metadataPropertyName] : {};
|
|
6220
|
-
if (filter && !matchesFilter2(metadata, filter)) {
|
|
6221
|
-
continue;
|
|
6222
|
-
}
|
|
6223
|
-
const vectorScore = cosineSimilarity2(query, vector);
|
|
6224
|
-
const metadataText = Object.values(metadata).join(" ").toLowerCase();
|
|
6225
|
-
const textScore = textRelevance2(metadataText, textQuery);
|
|
6226
|
-
const combinedScore = vectorWeight * vectorScore + (1 - vectorWeight) * textScore;
|
|
6227
|
-
if (combinedScore < scoreThreshold) {
|
|
6228
|
-
continue;
|
|
6229
|
-
}
|
|
6230
|
-
results.push({
|
|
6231
|
-
...entity,
|
|
6232
|
-
score: combinedScore
|
|
6233
|
-
});
|
|
6234
|
-
}
|
|
6235
|
-
results.sort((a, b) => b.score - a.score);
|
|
6236
|
-
const topResults = results.slice(0, topK);
|
|
6237
|
-
return topResults;
|
|
6238
|
-
}
|
|
6239
|
-
}
|
|
6240
2467
|
export {
|
|
6241
|
-
traced,
|
|
6242
2468
|
registerTabularRepository,
|
|
6243
2469
|
pickCoveringIndex,
|
|
6244
2470
|
isSearchCondition,
|
|
@@ -6246,18 +2472,11 @@ export {
|
|
|
6246
2472
|
getTabularRepository,
|
|
6247
2473
|
getMetadataProperty,
|
|
6248
2474
|
getGlobalTabularRepositories,
|
|
6249
|
-
ensureIndexedDbTable,
|
|
6250
|
-
dropIndexedDbTable,
|
|
6251
2475
|
TelemetryVectorStorage,
|
|
6252
2476
|
TelemetryTabularStorage,
|
|
6253
|
-
TelemetryQueueStorage,
|
|
6254
2477
|
TelemetryKvStorage,
|
|
6255
2478
|
TABULAR_REPOSITORY,
|
|
6256
2479
|
TABULAR_REPOSITORIES,
|
|
6257
|
-
SupabaseTabularStorage,
|
|
6258
|
-
SupabaseRateLimiterStorage,
|
|
6259
|
-
SupabaseQueueStorage,
|
|
6260
|
-
SupabaseKvStorage,
|
|
6261
2480
|
StorageValidationError,
|
|
6262
2481
|
StorageUnsupportedError,
|
|
6263
2482
|
StorageInvalidLimitError,
|
|
@@ -6265,13 +2484,7 @@ export {
|
|
|
6265
2484
|
StorageError,
|
|
6266
2485
|
StorageEmptyCriteriaError,
|
|
6267
2486
|
SharedInMemoryTabularStorage,
|
|
6268
|
-
SUPABASE_TABULAR_REPOSITORY,
|
|
6269
|
-
SUPABASE_RATE_LIMITER_STORAGE,
|
|
6270
|
-
SUPABASE_QUEUE_STORAGE,
|
|
6271
|
-
SUPABASE_KV_REPOSITORY,
|
|
6272
2487
|
SHARED_IN_MEMORY_TABULAR_REPOSITORY,
|
|
6273
|
-
RATE_LIMITER_STORAGE,
|
|
6274
|
-
QUEUE_STORAGE,
|
|
6275
2488
|
PollingSubscriptionManager,
|
|
6276
2489
|
MEMORY_TABULAR_REPOSITORY,
|
|
6277
2490
|
MEMORY_KV_REPOSITORY,
|
|
@@ -6279,24 +2492,9 @@ export {
|
|
|
6279
2492
|
KvViaTabularStorage,
|
|
6280
2493
|
KvStorage,
|
|
6281
2494
|
KV_REPOSITORY,
|
|
6282
|
-
JobStatus,
|
|
6283
|
-
IndexedDbVectorStorage,
|
|
6284
|
-
IndexedDbTabularStorage,
|
|
6285
|
-
IndexedDbRateLimiterStorage,
|
|
6286
|
-
IndexedDbQueueStorage,
|
|
6287
|
-
IndexedDbKvStorage,
|
|
6288
2495
|
InMemoryVectorStorage,
|
|
6289
2496
|
InMemoryTabularStorage,
|
|
6290
|
-
InMemoryRateLimiterStorage,
|
|
6291
|
-
InMemoryQueueStorage,
|
|
6292
2497
|
InMemoryKvStorage,
|
|
6293
|
-
IN_MEMORY_RATE_LIMITER_STORAGE,
|
|
6294
|
-
IN_MEMORY_QUEUE_STORAGE,
|
|
6295
|
-
INDEXED_DB_RATE_LIMITER_STORAGE,
|
|
6296
|
-
INDEXED_DB_QUEUE_STORAGE,
|
|
6297
|
-
IDB_VECTOR_REPOSITORY,
|
|
6298
|
-
IDB_TABULAR_REPOSITORY,
|
|
6299
|
-
IDB_KV_REPOSITORY,
|
|
6300
2498
|
HybridSubscriptionManager,
|
|
6301
2499
|
HuggingFaceTabularStorage,
|
|
6302
2500
|
HF_TABULAR_REPOSITORY,
|
|
@@ -6306,7 +2504,8 @@ export {
|
|
|
6306
2504
|
CoveringIndexMissingError,
|
|
6307
2505
|
CachedTabularStorage,
|
|
6308
2506
|
CACHED_TABULAR_REPOSITORY,
|
|
6309
|
-
BaseTabularStorage
|
|
2507
|
+
BaseTabularStorage,
|
|
2508
|
+
BaseSqlTabularStorage
|
|
6310
2509
|
};
|
|
6311
2510
|
|
|
6312
|
-
//# debugId=
|
|
2511
|
+
//# debugId=23715A15DDE7420E64756E2164756E21
|