@workglow/storage 0.2.25 → 0.2.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +512 -9
- package/dist/browser.js.map +23 -21
- package/dist/bun.js +1087 -21
- package/dist/bun.js.map +30 -27
- package/dist/common-server.d.ts +1 -0
- package/dist/common-server.d.ts.map +1 -1
- package/dist/common.d.ts +3 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/node.js +1087 -21
- package/dist/node.js.map +30 -27
- package/dist/queue/IQueueStorage.d.ts +25 -2
- package/dist/queue/IQueueStorage.d.ts.map +1 -1
- package/dist/queue/InMemoryQueueStorage.d.ts +1 -0
- package/dist/queue/InMemoryQueueStorage.d.ts.map +1 -1
- package/dist/queue/IndexedDbQueueStorage.d.ts +1 -0
- package/dist/queue/IndexedDbQueueStorage.d.ts.map +1 -1
- package/dist/queue/PostgresQueueStorage.d.ts +24 -3
- package/dist/queue/PostgresQueueStorage.d.ts.map +1 -1
- package/dist/queue/SqliteQueueStorage.d.ts +8 -0
- package/dist/queue/SqliteQueueStorage.d.ts.map +1 -1
- package/dist/queue/SupabaseQueueStorage.d.ts +1 -0
- package/dist/queue/SupabaseQueueStorage.d.ts.map +1 -1
- package/dist/queue/TelemetryQueueStorage.d.ts +2 -1
- package/dist/queue/TelemetryQueueStorage.d.ts.map +1 -1
- package/dist/queue-limiter/IRateLimiterStorage.d.ts +46 -0
- package/dist/queue-limiter/IRateLimiterStorage.d.ts.map +1 -1
- package/dist/queue-limiter/InMemoryRateLimiterStorage.d.ts +12 -1
- package/dist/queue-limiter/InMemoryRateLimiterStorage.d.ts.map +1 -1
- package/dist/queue-limiter/IndexedDbRateLimiterStorage.d.ts +28 -1
- package/dist/queue-limiter/IndexedDbRateLimiterStorage.d.ts.map +1 -1
- package/dist/queue-limiter/PostgresRateLimiterStorage.d.ts +4 -1
- package/dist/queue-limiter/PostgresRateLimiterStorage.d.ts.map +1 -1
- package/dist/queue-limiter/SqliteRateLimiterStorage.d.ts +10 -1
- package/dist/queue-limiter/SqliteRateLimiterStorage.d.ts.map +1 -1
- package/dist/queue-limiter/SupabaseRateLimiterStorage.d.ts +6 -1
- package/dist/queue-limiter/SupabaseRateLimiterStorage.d.ts.map +1 -1
- package/dist/tabular/BaseTabularStorage.d.ts +15 -1
- package/dist/tabular/BaseTabularStorage.d.ts.map +1 -1
- package/dist/tabular/CachedTabularStorage.d.ts +2 -1
- package/dist/tabular/CachedTabularStorage.d.ts.map +1 -1
- package/dist/tabular/CoveringIndexMissingError.d.ts +14 -0
- package/dist/tabular/CoveringIndexMissingError.d.ts.map +1 -0
- package/dist/tabular/FsFolderTabularStorage.d.ts +6 -1
- package/dist/tabular/FsFolderTabularStorage.d.ts.map +1 -1
- package/dist/tabular/ITabularStorage.d.ts +19 -1
- package/dist/tabular/ITabularStorage.d.ts.map +1 -1
- package/dist/tabular/InMemoryTabularStorage.d.ts +7 -1
- package/dist/tabular/InMemoryTabularStorage.d.ts.map +1 -1
- package/dist/tabular/IndexedDbTabularStorage.d.ts +10 -1
- package/dist/tabular/IndexedDbTabularStorage.d.ts.map +1 -1
- package/dist/tabular/PostgresTabularStorage.d.ts +11 -1
- package/dist/tabular/PostgresTabularStorage.d.ts.map +1 -1
- package/dist/tabular/SharedInMemoryTabularStorage.d.ts +2 -1
- package/dist/tabular/SharedInMemoryTabularStorage.d.ts.map +1 -1
- package/dist/tabular/SqliteTabularStorage.d.ts +11 -1
- package/dist/tabular/SqliteTabularStorage.d.ts.map +1 -1
- package/dist/tabular/SupabaseTabularStorage.d.ts +2 -1
- package/dist/tabular/SupabaseTabularStorage.d.ts.map +1 -1
- package/dist/tabular/TelemetryTabularStorage.d.ts +2 -1
- package/dist/tabular/TelemetryTabularStorage.d.ts.map +1 -1
- package/dist/tabular/coveringIndexPicker.d.ts +37 -0
- package/dist/tabular/coveringIndexPicker.d.ts.map +1 -0
- package/dist/vector/IVectorStorage.d.ts +3 -1
- package/dist/vector/IVectorStorage.d.ts.map +1 -1
- package/package.json +3 -3
package/dist/bun.js
CHANGED
|
@@ -48,6 +48,21 @@ class StorageUnsupportedError extends StorageError {
|
|
|
48
48
|
}
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
+
// src/tabular/CoveringIndexMissingError.ts
|
|
52
|
+
class CoveringIndexMissingError extends StorageError {
|
|
53
|
+
static type = "CoveringIndexMissingError";
|
|
54
|
+
table;
|
|
55
|
+
requiredColumns;
|
|
56
|
+
registeredIndexes;
|
|
57
|
+
constructor(table, requiredColumns, registeredIndexes) {
|
|
58
|
+
const indexList = registeredIndexes.map((cols) => `[${cols.join(", ")}]`).join(", ");
|
|
59
|
+
super(`No covering index for table "${table}". ` + `Required columns: [${requiredColumns.join(", ")}]. ` + `Registered indexes: ${indexList || "(none)"}.`);
|
|
60
|
+
this.table = table;
|
|
61
|
+
this.requiredColumns = requiredColumns;
|
|
62
|
+
this.registeredIndexes = registeredIndexes;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
51
66
|
// src/tabular/BaseTabularStorage.ts
|
|
52
67
|
var TABULAR_REPOSITORY = createServiceToken("storage.tabularRepository");
|
|
53
68
|
|
|
@@ -171,6 +186,15 @@ class BaseTabularStorage {
|
|
|
171
186
|
const entities = await this.query(criteria);
|
|
172
187
|
return entities?.length ?? 0;
|
|
173
188
|
}
|
|
189
|
+
queryIndex(criteria, options) {
|
|
190
|
+
this.validateSelect(options);
|
|
191
|
+
const requiredColumns = [
|
|
192
|
+
...Object.keys(criteria),
|
|
193
|
+
...(options.orderBy ?? []).map((o) => String(o.column)),
|
|
194
|
+
...options.select.map(String)
|
|
195
|
+
];
|
|
196
|
+
throw new CoveringIndexMissingError(this.constructor.name, requiredColumns, []);
|
|
197
|
+
}
|
|
174
198
|
async* records(pageSize = 100) {
|
|
175
199
|
if (pageSize <= 0) {
|
|
176
200
|
throw new RangeError(`pageSize must be greater than 0, got ${pageSize}`);
|
|
@@ -266,6 +290,18 @@ class BaseTabularStorage {
|
|
|
266
290
|
}
|
|
267
291
|
}
|
|
268
292
|
}
|
|
293
|
+
validateSelect(options) {
|
|
294
|
+
if (!options.select || options.select.length === 0) {
|
|
295
|
+
throw new StorageValidationError("queryIndex requires a non-empty select array");
|
|
296
|
+
}
|
|
297
|
+
const schemaProps = Object.keys(this.schema.properties);
|
|
298
|
+
for (const col of options.select) {
|
|
299
|
+
const colStr = String(col);
|
|
300
|
+
if (!schemaProps.includes(colStr)) {
|
|
301
|
+
throw new StorageValidationError(`queryIndex select column "${colStr}" is not in schema`);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
269
305
|
primaryKeyColumns() {
|
|
270
306
|
const columns = [];
|
|
271
307
|
for (const key of Object.keys(this.primaryKeySchema.properties)) {
|
|
@@ -386,6 +422,61 @@ import { createServiceToken as createServiceToken3, getLogger } from "@workglow/
|
|
|
386
422
|
|
|
387
423
|
// src/tabular/InMemoryTabularStorage.ts
|
|
388
424
|
import { createServiceToken as createServiceToken2, makeFingerprint as makeFingerprint2, uuid4 } from "@workglow/util";
|
|
425
|
+
|
|
426
|
+
// src/tabular/coveringIndexPicker.ts
|
|
427
|
+
function pickCoveringIndex(input) {
|
|
428
|
+
const { table, indexes, criteriaColumns, orderByColumns, selectColumns, primaryKeyColumns } = input;
|
|
429
|
+
const required = uniqueColumns([
|
|
430
|
+
...criteriaColumns,
|
|
431
|
+
...orderByColumns.map((o) => o.column),
|
|
432
|
+
...selectColumns
|
|
433
|
+
]);
|
|
434
|
+
const pkSet = new Set(primaryKeyColumns);
|
|
435
|
+
for (const idx of indexes) {
|
|
436
|
+
const fit = tryFitIndex(idx, criteriaColumns, orderByColumns, selectColumns, pkSet);
|
|
437
|
+
if (fit !== undefined) {
|
|
438
|
+
return { name: idx.name, keyPath: idx.keyPath, reverseDirection: fit.reverseDirection };
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
throw new CoveringIndexMissingError(table, required, indexes.map((i) => i.keyPath));
|
|
442
|
+
}
|
|
443
|
+
function tryFitIndex(index, criteria, orderBy, select, pkSet) {
|
|
444
|
+
const keyPath = index.keyPath;
|
|
445
|
+
const criteriaSet = new Set(criteria);
|
|
446
|
+
if (criteria.length > keyPath.length)
|
|
447
|
+
return;
|
|
448
|
+
for (let i = 0;i < criteria.length; i++) {
|
|
449
|
+
if (!criteriaSet.has(keyPath[i]))
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
let reverseDirection = false;
|
|
453
|
+
if (orderBy.length > 0) {
|
|
454
|
+
const start = criteria.length;
|
|
455
|
+
if (start + orderBy.length > keyPath.length)
|
|
456
|
+
return;
|
|
457
|
+
for (let i = 0;i < orderBy.length; i++) {
|
|
458
|
+
if (keyPath[start + i] !== orderBy[i].column)
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
461
|
+
const allDesc = orderBy.every((o) => o.direction === "DESC");
|
|
462
|
+
const allAsc = orderBy.every((o) => o.direction === "ASC");
|
|
463
|
+
if (allDesc)
|
|
464
|
+
reverseDirection = true;
|
|
465
|
+
else if (!allAsc)
|
|
466
|
+
return;
|
|
467
|
+
}
|
|
468
|
+
const keyPathSet = new Set(keyPath);
|
|
469
|
+
for (const col of select) {
|
|
470
|
+
if (!keyPathSet.has(col) && !pkSet.has(col))
|
|
471
|
+
return;
|
|
472
|
+
}
|
|
473
|
+
return { reverseDirection };
|
|
474
|
+
}
|
|
475
|
+
function uniqueColumns(cols) {
|
|
476
|
+
return Array.from(new Set(cols));
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// src/tabular/InMemoryTabularStorage.ts
|
|
389
480
|
var MEMORY_TABULAR_REPOSITORY = createServiceToken2("storage.tabularRepository.inMemory");
|
|
390
481
|
|
|
391
482
|
class InMemoryTabularStorage extends BaseTabularStorage {
|
|
@@ -628,6 +719,38 @@ class InMemoryTabularStorage extends BaseTabularStorage {
|
|
|
628
719
|
this.events.emit("query", criteria, result);
|
|
629
720
|
return result;
|
|
630
721
|
}
|
|
722
|
+
async queryIndex(criteria, options) {
|
|
723
|
+
this.validateSelect(options);
|
|
724
|
+
this.validateQueryParams(criteria, options);
|
|
725
|
+
const registered = this.indexes.map((cols, i) => ({
|
|
726
|
+
name: `idx_${i}`,
|
|
727
|
+
keyPath: cols
|
|
728
|
+
}));
|
|
729
|
+
pickCoveringIndex({
|
|
730
|
+
table: "InMemoryTabularStorage",
|
|
731
|
+
indexes: registered,
|
|
732
|
+
criteriaColumns: Object.keys(criteria),
|
|
733
|
+
orderByColumns: (options.orderBy ?? []).map((o) => ({
|
|
734
|
+
column: String(o.column),
|
|
735
|
+
direction: o.direction
|
|
736
|
+
})),
|
|
737
|
+
selectColumns: options.select.map(String),
|
|
738
|
+
primaryKeyColumns: this.primaryKeyNames.map(String)
|
|
739
|
+
});
|
|
740
|
+
const full = await this.query(criteria, {
|
|
741
|
+
orderBy: options.orderBy,
|
|
742
|
+
limit: options.limit,
|
|
743
|
+
offset: options.offset
|
|
744
|
+
});
|
|
745
|
+
if (!full)
|
|
746
|
+
return [];
|
|
747
|
+
return full.map((row) => {
|
|
748
|
+
const out = {};
|
|
749
|
+
for (const k of options.select)
|
|
750
|
+
out[k] = row[k];
|
|
751
|
+
return out;
|
|
752
|
+
});
|
|
753
|
+
}
|
|
631
754
|
subscribeToChanges(callback, options) {
|
|
632
755
|
const handlePut = (entity) => {
|
|
633
756
|
callback({ type: this._lastPutWasInsert ? "INSERT" : "UPDATE", new: entity });
|
|
@@ -770,6 +893,10 @@ class CachedTabularStorage extends BaseTabularStorage {
|
|
|
770
893
|
await this.initializeCache();
|
|
771
894
|
return await this.cache.query(criteria, options);
|
|
772
895
|
}
|
|
896
|
+
async queryIndex(criteria, options) {
|
|
897
|
+
await this.initializeCache();
|
|
898
|
+
return await this.cache.queryIndex(criteria, options);
|
|
899
|
+
}
|
|
773
900
|
async deleteSearch(criteria) {
|
|
774
901
|
await this.initializeCache();
|
|
775
902
|
await this.durable.deleteSearch(criteria);
|
|
@@ -1210,6 +1337,9 @@ class TelemetryTabularStorage {
|
|
|
1210
1337
|
query(criteria, options) {
|
|
1211
1338
|
return traced("workglow.storage.tabular.query", this.storageName, () => this.inner.query(criteria, options));
|
|
1212
1339
|
}
|
|
1340
|
+
queryIndex(criteria, options) {
|
|
1341
|
+
return traced("workglow.storage.tabular.queryIndex", this.storageName, () => this.inner.queryIndex(criteria, options));
|
|
1342
|
+
}
|
|
1213
1343
|
records(pageSize) {
|
|
1214
1344
|
return this.inner.records(pageSize);
|
|
1215
1345
|
}
|
|
@@ -1447,6 +1577,7 @@ var IN_MEMORY_QUEUE_STORAGE = createServiceToken9("jobqueue.storage.inMemory");
|
|
|
1447
1577
|
|
|
1448
1578
|
class InMemoryQueueStorage {
|
|
1449
1579
|
queueName;
|
|
1580
|
+
scope = "process";
|
|
1450
1581
|
prefixValues;
|
|
1451
1582
|
events = new EventEmitter3;
|
|
1452
1583
|
constructor(queueName, options) {
|
|
@@ -1656,6 +1787,9 @@ class TelemetryQueueStorage {
|
|
|
1656
1787
|
this.storageName = storageName;
|
|
1657
1788
|
this.inner = inner;
|
|
1658
1789
|
}
|
|
1790
|
+
get scope() {
|
|
1791
|
+
return this.inner.scope;
|
|
1792
|
+
}
|
|
1659
1793
|
add(job) {
|
|
1660
1794
|
return traced("workglow.storage.queue.add", this.storageName, () => this.inner.add(job));
|
|
1661
1795
|
}
|
|
@@ -1706,13 +1840,15 @@ class TelemetryQueueStorage {
|
|
|
1706
1840
|
}
|
|
1707
1841
|
}
|
|
1708
1842
|
// src/queue-limiter/InMemoryRateLimiterStorage.ts
|
|
1709
|
-
import { createServiceToken as createServiceToken10, sleep as sleep2 } from "@workglow/util";
|
|
1843
|
+
import { createServiceToken as createServiceToken10, sleep as sleep2, uuid4 as uuid43 } from "@workglow/util";
|
|
1710
1844
|
var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken10("ratelimiter.storage.inMemory");
|
|
1711
1845
|
|
|
1712
1846
|
class InMemoryRateLimiterStorage {
|
|
1847
|
+
scope = "process";
|
|
1713
1848
|
prefixValues;
|
|
1714
1849
|
executions = new Map;
|
|
1715
1850
|
nextAvailableTimes = new Map;
|
|
1851
|
+
reserveChains = new Map;
|
|
1716
1852
|
constructor(options) {
|
|
1717
1853
|
this.prefixValues = options?.prefixValues ?? {};
|
|
1718
1854
|
}
|
|
@@ -1721,11 +1857,67 @@ class InMemoryRateLimiterStorage {
|
|
|
1721
1857
|
return prefixPart ? `${prefixPart}|${queueName}` : queueName;
|
|
1722
1858
|
}
|
|
1723
1859
|
async setupDatabase() {}
|
|
1860
|
+
async withKeyLock(key, fn) {
|
|
1861
|
+
const previous = this.reserveChains.get(key) ?? Promise.resolve();
|
|
1862
|
+
let release;
|
|
1863
|
+
const next = new Promise((resolve) => {
|
|
1864
|
+
release = resolve;
|
|
1865
|
+
});
|
|
1866
|
+
this.reserveChains.set(key, next);
|
|
1867
|
+
try {
|
|
1868
|
+
await previous;
|
|
1869
|
+
return await fn();
|
|
1870
|
+
} finally {
|
|
1871
|
+
release(undefined);
|
|
1872
|
+
if (this.reserveChains.get(key) === next) {
|
|
1873
|
+
this.reserveChains.delete(key);
|
|
1874
|
+
}
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
1878
|
+
const key = this.makeKey(queueName);
|
|
1879
|
+
return this.withKeyLock(key, () => {
|
|
1880
|
+
const now = Date.now();
|
|
1881
|
+
const windowStart = new Date(now - windowMs);
|
|
1882
|
+
const executions = this.executions.get(key) ?? [];
|
|
1883
|
+
const live = executions.filter((e) => e.executedAt > windowStart);
|
|
1884
|
+
if (live.length >= maxExecutions) {
|
|
1885
|
+
if (live.length !== executions.length) {
|
|
1886
|
+
this.executions.set(key, live);
|
|
1887
|
+
}
|
|
1888
|
+
return null;
|
|
1889
|
+
}
|
|
1890
|
+
const next = this.nextAvailableTimes.get(key);
|
|
1891
|
+
if (next && next.getTime() > now) {
|
|
1892
|
+
return null;
|
|
1893
|
+
}
|
|
1894
|
+
const id = uuid43();
|
|
1895
|
+
live.push({ id, queueName, executedAt: new Date(now) });
|
|
1896
|
+
this.executions.set(key, live);
|
|
1897
|
+
return id;
|
|
1898
|
+
});
|
|
1899
|
+
}
|
|
1900
|
+
async releaseExecution(queueName, token) {
|
|
1901
|
+
if (token === null || token === undefined)
|
|
1902
|
+
return;
|
|
1903
|
+
const key = this.makeKey(queueName);
|
|
1904
|
+
await this.withKeyLock(key, () => {
|
|
1905
|
+
const executions = this.executions.get(key);
|
|
1906
|
+
if (!executions || executions.length === 0)
|
|
1907
|
+
return;
|
|
1908
|
+
const idx = executions.findIndex((e) => e.id === token);
|
|
1909
|
+
if (idx === -1)
|
|
1910
|
+
return;
|
|
1911
|
+
executions.splice(idx, 1);
|
|
1912
|
+
this.executions.set(key, executions);
|
|
1913
|
+
});
|
|
1914
|
+
}
|
|
1724
1915
|
async recordExecution(queueName) {
|
|
1725
1916
|
await sleep2(0);
|
|
1726
1917
|
const key = this.makeKey(queueName);
|
|
1727
1918
|
const executions = this.executions.get(key) ?? [];
|
|
1728
1919
|
executions.push({
|
|
1920
|
+
id: uuid43(),
|
|
1729
1921
|
queueName,
|
|
1730
1922
|
executedAt: new Date
|
|
1731
1923
|
});
|
|
@@ -2313,7 +2505,7 @@ import {
|
|
|
2313
2505
|
getLogger as getLogger3,
|
|
2314
2506
|
makeFingerprint as makeFingerprint5,
|
|
2315
2507
|
sleep as sleep3,
|
|
2316
|
-
uuid4 as
|
|
2508
|
+
uuid4 as uuid44
|
|
2317
2509
|
} from "@workglow/util";
|
|
2318
2510
|
import { mkdir, readdir, readFile, rm, writeFile } from "fs/promises";
|
|
2319
2511
|
import path from "path";
|
|
@@ -2341,7 +2533,7 @@ class FsFolderTabularStorage extends BaseTabularStorage {
|
|
|
2341
2533
|
if (strategy === "autoincrement") {
|
|
2342
2534
|
return ++this.autoIncrementCounter;
|
|
2343
2535
|
} else {
|
|
2344
|
-
return
|
|
2536
|
+
return uuid44();
|
|
2345
2537
|
}
|
|
2346
2538
|
}
|
|
2347
2539
|
async put(entity) {
|
|
@@ -2495,6 +2687,9 @@ class FsFolderTabularStorage extends BaseTabularStorage {
|
|
|
2495
2687
|
async query(_criteria, _options) {
|
|
2496
2688
|
throw new StorageUnsupportedError("query", "FsFolderTabularStorage");
|
|
2497
2689
|
}
|
|
2690
|
+
async queryIndex(_criteria, _options) {
|
|
2691
|
+
throw new StorageUnsupportedError("queryIndex", "FsFolderTabularStorage");
|
|
2692
|
+
}
|
|
2498
2693
|
async deleteSearch(_criteria) {
|
|
2499
2694
|
throw new Error("deleteSearch is not supported for FsFolderTabularStorage");
|
|
2500
2695
|
}
|
|
@@ -3238,6 +3433,49 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
3238
3433
|
this.events.emit("query", criteria, undefined);
|
|
3239
3434
|
return;
|
|
3240
3435
|
}
|
|
3436
|
+
async queryIndex(criteria, options) {
|
|
3437
|
+
this.validateSelect(options);
|
|
3438
|
+
this.validateQueryParams(criteria, options);
|
|
3439
|
+
const registered = this.indexes.map((cols2, i) => {
|
|
3440
|
+
const cs = Array.isArray(cols2) ? cols2 : [cols2];
|
|
3441
|
+
return { name: `idx_${i}`, keyPath: cs };
|
|
3442
|
+
});
|
|
3443
|
+
pickCoveringIndex({
|
|
3444
|
+
table: this.table,
|
|
3445
|
+
indexes: registered,
|
|
3446
|
+
criteriaColumns: Object.keys(criteria),
|
|
3447
|
+
orderByColumns: (options.orderBy ?? []).map((o) => ({
|
|
3448
|
+
column: String(o.column),
|
|
3449
|
+
direction: o.direction
|
|
3450
|
+
})),
|
|
3451
|
+
selectColumns: options.select.map(String),
|
|
3452
|
+
primaryKeyColumns: this.primaryKeyNames.map(String)
|
|
3453
|
+
});
|
|
3454
|
+
const cols = options.select.map((c) => `"${String(c)}"`).join(", ");
|
|
3455
|
+
let sql = `SELECT ${cols} FROM "${this.table}"`;
|
|
3456
|
+
const { whereClause, params } = this.buildDeleteSearchWhere(criteria);
|
|
3457
|
+
sql += ` WHERE ${whereClause}`;
|
|
3458
|
+
if (options.orderBy && options.orderBy.length > 0) {
|
|
3459
|
+
sql += ` ORDER BY ` + options.orderBy.map((o) => `"${String(o.column)}" ${o.direction}`).join(", ");
|
|
3460
|
+
}
|
|
3461
|
+
if (options.limit !== undefined) {
|
|
3462
|
+
sql += ` LIMIT $${params.length + 1}`;
|
|
3463
|
+
params.push(options.limit);
|
|
3464
|
+
}
|
|
3465
|
+
if (options.offset !== undefined) {
|
|
3466
|
+
sql += ` OFFSET $${params.length + 1}`;
|
|
3467
|
+
params.push(options.offset);
|
|
3468
|
+
}
|
|
3469
|
+
const db = this.db;
|
|
3470
|
+
const result = await db.query(sql, params);
|
|
3471
|
+
for (const row of result.rows) {
|
|
3472
|
+
const record = row;
|
|
3473
|
+
for (const k of Object.keys(record)) {
|
|
3474
|
+
record[k] = this.sqlToJsValue(k, record[k]);
|
|
3475
|
+
}
|
|
3476
|
+
}
|
|
3477
|
+
return result.rows;
|
|
3478
|
+
}
|
|
3241
3479
|
subscribeToChanges(callback, options) {
|
|
3242
3480
|
throw new Error("subscribeToChanges is not supported for PostgresTabularStorage");
|
|
3243
3481
|
}
|
|
@@ -3247,7 +3485,7 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
3247
3485
|
}
|
|
3248
3486
|
// src/tabular/SqliteTabularStorage.ts
|
|
3249
3487
|
import { Sqlite } from "@workglow/storage/sqlite";
|
|
3250
|
-
import { createServiceToken as createServiceToken14, uuid4 as
|
|
3488
|
+
import { createServiceToken as createServiceToken14, uuid4 as uuid45 } from "@workglow/util";
|
|
3251
3489
|
var SQLITE_TABULAR_REPOSITORY = createServiceToken14("storage.tabularRepository.sqlite");
|
|
3252
3490
|
|
|
3253
3491
|
class SqliteTabularStorage extends BaseSqlTabularStorage {
|
|
@@ -3425,7 +3663,7 @@ class SqliteTabularStorage extends BaseSqlTabularStorage {
|
|
|
3425
3663
|
}
|
|
3426
3664
|
generateKeyValue(columnName, strategy) {
|
|
3427
3665
|
if (strategy === "uuid") {
|
|
3428
|
-
return
|
|
3666
|
+
return uuid45();
|
|
3429
3667
|
}
|
|
3430
3668
|
throw new Error(`SQLite autoincrement keys are generated by the database, not client-side. Column: ${columnName}`);
|
|
3431
3669
|
}
|
|
@@ -3722,6 +3960,50 @@ class SqliteTabularStorage extends BaseSqlTabularStorage {
|
|
|
3722
3960
|
this.events.emit("query", criteria, undefined);
|
|
3723
3961
|
return;
|
|
3724
3962
|
}
|
|
3963
|
+
async queryIndex(criteria, options) {
|
|
3964
|
+
this.validateSelect(options);
|
|
3965
|
+
this.validateQueryParams(criteria, options);
|
|
3966
|
+
const registered = this.indexes.map((cols2, i) => {
|
|
3967
|
+
const cs = Array.isArray(cols2) ? cols2 : [cols2];
|
|
3968
|
+
return { name: `idx_${i}`, keyPath: cs };
|
|
3969
|
+
});
|
|
3970
|
+
pickCoveringIndex({
|
|
3971
|
+
table: this.table,
|
|
3972
|
+
indexes: registered,
|
|
3973
|
+
criteriaColumns: Object.keys(criteria),
|
|
3974
|
+
orderByColumns: (options.orderBy ?? []).map((o) => ({
|
|
3975
|
+
column: String(o.column),
|
|
3976
|
+
direction: o.direction
|
|
3977
|
+
})),
|
|
3978
|
+
selectColumns: options.select.map(String),
|
|
3979
|
+
primaryKeyColumns: this.primaryKeyNames.map(String)
|
|
3980
|
+
});
|
|
3981
|
+
const cols = options.select.map((c) => `\`${String(c)}\``).join(", ");
|
|
3982
|
+
let sql = `SELECT ${cols} FROM \`${this.table}\``;
|
|
3983
|
+
const { whereClause, params } = this.buildDeleteSearchWhere(criteria);
|
|
3984
|
+
sql += ` WHERE ${whereClause}`;
|
|
3985
|
+
if (options.orderBy && options.orderBy.length > 0) {
|
|
3986
|
+
sql += ` ORDER BY ` + options.orderBy.map((o) => `\`${String(o.column)}\` ${o.direction}`).join(", ");
|
|
3987
|
+
}
|
|
3988
|
+
if (options.limit !== undefined) {
|
|
3989
|
+
sql += ` LIMIT ?`;
|
|
3990
|
+
params.push(options.limit);
|
|
3991
|
+
}
|
|
3992
|
+
if (options.offset !== undefined) {
|
|
3993
|
+
if (options.limit === undefined)
|
|
3994
|
+
sql += ` LIMIT -1`;
|
|
3995
|
+
sql += ` OFFSET ?`;
|
|
3996
|
+
params.push(options.offset);
|
|
3997
|
+
}
|
|
3998
|
+
const stmt = this.db.prepare(sql);
|
|
3999
|
+
const rows = stmt.all(...params);
|
|
4000
|
+
for (const row of rows) {
|
|
4001
|
+
for (const k of Object.keys(row)) {
|
|
4002
|
+
row[k] = this.sqlToJsValue(k, row[k]);
|
|
4003
|
+
}
|
|
4004
|
+
}
|
|
4005
|
+
return rows;
|
|
4006
|
+
}
|
|
3725
4007
|
subscribeToChanges(callback, options) {
|
|
3726
4008
|
throw new Error("subscribeToChanges is not supported for SqliteTabularStorage");
|
|
3727
4009
|
}
|
|
@@ -4196,6 +4478,56 @@ class SupabaseTabularStorage extends BaseSqlTabularStorage {
|
|
|
4196
4478
|
this.events.emit("query", criteria, undefined);
|
|
4197
4479
|
return;
|
|
4198
4480
|
}
|
|
4481
|
+
async queryIndex(criteria, options) {
|
|
4482
|
+
this.validateSelect(options);
|
|
4483
|
+
this.validateQueryParams(criteria, options);
|
|
4484
|
+
const registered = this.indexes.map((cols, i) => {
|
|
4485
|
+
const cs = Array.isArray(cols) ? cols : [cols];
|
|
4486
|
+
return { name: `idx_${i}`, keyPath: cs };
|
|
4487
|
+
});
|
|
4488
|
+
pickCoveringIndex({
|
|
4489
|
+
table: this.table,
|
|
4490
|
+
indexes: registered,
|
|
4491
|
+
criteriaColumns: Object.keys(criteria),
|
|
4492
|
+
orderByColumns: (options.orderBy ?? []).map((o) => ({
|
|
4493
|
+
column: String(o.column),
|
|
4494
|
+
direction: o.direction
|
|
4495
|
+
})),
|
|
4496
|
+
selectColumns: options.select.map(String),
|
|
4497
|
+
primaryKeyColumns: this.primaryKeyNames.map(String)
|
|
4498
|
+
});
|
|
4499
|
+
const colList = options.select.map(String).join(",");
|
|
4500
|
+
let q = this.applyCriteriaToFilter(this.client.from(this.table).select(colList), criteria);
|
|
4501
|
+
if (options.orderBy) {
|
|
4502
|
+
for (const { column, direction } of options.orderBy) {
|
|
4503
|
+
q = q.order(String(column), { ascending: direction === "ASC" });
|
|
4504
|
+
}
|
|
4505
|
+
}
|
|
4506
|
+
if (options.offset !== undefined && options.limit === undefined) {
|
|
4507
|
+
throw new StorageValidationError("queryIndex with offset requires limit (no implicit cap)");
|
|
4508
|
+
}
|
|
4509
|
+
if (options.offset !== undefined || options.limit !== undefined) {
|
|
4510
|
+
const start = options.offset ?? 0;
|
|
4511
|
+
if (options.limit !== undefined) {
|
|
4512
|
+
q = q.range(start, start + options.limit - 1);
|
|
4513
|
+
}
|
|
4514
|
+
}
|
|
4515
|
+
const { data, error } = await q;
|
|
4516
|
+
if (error)
|
|
4517
|
+
throw error;
|
|
4518
|
+
if (!data)
|
|
4519
|
+
return [];
|
|
4520
|
+
const rows = data;
|
|
4521
|
+
const sel = new Set(options.select.map(String));
|
|
4522
|
+
for (const row of rows) {
|
|
4523
|
+
for (const key of Object.keys(row)) {
|
|
4524
|
+
if (sel.has(key)) {
|
|
4525
|
+
row[key] = this.sqlToJsValue(key, row[key]);
|
|
4526
|
+
}
|
|
4527
|
+
}
|
|
4528
|
+
}
|
|
4529
|
+
return rows;
|
|
4530
|
+
}
|
|
4199
4531
|
convertRealtimeRow(row) {
|
|
4200
4532
|
const entity = { ...row };
|
|
4201
4533
|
const record = entity;
|
|
@@ -4371,13 +4703,15 @@ class SupabaseKvStorage extends KvViaTabularStorage {
|
|
|
4371
4703
|
}
|
|
4372
4704
|
}
|
|
4373
4705
|
// src/queue/PostgresQueueStorage.ts
|
|
4374
|
-
import {
|
|
4706
|
+
import { createHash } from "crypto";
|
|
4707
|
+
import { createServiceToken as createServiceToken21, getLogger as getLogger4, makeFingerprint as makeFingerprint6, uuid4 as uuid46 } from "@workglow/util";
|
|
4375
4708
|
var POSTGRES_QUEUE_STORAGE = createServiceToken21("jobqueue.storage.postgres");
|
|
4376
4709
|
var SAFE_IDENTIFIER = /^[a-zA-Z][a-zA-Z0-9_]*$/;
|
|
4377
4710
|
|
|
4378
4711
|
class PostgresQueueStorage {
|
|
4379
4712
|
db;
|
|
4380
4713
|
queueName;
|
|
4714
|
+
scope = "cluster";
|
|
4381
4715
|
prefixes;
|
|
4382
4716
|
prefixValues;
|
|
4383
4717
|
tableName;
|
|
@@ -4474,14 +4808,46 @@ class PostgresQueueStorage {
|
|
|
4474
4808
|
ON ${this.tableName} (${prefixIndexPrefix}queue, status, run_after)`;
|
|
4475
4809
|
await this.db.query(sql);
|
|
4476
4810
|
sql = `
|
|
4477
|
-
CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx
|
|
4811
|
+
CREATE INDEX IF NOT EXISTS jobs_fingerprint${indexSuffix}_unique_idx
|
|
4478
4812
|
ON ${this.tableName} (${prefixIndexPrefix}queue, fingerprint, status)`;
|
|
4479
4813
|
await this.db.query(sql);
|
|
4814
|
+
const fnName = `${this.tableName}_notify`;
|
|
4815
|
+
const trgName = `${this.tableName}_notify_trg`;
|
|
4816
|
+
try {
|
|
4817
|
+
await this.db.query(`
|
|
4818
|
+
CREATE OR REPLACE FUNCTION ${fnName}() RETURNS trigger AS $fn$
|
|
4819
|
+
DECLARE
|
|
4820
|
+
channel TEXT := 'wglw_q_' || md5('${this.tableName}' || COALESCE(NEW.queue, OLD.queue));
|
|
4821
|
+
payload TEXT;
|
|
4822
|
+
BEGIN
|
|
4823
|
+
payload := json_build_object(
|
|
4824
|
+
'op', TG_OP,
|
|
4825
|
+
'id', COALESCE(NEW.id, OLD.id),
|
|
4826
|
+
'queue', COALESCE(NEW.queue, OLD.queue),
|
|
4827
|
+
'status', COALESCE(NEW.status::text, OLD.status::text)
|
|
4828
|
+
)::text;
|
|
4829
|
+
PERFORM pg_notify(channel, payload);
|
|
4830
|
+
RETURN NULL;
|
|
4831
|
+
END;
|
|
4832
|
+
$fn$ LANGUAGE plpgsql;
|
|
4833
|
+
`);
|
|
4834
|
+
await this.db.query(`DROP TRIGGER IF EXISTS ${trgName} ON ${this.tableName}`);
|
|
4835
|
+
await this.db.query(`
|
|
4836
|
+
CREATE TRIGGER ${trgName}
|
|
4837
|
+
AFTER INSERT OR UPDATE ON ${this.tableName}
|
|
4838
|
+
FOR EACH ROW EXECUTE FUNCTION ${fnName}();
|
|
4839
|
+
`);
|
|
4840
|
+
} catch {}
|
|
4841
|
+
}
|
|
4842
|
+
notifyChannelName() {
|
|
4843
|
+
const tableAndQueue = `${this.tableName}${this.queueName}`;
|
|
4844
|
+
const hash = createHash("md5").update(tableAndQueue).digest("hex");
|
|
4845
|
+
return `wglw_q_${hash}`;
|
|
4480
4846
|
}
|
|
4481
4847
|
async add(job) {
|
|
4482
4848
|
const now = new Date().toISOString();
|
|
4483
4849
|
job.queue = this.queueName;
|
|
4484
|
-
job.job_run_id = job.job_run_id ??
|
|
4850
|
+
job.job_run_id = job.job_run_id ?? uuid46();
|
|
4485
4851
|
job.fingerprint = await makeFingerprint6(job.input);
|
|
4486
4852
|
job.status = JobStatus.PENDING;
|
|
4487
4853
|
job.progress = 0;
|
|
@@ -4720,17 +5086,138 @@ class PostgresQueueStorage {
|
|
|
4720
5086
|
AND completed_at <= $3${prefixConditions}`, [this.queueName, status, cutoffDate, ...prefixParams]);
|
|
4721
5087
|
}
|
|
4722
5088
|
subscribeToChanges(callback, options) {
|
|
4723
|
-
|
|
5089
|
+
const poolMaybe = this.db;
|
|
5090
|
+
if (typeof poolMaybe.connect !== "function") {
|
|
5091
|
+
throw new Error("PostgresQueueStorage.subscribeToChanges requires a pg.Pool (got a single-connection wrapper)");
|
|
5092
|
+
}
|
|
5093
|
+
const pool = poolMaybe;
|
|
5094
|
+
const channel = this.notifyChannelName();
|
|
5095
|
+
const effectivePrefixFilter = options?.prefixFilter === undefined ? this.prefixes.length > 0 ? this.prefixValues : null : Object.keys(options.prefixFilter).length === 0 ? null : options.prefixFilter;
|
|
5096
|
+
let unsubscribed = false;
|
|
5097
|
+
let activeClient = null;
|
|
5098
|
+
let reconnectTimer = null;
|
|
5099
|
+
let backoffMs = 250;
|
|
5100
|
+
const dispatch = (change) => {
|
|
5101
|
+
try {
|
|
5102
|
+
callback(change);
|
|
5103
|
+
} catch (err) {
|
|
5104
|
+
getLogger4().debug("PostgresQueueStorage subscribe callback threw", {
|
|
5105
|
+
channel,
|
|
5106
|
+
changeType: change.type,
|
|
5107
|
+
error: err
|
|
5108
|
+
});
|
|
5109
|
+
}
|
|
5110
|
+
};
|
|
5111
|
+
const matchesPrefix = (change, filter) => {
|
|
5112
|
+
const row = change.new ?? change.old;
|
|
5113
|
+
if (!row)
|
|
5114
|
+
return false;
|
|
5115
|
+
for (const [k, v] of Object.entries(filter)) {
|
|
5116
|
+
if (row[k] !== v)
|
|
5117
|
+
return false;
|
|
5118
|
+
}
|
|
5119
|
+
return true;
|
|
5120
|
+
};
|
|
5121
|
+
const hydrate = async (id) => {
|
|
5122
|
+
try {
|
|
5123
|
+
return await this.get(id);
|
|
5124
|
+
} catch {
|
|
5125
|
+
return;
|
|
5126
|
+
}
|
|
5127
|
+
};
|
|
5128
|
+
const handleNotification = (msg) => {
|
|
5129
|
+
if (msg.channel !== channel || !msg.payload)
|
|
5130
|
+
return;
|
|
5131
|
+
let parsed;
|
|
5132
|
+
try {
|
|
5133
|
+
parsed = JSON.parse(msg.payload);
|
|
5134
|
+
} catch {
|
|
5135
|
+
return;
|
|
5136
|
+
}
|
|
5137
|
+
(async () => {
|
|
5138
|
+
const op = parsed.op;
|
|
5139
|
+
const fallback = {
|
|
5140
|
+
id: parsed.id,
|
|
5141
|
+
queue: parsed.queue,
|
|
5142
|
+
status: parsed.status
|
|
5143
|
+
};
|
|
5144
|
+
const fullRow = op === "DELETE" ? undefined : await hydrate(parsed.id);
|
|
5145
|
+
const change = {
|
|
5146
|
+
type: op === "INSERT" ? "INSERT" : op === "DELETE" ? "DELETE" : "UPDATE",
|
|
5147
|
+
new: op === "DELETE" ? undefined : fullRow ?? fallback,
|
|
5148
|
+
old: op === "DELETE" ? fallback : undefined
|
|
5149
|
+
};
|
|
5150
|
+
if (effectivePrefixFilter && !matchesPrefix(change, effectivePrefixFilter))
|
|
5151
|
+
return;
|
|
5152
|
+
dispatch(change);
|
|
5153
|
+
})();
|
|
5154
|
+
};
|
|
5155
|
+
const connect = async () => {
|
|
5156
|
+
if (unsubscribed)
|
|
5157
|
+
return;
|
|
5158
|
+
try {
|
|
5159
|
+
const client = await pool.connect();
|
|
5160
|
+
if (unsubscribed) {
|
|
5161
|
+
client.release();
|
|
5162
|
+
return;
|
|
5163
|
+
}
|
|
5164
|
+
activeClient = client;
|
|
5165
|
+
client.on("notification", handleNotification);
|
|
5166
|
+
client.on("error", () => scheduleReconnect());
|
|
5167
|
+
await client.query(`LISTEN ${channel}`);
|
|
5168
|
+
backoffMs = 250;
|
|
5169
|
+
dispatch({ type: "RESYNC" });
|
|
5170
|
+
} catch {
|
|
5171
|
+
scheduleReconnect();
|
|
5172
|
+
}
|
|
5173
|
+
};
|
|
5174
|
+
const scheduleReconnect = () => {
|
|
5175
|
+
if (unsubscribed || reconnectTimer)
|
|
5176
|
+
return;
|
|
5177
|
+
const c = activeClient;
|
|
5178
|
+
activeClient = null;
|
|
5179
|
+
try {
|
|
5180
|
+
c?.removeAllListeners?.("notification");
|
|
5181
|
+
c?.removeAllListeners?.("error");
|
|
5182
|
+
c?.release();
|
|
5183
|
+
} catch {}
|
|
5184
|
+
const delay = Math.min(backoffMs, 30000);
|
|
5185
|
+
backoffMs = Math.min(backoffMs * 2, 30000);
|
|
5186
|
+
reconnectTimer = setTimeout(() => {
|
|
5187
|
+
reconnectTimer = null;
|
|
5188
|
+
connect();
|
|
5189
|
+
}, delay);
|
|
5190
|
+
};
|
|
5191
|
+
connect();
|
|
5192
|
+
return () => {
|
|
5193
|
+
unsubscribed = true;
|
|
5194
|
+
if (reconnectTimer) {
|
|
5195
|
+
clearTimeout(reconnectTimer);
|
|
5196
|
+
reconnectTimer = null;
|
|
5197
|
+
}
|
|
5198
|
+
const c = activeClient;
|
|
5199
|
+
activeClient = null;
|
|
5200
|
+
if (c) {
|
|
5201
|
+
c.query(`UNLISTEN ${channel}`).catch(() => {}).finally(() => {
|
|
5202
|
+
try {
|
|
5203
|
+
c.removeAllListeners?.("notification");
|
|
5204
|
+
c.removeAllListeners?.("error");
|
|
5205
|
+
c.release();
|
|
5206
|
+
} catch {}
|
|
5207
|
+
});
|
|
5208
|
+
}
|
|
5209
|
+
};
|
|
4724
5210
|
}
|
|
4725
5211
|
}
|
|
4726
5212
|
// src/queue/SqliteQueueStorage.ts
|
|
4727
|
-
import { createServiceToken as createServiceToken22, makeFingerprint as makeFingerprint7, sleep as sleep4, uuid4 as
|
|
5213
|
+
import { createServiceToken as createServiceToken22, makeFingerprint as makeFingerprint7, sleep as sleep4, uuid4 as uuid47 } from "@workglow/util";
|
|
4728
5214
|
var SQLITE_QUEUE_STORAGE = createServiceToken22("jobqueue.storage.sqlite");
|
|
4729
5215
|
|
|
4730
5216
|
class SqliteQueueStorage {
|
|
4731
5217
|
db;
|
|
4732
5218
|
queueName;
|
|
4733
5219
|
options;
|
|
5220
|
+
scope = "process";
|
|
4734
5221
|
prefixes;
|
|
4735
5222
|
prefixValues;
|
|
4736
5223
|
tableName;
|
|
@@ -4807,7 +5294,7 @@ class SqliteQueueStorage {
|
|
|
4807
5294
|
}
|
|
4808
5295
|
async add(job) {
|
|
4809
5296
|
const now = new Date().toISOString();
|
|
4810
|
-
job.job_run_id = job.job_run_id ??
|
|
5297
|
+
job.job_run_id = job.job_run_id ?? uuid47();
|
|
4811
5298
|
job.queue = this.queueName;
|
|
4812
5299
|
job.fingerprint = await makeFingerprint7(job.input);
|
|
4813
5300
|
job.status = JobStatus.PENDING;
|
|
@@ -5074,11 +5561,12 @@ class SqliteQueueStorage {
|
|
|
5074
5561
|
}
|
|
5075
5562
|
}
|
|
5076
5563
|
// src/queue/SupabaseQueueStorage.ts
|
|
5077
|
-
import { createServiceToken as createServiceToken23, deepEqual as deepEqual2, makeFingerprint as makeFingerprint8, uuid4 as
|
|
5564
|
+
import { createServiceToken as createServiceToken23, deepEqual as deepEqual2, makeFingerprint as makeFingerprint8, uuid4 as uuid48 } from "@workglow/util";
|
|
5078
5565
|
var SUPABASE_QUEUE_STORAGE = createServiceToken23("jobqueue.storage.supabase");
|
|
5079
5566
|
|
|
5080
5567
|
class SupabaseQueueStorage {
|
|
5081
5568
|
queueName;
|
|
5569
|
+
scope = "cluster";
|
|
5082
5570
|
client;
|
|
5083
5571
|
prefixes;
|
|
5084
5572
|
prefixValues;
|
|
@@ -5203,7 +5691,7 @@ class SupabaseQueueStorage {
|
|
|
5203
5691
|
async add(job) {
|
|
5204
5692
|
const now = new Date().toISOString();
|
|
5205
5693
|
job.queue = this.queueName;
|
|
5206
|
-
job.job_run_id = job.job_run_id ??
|
|
5694
|
+
job.job_run_id = job.job_run_id ?? uuid48();
|
|
5207
5695
|
job.fingerprint = await makeFingerprint8(job.input);
|
|
5208
5696
|
job.status = JobStatus.PENDING;
|
|
5209
5697
|
job.progress = 0;
|
|
@@ -5606,6 +6094,7 @@ var POSTGRES_RATE_LIMITER_STORAGE = createServiceToken24("ratelimiter.storage.po
|
|
|
5606
6094
|
|
|
5607
6095
|
class PostgresRateLimiterStorage {
|
|
5608
6096
|
db;
|
|
6097
|
+
scope = "cluster";
|
|
5609
6098
|
prefixes;
|
|
5610
6099
|
prefixValues;
|
|
5611
6100
|
executionTableName;
|
|
@@ -5672,6 +6161,80 @@ class PostgresRateLimiterStorage {
|
|
|
5672
6161
|
)
|
|
5673
6162
|
`);
|
|
5674
6163
|
}
|
|
6164
|
+
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
6165
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
6166
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
6167
|
+
const prefixCount = prefixColumnNames.length;
|
|
6168
|
+
const queueParam = `$${prefixCount + 1}`;
|
|
6169
|
+
const windowStartParam = `$${prefixCount + 2}`;
|
|
6170
|
+
const lockKeyParts = [`'${this.executionTableName}'`];
|
|
6171
|
+
for (let i = 0;i < prefixCount; i++) {
|
|
6172
|
+
lockKeyParts.push(`$${i + 1}::text`);
|
|
6173
|
+
}
|
|
6174
|
+
lockKeyParts.push(`${queueParam}::text`);
|
|
6175
|
+
const lockKeyExpr = `hashtextextended(${lockKeyParts.join(" || '|' || ")}, 0)`;
|
|
6176
|
+
const prefixWhere = prefixCount > 0 ? " AND " + prefixColumnNames.map((p, i) => `${p} = $${i + 1}`).join(" AND ") : "";
|
|
6177
|
+
const prefixInsertCols = prefixCount > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
6178
|
+
const prefixInsertPlaceholders = prefixCount > 0 ? prefixColumnNames.map((_, i) => `$${i + 1}`).join(", ") + ", " : "";
|
|
6179
|
+
const windowStart = new Date(Date.now() - windowMs).toISOString();
|
|
6180
|
+
const supportsConnect = typeof this.db.connect === "function";
|
|
6181
|
+
if (!supportsConnect) {
|
|
6182
|
+
const dbAny = this.db;
|
|
6183
|
+
const ctorName = dbAny.constructor?.name;
|
|
6184
|
+
const looksLikePGlite = typeof dbAny.exec === "function" && dbAny.waitReady !== undefined;
|
|
6185
|
+
const looksLikePGLitePool = ctorName === "PGLitePool";
|
|
6186
|
+
if (!looksLikePGlite && !looksLikePGLitePool) {
|
|
6187
|
+
throw new Error(`PostgresRateLimiterStorage.tryReserveExecution requires a pg.Pool with connect() or a known single-connection wrapper (PGLitePool, PGlite); got ${ctorName ?? typeof this.db}. A multi-connection pool without connect() would dispatch the advisory lock and the INSERT to different sessions, breaking atomicity.`);
|
|
6188
|
+
}
|
|
6189
|
+
}
|
|
6190
|
+
const conn = supportsConnect ? await this.db.connect() : { query: this.db.query.bind(this.db), release: () => {} };
|
|
6191
|
+
try {
|
|
6192
|
+
await conn.query("BEGIN");
|
|
6193
|
+
try {
|
|
6194
|
+
await conn.query(`SELECT pg_advisory_xact_lock(${lockKeyExpr})`, [...prefixParamValues, queueName]);
|
|
6195
|
+
const countResult = await conn.query(`
|
|
6196
|
+
SELECT COUNT(*)::int AS n
|
|
6197
|
+
FROM ${this.executionTableName}
|
|
6198
|
+
WHERE queue_name = ${queueParam} AND executed_at > ${windowStartParam}${prefixWhere}
|
|
6199
|
+
`, [...prefixParamValues, queueName, windowStart]);
|
|
6200
|
+
const n = countResult.rows[0]?.n ?? 0;
|
|
6201
|
+
if (n >= maxExecutions) {
|
|
6202
|
+
await conn.query("COMMIT");
|
|
6203
|
+
return null;
|
|
6204
|
+
}
|
|
6205
|
+
const naResult = await conn.query(`
|
|
6206
|
+
SELECT next_available_at
|
|
6207
|
+
FROM ${this.nextAvailableTableName}
|
|
6208
|
+
WHERE queue_name = ${queueParam}${prefixWhere}
|
|
6209
|
+
`, [...prefixParamValues, queueName]);
|
|
6210
|
+
const nextAvailableAt = naResult.rows[0]?.next_available_at ?? null;
|
|
6211
|
+
if (nextAvailableAt && new Date(nextAvailableAt).getTime() > Date.now()) {
|
|
6212
|
+
await conn.query("COMMIT");
|
|
6213
|
+
return null;
|
|
6214
|
+
}
|
|
6215
|
+
const insertResult = await conn.query(`
|
|
6216
|
+
INSERT INTO ${this.executionTableName} (${prefixInsertCols}queue_name)
|
|
6217
|
+
VALUES (${prefixInsertPlaceholders}${queueParam})
|
|
6218
|
+
RETURNING id
|
|
6219
|
+
`, [...prefixParamValues, queueName]);
|
|
6220
|
+
await conn.query("COMMIT");
|
|
6221
|
+
return insertResult.rows[0]?.id ?? null;
|
|
6222
|
+
} catch (err) {
|
|
6223
|
+
try {
|
|
6224
|
+
await conn.query("ROLLBACK");
|
|
6225
|
+
} catch {}
|
|
6226
|
+
throw err;
|
|
6227
|
+
}
|
|
6228
|
+
} finally {
|
|
6229
|
+
conn.release();
|
|
6230
|
+
}
|
|
6231
|
+
}
|
|
6232
|
+
async releaseExecution(queueName, token) {
|
|
6233
|
+
if (token === null || token === undefined)
|
|
6234
|
+
return;
|
|
6235
|
+
const { conditions: prefixConditions, params: prefixParams } = this.buildPrefixWhereClause(3);
|
|
6236
|
+
await this.db.query(`DELETE FROM ${this.executionTableName} WHERE id = $1 AND queue_name = $2${prefixConditions}`, [token, queueName, ...prefixParams]);
|
|
6237
|
+
}
|
|
5675
6238
|
async recordExecution(queueName) {
|
|
5676
6239
|
const prefixColumnNames = this.getPrefixColumnNames();
|
|
5677
6240
|
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
@@ -5744,6 +6307,7 @@ var SQLITE_RATE_LIMITER_STORAGE = createServiceToken25("ratelimiter.storage.sqli
|
|
|
5744
6307
|
|
|
5745
6308
|
class SqliteRateLimiterStorage {
|
|
5746
6309
|
db;
|
|
6310
|
+
scope = "process";
|
|
5747
6311
|
prefixes;
|
|
5748
6312
|
prefixValues;
|
|
5749
6313
|
executionTableName;
|
|
@@ -5807,6 +6371,55 @@ class SqliteRateLimiterStorage {
|
|
|
5807
6371
|
);
|
|
5808
6372
|
`);
|
|
5809
6373
|
}
|
|
6374
|
+
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
6375
|
+
const prefixColumnNames = this.getPrefixColumnNames();
|
|
6376
|
+
const prefixParamValues = this.getPrefixParamValues();
|
|
6377
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
6378
|
+
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
6379
|
+
const prefixPlaceholders = prefixColumnNames.length > 0 ? prefixColumnNames.map(() => "?").join(", ") + ", " : "";
|
|
6380
|
+
const windowStart = toSQLiteTimestamp(new Date(Date.now() - windowMs));
|
|
6381
|
+
let insertedId = null;
|
|
6382
|
+
const txn = this.db.transaction(() => {
|
|
6383
|
+
const countStmt = this.db.prepare(`
|
|
6384
|
+
SELECT COUNT(*) AS count
|
|
6385
|
+
FROM ${this.executionTableName}
|
|
6386
|
+
WHERE queue_name = ? AND executed_at > ?${prefixConditions}
|
|
6387
|
+
`);
|
|
6388
|
+
const countRow = countStmt.get(queueName, windowStart, ...prefixParamValues);
|
|
6389
|
+
if ((countRow?.count ?? 0) >= maxExecutions) {
|
|
6390
|
+
return;
|
|
6391
|
+
}
|
|
6392
|
+
const naStmt = this.db.prepare(`
|
|
6393
|
+
SELECT next_available_at
|
|
6394
|
+
FROM ${this.nextAvailableTableName}
|
|
6395
|
+
WHERE queue_name = ?${prefixConditions}
|
|
6396
|
+
`);
|
|
6397
|
+
const naRow = naStmt.get(queueName, ...prefixParamValues);
|
|
6398
|
+
if (naRow?.next_available_at) {
|
|
6399
|
+
const nextAvailable = new Date(naRow.next_available_at + "Z").getTime();
|
|
6400
|
+
if (nextAvailable > Date.now()) {
|
|
6401
|
+
return;
|
|
6402
|
+
}
|
|
6403
|
+
}
|
|
6404
|
+
const insertStmt = this.db.prepare(`
|
|
6405
|
+
INSERT INTO ${this.executionTableName} (${prefixColumnsInsert}queue_name)
|
|
6406
|
+
VALUES (${prefixPlaceholders}?)
|
|
6407
|
+
`);
|
|
6408
|
+
const info = insertStmt.run(...prefixParamValues, queueName);
|
|
6409
|
+
insertedId = info.lastInsertRowid;
|
|
6410
|
+
});
|
|
6411
|
+
txn();
|
|
6412
|
+
if (insertedId == null)
|
|
6413
|
+
return null;
|
|
6414
|
+
return typeof insertedId === "bigint" ? Number(insertedId) : insertedId;
|
|
6415
|
+
}
|
|
6416
|
+
async releaseExecution(queueName, token) {
|
|
6417
|
+
if (token === null || token === undefined)
|
|
6418
|
+
return;
|
|
6419
|
+
const prefixConditions = this.buildPrefixWhereClause();
|
|
6420
|
+
const prefixParams = this.getPrefixParamValues();
|
|
6421
|
+
this.db.prepare(`DELETE FROM ${this.executionTableName} WHERE id = ? AND queue_name = ?${prefixConditions}`).run(token, queueName, ...prefixParams);
|
|
6422
|
+
}
|
|
5810
6423
|
async recordExecution(queueName) {
|
|
5811
6424
|
const prefixColumnNames = this.getPrefixColumnNames();
|
|
5812
6425
|
const prefixColumnsInsert = prefixColumnNames.length > 0 ? prefixColumnNames.join(", ") + ", " : "";
|
|
@@ -5882,6 +6495,7 @@ import { createServiceToken as createServiceToken26 } from "@workglow/util";
|
|
|
5882
6495
|
var SUPABASE_RATE_LIMITER_STORAGE = createServiceToken26("ratelimiter.storage.supabase");
|
|
5883
6496
|
|
|
5884
6497
|
class SupabaseRateLimiterStorage {
|
|
6498
|
+
scope = "cluster";
|
|
5885
6499
|
client;
|
|
5886
6500
|
prefixes;
|
|
5887
6501
|
prefixValues;
|
|
@@ -5964,6 +6578,75 @@ class SupabaseRateLimiterStorage {
|
|
|
5964
6578
|
if (nextTableError && nextTableError.code !== "42P07") {
|
|
5965
6579
|
throw nextTableError;
|
|
5966
6580
|
}
|
|
6581
|
+
const fnName = this.atomicReserveFunctionName();
|
|
6582
|
+
const prefixSig = this.prefixes.map((p) => `${p.name} ${this.getPrefixColumnType(p.type)}`).join(", ");
|
|
6583
|
+
const prefixSigPrefix = prefixSig ? prefixSig + ", " : "";
|
|
6584
|
+
const prefixWhere = this.prefixes.length > 0 ? " AND " + this.prefixes.map((p) => `${p.name} = _${p.name}`).join(" AND ") : "";
|
|
6585
|
+
const prefixInsertCols = this.prefixes.length > 0 ? this.prefixes.map((p) => p.name).join(", ") + ", " : "";
|
|
6586
|
+
const prefixInsertVals = this.prefixes.length > 0 ? this.prefixes.map((p) => `_${p.name}`).join(", ") + ", " : "";
|
|
6587
|
+
const lockKeyParts = [`'${this.executionTableName}'`, ...this.prefixes.map((p) => `_${p.name}::text`), `_queue_name::text`];
|
|
6588
|
+
const lockKeyExpr = `hashtextextended(${lockKeyParts.join(" || '|' || ")}, 0)`;
|
|
6589
|
+
const createFnSql = `
|
|
6590
|
+
CREATE OR REPLACE FUNCTION ${fnName}(
|
|
6591
|
+
${prefixSigPrefix}_queue_name TEXT, _window_start TIMESTAMPTZ, _max_exec INT
|
|
6592
|
+
) RETURNS BIGINT AS $fn$
|
|
6593
|
+
DECLARE
|
|
6594
|
+
_count INT;
|
|
6595
|
+
_next TIMESTAMPTZ;
|
|
6596
|
+
_new_id BIGINT;
|
|
6597
|
+
BEGIN
|
|
6598
|
+
PERFORM pg_advisory_xact_lock(${lockKeyExpr});
|
|
6599
|
+
SELECT COUNT(*) INTO _count FROM ${this.executionTableName}
|
|
6600
|
+
WHERE queue_name = _queue_name AND executed_at > _window_start${prefixWhere};
|
|
6601
|
+
IF _count >= _max_exec THEN RETURN NULL; END IF;
|
|
6602
|
+
SELECT next_available_at INTO _next FROM ${this.nextAvailableTableName}
|
|
6603
|
+
WHERE queue_name = _queue_name${prefixWhere};
|
|
6604
|
+
IF _next IS NOT NULL AND _next > NOW() THEN RETURN NULL; END IF;
|
|
6605
|
+
INSERT INTO ${this.executionTableName} (${prefixInsertCols}queue_name)
|
|
6606
|
+
VALUES (${prefixInsertVals}_queue_name)
|
|
6607
|
+
RETURNING id INTO _new_id;
|
|
6608
|
+
RETURN _new_id;
|
|
6609
|
+
END;
|
|
6610
|
+
$fn$ LANGUAGE plpgsql;
|
|
6611
|
+
`;
|
|
6612
|
+
const { error: fnError } = await this.client.rpc("exec_sql", { query: createFnSql });
|
|
6613
|
+
if (fnError) {
|
|
6614
|
+
throw fnError;
|
|
6615
|
+
}
|
|
6616
|
+
}
|
|
6617
|
+
atomicReserveFunctionName() {
|
|
6618
|
+
return `${this.executionTableName}_try_reserve`.slice(0, 63);
|
|
6619
|
+
}
|
|
6620
|
+
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
6621
|
+
const args = {
|
|
6622
|
+
_queue_name: queueName,
|
|
6623
|
+
_window_start: new Date(Date.now() - windowMs).toISOString(),
|
|
6624
|
+
_max_exec: maxExecutions
|
|
6625
|
+
};
|
|
6626
|
+
for (const p of this.prefixes) {
|
|
6627
|
+
args[`_${p.name}`] = this.prefixValues[p.name];
|
|
6628
|
+
}
|
|
6629
|
+
const { data, error } = await this.client.rpc(this.atomicReserveFunctionName(), args);
|
|
6630
|
+
if (error)
|
|
6631
|
+
throw error;
|
|
6632
|
+
if (data === null || data === undefined)
|
|
6633
|
+
return null;
|
|
6634
|
+
if (Array.isArray(data)) {
|
|
6635
|
+
if (data.length === 0)
|
|
6636
|
+
return null;
|
|
6637
|
+
const first = Object.values(data[0])[0];
|
|
6638
|
+
return first ?? null;
|
|
6639
|
+
}
|
|
6640
|
+
return data;
|
|
6641
|
+
}
|
|
6642
|
+
async releaseExecution(queueName, token) {
|
|
6643
|
+
if (token === null || token === undefined)
|
|
6644
|
+
return;
|
|
6645
|
+
let del = this.client.from(this.executionTableName).delete().eq("id", token).eq("queue_name", queueName);
|
|
6646
|
+
del = this.applyPrefixFilters(del);
|
|
6647
|
+
const { error: delError } = await del;
|
|
6648
|
+
if (delError)
|
|
6649
|
+
throw delError;
|
|
5967
6650
|
}
|
|
5968
6651
|
async recordExecution(queueName) {
|
|
5969
6652
|
const prefixInsertValues = this.getPrefixInsertValues();
|
|
@@ -6729,7 +7412,7 @@ class SqliteAiVectorStorage extends SqliteTabularStorage {
|
|
|
6729
7412
|
import { createServiceToken as createServiceToken28 } from "@workglow/util";
|
|
6730
7413
|
|
|
6731
7414
|
// src/tabular/IndexedDbTabularStorage.ts
|
|
6732
|
-
import { createServiceToken as createServiceToken27, deepEqual as deepEqual4, makeFingerprint as makeFingerprint9, uuid4 as
|
|
7415
|
+
import { createServiceToken as createServiceToken27, deepEqual as deepEqual4, makeFingerprint as makeFingerprint9, uuid4 as uuid49 } from "@workglow/util";
|
|
6733
7416
|
|
|
6734
7417
|
// src/util/IndexedDbTable.ts
|
|
6735
7418
|
import { deepEqual as deepEqual3 } from "@workglow/util";
|
|
@@ -7120,7 +7803,7 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
|
|
|
7120
7803
|
}
|
|
7121
7804
|
generateKeyValue(columnName, strategy) {
|
|
7122
7805
|
if (strategy === "uuid") {
|
|
7123
|
-
return
|
|
7806
|
+
return uuid49();
|
|
7124
7807
|
}
|
|
7125
7808
|
throw new Error(`IndexedDB autoincrement keys are generated by the database, not client-side. Column: ${columnName}`);
|
|
7126
7809
|
}
|
|
@@ -7621,6 +8304,107 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
|
|
|
7621
8304
|
request.onerror = () => reject(request.error);
|
|
7622
8305
|
});
|
|
7623
8306
|
}
|
|
8307
|
+
async queryIndex(criteria, options) {
|
|
8308
|
+
this.validateSelect(options);
|
|
8309
|
+
this.validateQueryParams(criteria, options);
|
|
8310
|
+
const registered = this.indexes.map((cols) => {
|
|
8311
|
+
const cs = Array.isArray(cols) ? cols : [cols];
|
|
8312
|
+
return { name: cs.join("_"), keyPath: cs };
|
|
8313
|
+
});
|
|
8314
|
+
const picked = pickCoveringIndex({
|
|
8315
|
+
table: this.table,
|
|
8316
|
+
indexes: registered,
|
|
8317
|
+
criteriaColumns: Object.keys(criteria),
|
|
8318
|
+
orderByColumns: (options.orderBy ?? []).map((o) => ({
|
|
8319
|
+
column: String(o.column),
|
|
8320
|
+
direction: o.direction
|
|
8321
|
+
})),
|
|
8322
|
+
selectColumns: options.select.map(String),
|
|
8323
|
+
primaryKeyColumns: this.primaryKeyColumns().map(String)
|
|
8324
|
+
});
|
|
8325
|
+
const db = await this.getDb();
|
|
8326
|
+
return new Promise((resolve, reject) => {
|
|
8327
|
+
const tx = db.transaction(this.table, "readonly");
|
|
8328
|
+
const store = tx.objectStore(this.table);
|
|
8329
|
+
const idx = store.index(picked.name);
|
|
8330
|
+
const prefix = [];
|
|
8331
|
+
for (const col of picked.keyPath) {
|
|
8332
|
+
const c = criteria[col];
|
|
8333
|
+
if (c === undefined && !(col in criteria))
|
|
8334
|
+
break;
|
|
8335
|
+
if (isSearchCondition(c)) {
|
|
8336
|
+
if (c.operator !== "=")
|
|
8337
|
+
break;
|
|
8338
|
+
prefix.push(c.value);
|
|
8339
|
+
} else {
|
|
8340
|
+
prefix.push(c);
|
|
8341
|
+
}
|
|
8342
|
+
}
|
|
8343
|
+
const range = prefix.length === 0 ? undefined : prefix.length === picked.keyPath.length ? IDBKeyRange.only(prefix.length === 1 ? prefix[0] : prefix) : IDBKeyRange.bound(prefix, [...prefix, []]);
|
|
8344
|
+
const direction = picked.reverseDirection ? "prev" : "next";
|
|
8345
|
+
const request = idx.openKeyCursor(range, direction);
|
|
8346
|
+
const out = [];
|
|
8347
|
+
let toSkip = options.offset ?? 0;
|
|
8348
|
+
const keyPathPositions = new Map;
|
|
8349
|
+
picked.keyPath.forEach((col, i) => keyPathPositions.set(col, i));
|
|
8350
|
+
const pkCols = this.primaryKeyColumns().map(String);
|
|
8351
|
+
const pkPositions = new Map;
|
|
8352
|
+
pkCols.forEach((col, i) => pkPositions.set(col, i));
|
|
8353
|
+
request.onsuccess = () => {
|
|
8354
|
+
const cursor = request.result;
|
|
8355
|
+
if (!cursor) {
|
|
8356
|
+
resolve(out);
|
|
8357
|
+
return;
|
|
8358
|
+
}
|
|
8359
|
+
const key = cursor.key;
|
|
8360
|
+
const row = {};
|
|
8361
|
+
for (const col of options.select) {
|
|
8362
|
+
const colStr = String(col);
|
|
8363
|
+
const pos = keyPathPositions.get(colStr);
|
|
8364
|
+
if (pos !== undefined) {
|
|
8365
|
+
row[colStr] = Array.isArray(key) ? key[pos] : key;
|
|
8366
|
+
} else {
|
|
8367
|
+
if (pkCols.length === 1 && colStr === pkCols[0]) {
|
|
8368
|
+
row[colStr] = cursor.primaryKey;
|
|
8369
|
+
} else {
|
|
8370
|
+
const pkPos = pkPositions.get(colStr);
|
|
8371
|
+
if (pkPos !== undefined) {
|
|
8372
|
+
row[colStr] = Array.isArray(cursor.primaryKey) ? cursor.primaryKey[pkPos] : cursor.primaryKey;
|
|
8373
|
+
}
|
|
8374
|
+
}
|
|
8375
|
+
}
|
|
8376
|
+
}
|
|
8377
|
+
let matches = true;
|
|
8378
|
+
for (const [col, crit] of Object.entries(criteria)) {
|
|
8379
|
+
const pos = keyPathPositions.get(col);
|
|
8380
|
+
if (pos === undefined)
|
|
8381
|
+
continue;
|
|
8382
|
+
if (pos < prefix.length)
|
|
8383
|
+
continue;
|
|
8384
|
+
const valFromKey = Array.isArray(key) ? key[pos] : key;
|
|
8385
|
+
const op = isSearchCondition(crit) ? crit.operator : "=";
|
|
8386
|
+
const val = isSearchCondition(crit) ? crit.value : crit;
|
|
8387
|
+
if (!compareWithOperator(valFromKey, op, val)) {
|
|
8388
|
+
matches = false;
|
|
8389
|
+
break;
|
|
8390
|
+
}
|
|
8391
|
+
}
|
|
8392
|
+
if (matches) {
|
|
8393
|
+
if (toSkip > 0) {
|
|
8394
|
+
toSkip -= 1;
|
|
8395
|
+
} else {
|
|
8396
|
+
out.push(row);
|
|
8397
|
+
if (options.limit !== undefined && out.length >= options.limit) {
|
|
8398
|
+
resolve(out);
|
|
8399
|
+
return;
|
|
8400
|
+
}
|
|
8401
|
+
}
|
|
8402
|
+
}
|
|
8403
|
+
cursor.continue();
|
|
8404
|
+
};
|
|
8405
|
+
request.onerror = () => reject(request.error);
|
|
8406
|
+
});
|
|
8407
|
+
}
|
|
7624
8408
|
getHybridManager() {
|
|
7625
8409
|
if (!this.hybridManager) {
|
|
7626
8410
|
const channelName = `indexeddb-tabular-${this.table}`;
|
|
@@ -7658,6 +8442,22 @@ class IndexedDbTabularStorage extends BaseTabularStorage {
|
|
|
7658
8442
|
this.db?.close();
|
|
7659
8443
|
}
|
|
7660
8444
|
}
|
|
8445
|
+
function compareWithOperator(a, op, b) {
|
|
8446
|
+
const av = a;
|
|
8447
|
+
const bv = b;
|
|
8448
|
+
switch (op) {
|
|
8449
|
+
case "=":
|
|
8450
|
+
return av === bv;
|
|
8451
|
+
case "<":
|
|
8452
|
+
return av !== null && av !== undefined && av < bv;
|
|
8453
|
+
case "<=":
|
|
8454
|
+
return av !== null && av !== undefined && av <= bv;
|
|
8455
|
+
case ">":
|
|
8456
|
+
return av !== null && av !== undefined && av > bv;
|
|
8457
|
+
case ">=":
|
|
8458
|
+
return av !== null && av !== undefined && av >= bv;
|
|
8459
|
+
}
|
|
8460
|
+
}
|
|
7661
8461
|
|
|
7662
8462
|
// src/kv/IndexedDbKvStorage.ts
|
|
7663
8463
|
var IDB_KV_REPOSITORY = createServiceToken28("storage.kvRepository.indexedDb");
|
|
@@ -7676,6 +8476,7 @@ import { createServiceToken as createServiceToken29 } from "@workglow/util";
|
|
|
7676
8476
|
var INDEXED_DB_RATE_LIMITER_STORAGE = createServiceToken29("ratelimiter.storage.indexedDb");
|
|
7677
8477
|
|
|
7678
8478
|
class IndexedDbRateLimiterStorage {
|
|
8479
|
+
scope = "process";
|
|
7679
8480
|
executionDb;
|
|
7680
8481
|
nextAvailableDb;
|
|
7681
8482
|
executionTableName;
|
|
@@ -7744,6 +8545,72 @@ class IndexedDbRateLimiterStorage {
|
|
|
7744
8545
|
];
|
|
7745
8546
|
this.nextAvailableDb = await ensureIndexedDbTable(this.nextAvailableTableName, buildKeyPath(["queue_name"]).join("_"), nextAvailableIndexes, this.migrationOptions);
|
|
7746
8547
|
}
|
|
8548
|
+
async tryReserveExecution(queueName, maxExecutions, windowMs) {
|
|
8549
|
+
const nextIso = await this.getNextAvailableTime(queueName);
|
|
8550
|
+
if (nextIso && new Date(nextIso).getTime() > Date.now()) {
|
|
8551
|
+
return null;
|
|
8552
|
+
}
|
|
8553
|
+
const execDb = await this.getExecutionDb();
|
|
8554
|
+
const prefixKeyValues = this.getPrefixKeyValues();
|
|
8555
|
+
const windowStartIso = new Date(Date.now() - windowMs).toISOString();
|
|
8556
|
+
const execTx = execDb.transaction(this.executionTableName, "readwrite");
|
|
8557
|
+
const execStore = execTx.objectStore(this.executionTableName);
|
|
8558
|
+
const insertedId = crypto.randomUUID();
|
|
8559
|
+
return new Promise((resolve, reject) => {
|
|
8560
|
+
let liveCount = 0;
|
|
8561
|
+
let didInsert = false;
|
|
8562
|
+
const liveRange = IDBKeyRange.bound([...prefixKeyValues, queueName, windowStartIso], [...prefixKeyValues, queueName, "\uFFFF"], true, false);
|
|
8563
|
+
const cursorReq = execStore.index("queue_executed_at").openCursor(liveRange);
|
|
8564
|
+
cursorReq.onsuccess = (event) => {
|
|
8565
|
+
const cursor = event.target.result;
|
|
8566
|
+
if (cursor) {
|
|
8567
|
+
const record2 = cursor.value;
|
|
8568
|
+
if (this.matchesPrefixes(record2)) {
|
|
8569
|
+
liveCount++;
|
|
8570
|
+
}
|
|
8571
|
+
cursor.continue();
|
|
8572
|
+
return;
|
|
8573
|
+
}
|
|
8574
|
+
if (liveCount >= maxExecutions) {
|
|
8575
|
+
execTx.abort();
|
|
8576
|
+
return;
|
|
8577
|
+
}
|
|
8578
|
+
const record = {
|
|
8579
|
+
id: insertedId,
|
|
8580
|
+
queue_name: queueName,
|
|
8581
|
+
executed_at: new Date().toISOString()
|
|
8582
|
+
};
|
|
8583
|
+
for (const [k, v] of Object.entries(this.prefixValues)) {
|
|
8584
|
+
record[k] = v;
|
|
8585
|
+
}
|
|
8586
|
+
const addReq = execStore.add(record);
|
|
8587
|
+
didInsert = true;
|
|
8588
|
+
addReq.onerror = () => {
|
|
8589
|
+
try {
|
|
8590
|
+
execTx.abort();
|
|
8591
|
+
} catch {}
|
|
8592
|
+
reject(addReq.error);
|
|
8593
|
+
};
|
|
8594
|
+
};
|
|
8595
|
+
cursorReq.onerror = () => reject(cursorReq.error);
|
|
8596
|
+
execTx.oncomplete = () => resolve(didInsert ? insertedId : null);
|
|
8597
|
+
execTx.onerror = () => reject(execTx.error);
|
|
8598
|
+
execTx.onabort = () => resolve(null);
|
|
8599
|
+
});
|
|
8600
|
+
}
|
|
8601
|
+
async releaseExecution(queueName, token) {
|
|
8602
|
+
if (token === null || token === undefined)
|
|
8603
|
+
return;
|
|
8604
|
+
const db = await this.getExecutionDb();
|
|
8605
|
+
const tx = db.transaction(this.executionTableName, "readwrite");
|
|
8606
|
+
const store = tx.objectStore(this.executionTableName);
|
|
8607
|
+
return new Promise((resolve, reject) => {
|
|
8608
|
+
const req = store.delete(token);
|
|
8609
|
+
req.onerror = () => reject(req.error);
|
|
8610
|
+
tx.oncomplete = () => resolve();
|
|
8611
|
+
tx.onerror = () => reject(tx.error);
|
|
8612
|
+
});
|
|
8613
|
+
}
|
|
7747
8614
|
async recordExecution(queueName) {
|
|
7748
8615
|
const db = await this.getExecutionDb();
|
|
7749
8616
|
const tx = db.transaction(this.executionTableName, "readwrite");
|
|
@@ -7893,11 +8760,12 @@ class IndexedDbRateLimiterStorage {
|
|
|
7893
8760
|
}
|
|
7894
8761
|
}
|
|
7895
8762
|
// src/queue/IndexedDbQueueStorage.ts
|
|
7896
|
-
import { createServiceToken as createServiceToken30, deepEqual as deepEqual5, makeFingerprint as makeFingerprint10, uuid4 as
|
|
8763
|
+
import { createServiceToken as createServiceToken30, deepEqual as deepEqual5, makeFingerprint as makeFingerprint10, uuid4 as uuid410 } from "@workglow/util";
|
|
7897
8764
|
var INDEXED_DB_QUEUE_STORAGE = createServiceToken30("jobqueue.storage.indexedDb");
|
|
7898
8765
|
|
|
7899
8766
|
class IndexedDbQueueStorage {
|
|
7900
8767
|
queueName;
|
|
8768
|
+
scope = "process";
|
|
7901
8769
|
db;
|
|
7902
8770
|
tableName;
|
|
7903
8771
|
migrationOptions;
|
|
@@ -7974,8 +8842,8 @@ class IndexedDbQueueStorage {
|
|
|
7974
8842
|
const db = await this.getDb();
|
|
7975
8843
|
const now = new Date().toISOString();
|
|
7976
8844
|
const jobWithPrefixes = job;
|
|
7977
|
-
jobWithPrefixes.id = jobWithPrefixes.id ??
|
|
7978
|
-
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ??
|
|
8845
|
+
jobWithPrefixes.id = jobWithPrefixes.id ?? uuid410();
|
|
8846
|
+
jobWithPrefixes.job_run_id = jobWithPrefixes.job_run_id ?? uuid410();
|
|
7979
8847
|
jobWithPrefixes.queue = this.queueName;
|
|
7980
8848
|
jobWithPrefixes.fingerprint = await makeFingerprint10(jobWithPrefixes.input);
|
|
7981
8849
|
jobWithPrefixes.status = JobStatus.PENDING;
|
|
@@ -8472,10 +9340,204 @@ class IndexedDbQueueStorage {
|
|
|
8472
9340
|
}
|
|
8473
9341
|
}
|
|
8474
9342
|
}
|
|
8475
|
-
// src/
|
|
9343
|
+
// src/tabular/SharedInMemoryTabularStorage.ts
|
|
8476
9344
|
import { createServiceToken as createServiceToken31 } from "@workglow/util";
|
|
9345
|
+
var SHARED_IN_MEMORY_TABULAR_REPOSITORY = createServiceToken31("storage.tabularRepository.sharedInMemory");
|
|
9346
|
+
var SYNC_TIMEOUT = 1000;
|
|
9347
|
+
var MAX_PENDING_MESSAGES = 1000;
|
|
9348
|
+
|
|
9349
|
+
class SharedInMemoryTabularStorage extends BaseTabularStorage {
|
|
9350
|
+
channel = null;
|
|
9351
|
+
channelName;
|
|
9352
|
+
inMemoryRepo;
|
|
9353
|
+
isInitialized = false;
|
|
9354
|
+
syncInProgress = false;
|
|
9355
|
+
pendingMessages = [];
|
|
9356
|
+
constructor(channelName = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
|
|
9357
|
+
super(schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
9358
|
+
this.channelName = channelName;
|
|
9359
|
+
this.inMemoryRepo = new InMemoryTabularStorage(schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
9360
|
+
this.setupEventForwarding();
|
|
9361
|
+
this.initializeBroadcastChannel();
|
|
9362
|
+
}
|
|
9363
|
+
isBroadcastChannelAvailable() {
|
|
9364
|
+
return typeof BroadcastChannel !== "undefined";
|
|
9365
|
+
}
|
|
9366
|
+
initializeBroadcastChannel() {
|
|
9367
|
+
if (!this.isBroadcastChannelAvailable()) {
|
|
9368
|
+
console.warn("BroadcastChannel is not available. Tab synchronization will not work.");
|
|
9369
|
+
return;
|
|
9370
|
+
}
|
|
9371
|
+
try {
|
|
9372
|
+
this.channel = new BroadcastChannel(this.channelName);
|
|
9373
|
+
this.channel.onmessage = (event) => {
|
|
9374
|
+
this.handleBroadcastMessage(event.data);
|
|
9375
|
+
};
|
|
9376
|
+
this.syncFromOtherTabs();
|
|
9377
|
+
} catch (error) {
|
|
9378
|
+
console.error("Failed to initialize BroadcastChannel:", error);
|
|
9379
|
+
}
|
|
9380
|
+
}
|
|
9381
|
+
setupEventForwarding() {
|
|
9382
|
+
this.inMemoryRepo.on("put", (entity) => {
|
|
9383
|
+
this.events.emit("put", entity);
|
|
9384
|
+
});
|
|
9385
|
+
this.inMemoryRepo.on("get", (key, entity) => {
|
|
9386
|
+
this.events.emit("get", key, entity);
|
|
9387
|
+
});
|
|
9388
|
+
this.inMemoryRepo.on("query", (key, entities) => {
|
|
9389
|
+
this.events.emit("query", key, entities);
|
|
9390
|
+
});
|
|
9391
|
+
this.inMemoryRepo.on("delete", (key) => {
|
|
9392
|
+
this.events.emit("delete", key);
|
|
9393
|
+
});
|
|
9394
|
+
this.inMemoryRepo.on("clearall", () => {
|
|
9395
|
+
this.events.emit("clearall");
|
|
9396
|
+
});
|
|
9397
|
+
}
|
|
9398
|
+
async handleBroadcastMessage(message) {
|
|
9399
|
+
if (this.syncInProgress && message.type !== "SYNC_RESPONSE") {
|
|
9400
|
+
if (this.pendingMessages.length < MAX_PENDING_MESSAGES) {
|
|
9401
|
+
this.pendingMessages.push(message);
|
|
9402
|
+
}
|
|
9403
|
+
return;
|
|
9404
|
+
}
|
|
9405
|
+
switch (message.type) {
|
|
9406
|
+
case "SYNC_REQUEST":
|
|
9407
|
+
const all = await this.inMemoryRepo.getAll();
|
|
9408
|
+
if (this.channel && all) {
|
|
9409
|
+
this.channel.postMessage({
|
|
9410
|
+
type: "SYNC_RESPONSE",
|
|
9411
|
+
data: all
|
|
9412
|
+
});
|
|
9413
|
+
}
|
|
9414
|
+
break;
|
|
9415
|
+
case "SYNC_RESPONSE":
|
|
9416
|
+
if (message.data && Array.isArray(message.data)) {
|
|
9417
|
+
await this.copyDataFromArray(message.data);
|
|
9418
|
+
}
|
|
9419
|
+
this.syncInProgress = false;
|
|
9420
|
+
await this.drainPendingMessages();
|
|
9421
|
+
break;
|
|
9422
|
+
case "PUT":
|
|
9423
|
+
await this.inMemoryRepo.put(message.entity);
|
|
9424
|
+
break;
|
|
9425
|
+
case "PUT_BULK":
|
|
9426
|
+
await this.inMemoryRepo.putBulk(message.entities);
|
|
9427
|
+
break;
|
|
9428
|
+
case "DELETE":
|
|
9429
|
+
await this.inMemoryRepo.delete(message.key);
|
|
9430
|
+
break;
|
|
9431
|
+
case "DELETE_ALL":
|
|
9432
|
+
await this.inMemoryRepo.deleteAll();
|
|
9433
|
+
break;
|
|
9434
|
+
case "DELETE_SEARCH":
|
|
9435
|
+
await this.inMemoryRepo.deleteSearch(message.criteria);
|
|
9436
|
+
break;
|
|
9437
|
+
}
|
|
9438
|
+
}
|
|
9439
|
+
async drainPendingMessages() {
|
|
9440
|
+
while (!this.syncInProgress && this.pendingMessages.length > 0) {
|
|
9441
|
+
const messages = this.pendingMessages;
|
|
9442
|
+
this.pendingMessages = [];
|
|
9443
|
+
for (const message of messages) {
|
|
9444
|
+
await this.handleBroadcastMessage(message);
|
|
9445
|
+
if (this.syncInProgress) {
|
|
9446
|
+
break;
|
|
9447
|
+
}
|
|
9448
|
+
}
|
|
9449
|
+
}
|
|
9450
|
+
}
|
|
9451
|
+
syncFromOtherTabs() {
|
|
9452
|
+
if (!this.channel)
|
|
9453
|
+
return;
|
|
9454
|
+
this.syncInProgress = true;
|
|
9455
|
+
this.channel.postMessage({ type: "SYNC_REQUEST" });
|
|
9456
|
+
setTimeout(() => {
|
|
9457
|
+
if (this.syncInProgress) {
|
|
9458
|
+
this.syncInProgress = false;
|
|
9459
|
+
this.drainPendingMessages().catch((error) => {
|
|
9460
|
+
console.error("Failed to drain pending messages after sync timeout", error);
|
|
9461
|
+
});
|
|
9462
|
+
}
|
|
9463
|
+
}, SYNC_TIMEOUT);
|
|
9464
|
+
}
|
|
9465
|
+
async copyDataFromArray(entities) {
|
|
9466
|
+
if (entities.length === 0)
|
|
9467
|
+
return;
|
|
9468
|
+
await this.inMemoryRepo.deleteAll();
|
|
9469
|
+
await this.inMemoryRepo.putBulk(entities);
|
|
9470
|
+
}
|
|
9471
|
+
broadcast(message) {
|
|
9472
|
+
if (this.channel) {
|
|
9473
|
+
this.channel.postMessage(message);
|
|
9474
|
+
}
|
|
9475
|
+
}
|
|
9476
|
+
async setupDatabase() {
|
|
9477
|
+
if (this.isInitialized)
|
|
9478
|
+
return;
|
|
9479
|
+
this.isInitialized = true;
|
|
9480
|
+
await this.syncFromOtherTabs();
|
|
9481
|
+
}
|
|
9482
|
+
async put(value) {
|
|
9483
|
+
const result = await this.inMemoryRepo.put(value);
|
|
9484
|
+
this.broadcast({ type: "PUT", entity: result });
|
|
9485
|
+
return result;
|
|
9486
|
+
}
|
|
9487
|
+
async putBulk(values) {
|
|
9488
|
+
const result = await this.inMemoryRepo.putBulk(values);
|
|
9489
|
+
this.broadcast({ type: "PUT_BULK", entities: result });
|
|
9490
|
+
return result;
|
|
9491
|
+
}
|
|
9492
|
+
async get(key) {
|
|
9493
|
+
return await this.inMemoryRepo.get(key);
|
|
9494
|
+
}
|
|
9495
|
+
async delete(value) {
|
|
9496
|
+
await this.inMemoryRepo.delete(value);
|
|
9497
|
+
const { key } = this.separateKeyValueFromCombined(value);
|
|
9498
|
+
this.broadcast({ type: "DELETE", key });
|
|
9499
|
+
}
|
|
9500
|
+
async deleteAll() {
|
|
9501
|
+
await this.inMemoryRepo.deleteAll();
|
|
9502
|
+
this.broadcast({ type: "DELETE_ALL" });
|
|
9503
|
+
}
|
|
9504
|
+
async getAll(options) {
|
|
9505
|
+
return await this.inMemoryRepo.getAll(options);
|
|
9506
|
+
}
|
|
9507
|
+
async size() {
|
|
9508
|
+
return await this.inMemoryRepo.size();
|
|
9509
|
+
}
|
|
9510
|
+
async getBulk(offset, limit) {
|
|
9511
|
+
return await this.inMemoryRepo.getBulk(offset, limit);
|
|
9512
|
+
}
|
|
9513
|
+
async query(criteria, options) {
|
|
9514
|
+
return await this.inMemoryRepo.query(criteria, options);
|
|
9515
|
+
}
|
|
9516
|
+
async queryIndex(criteria, options) {
|
|
9517
|
+
return await this.inMemoryRepo.queryIndex(criteria, options);
|
|
9518
|
+
}
|
|
9519
|
+
async deleteSearch(criteria) {
|
|
9520
|
+
await this.inMemoryRepo.deleteSearch(criteria);
|
|
9521
|
+
this.broadcast({
|
|
9522
|
+
type: "DELETE_SEARCH",
|
|
9523
|
+
criteria
|
|
9524
|
+
});
|
|
9525
|
+
}
|
|
9526
|
+
subscribeToChanges(callback, options) {
|
|
9527
|
+
return this.inMemoryRepo.subscribeToChanges(callback, options);
|
|
9528
|
+
}
|
|
9529
|
+
destroy() {
|
|
9530
|
+
if (this.channel) {
|
|
9531
|
+
this.channel.close();
|
|
9532
|
+
this.channel = null;
|
|
9533
|
+
}
|
|
9534
|
+
this.inMemoryRepo.destroy();
|
|
9535
|
+
}
|
|
9536
|
+
}
|
|
9537
|
+
// src/vector/IndexedDbVectorStorage.ts
|
|
9538
|
+
import { createServiceToken as createServiceToken32 } from "@workglow/util";
|
|
8477
9539
|
import { cosineSimilarity as cosineSimilarity5 } from "@workglow/util/schema";
|
|
8478
|
-
var IDB_VECTOR_REPOSITORY =
|
|
9540
|
+
var IDB_VECTOR_REPOSITORY = createServiceToken32("storage.vectorRepository.indexedDb");
|
|
8479
9541
|
function matchesFilter4(metadata, filter) {
|
|
8480
9542
|
for (const [key, value] of Object.entries(filter)) {
|
|
8481
9543
|
if (metadata[key] !== value) {
|
|
@@ -8573,6 +9635,7 @@ class IndexedDbVectorStorage extends IndexedDbTabularStorage {
|
|
|
8573
9635
|
export {
|
|
8574
9636
|
traced,
|
|
8575
9637
|
registerTabularRepository,
|
|
9638
|
+
pickCoveringIndex,
|
|
8576
9639
|
isSearchCondition,
|
|
8577
9640
|
getVectorProperty,
|
|
8578
9641
|
getTabularRepository,
|
|
@@ -8602,6 +9665,7 @@ export {
|
|
|
8602
9665
|
SqliteQueueStorage,
|
|
8603
9666
|
SqliteKvStorage,
|
|
8604
9667
|
SqliteAiVectorStorage,
|
|
9668
|
+
SharedInMemoryTabularStorage,
|
|
8605
9669
|
SUPABASE_TABULAR_REPOSITORY,
|
|
8606
9670
|
SUPABASE_RATE_LIMITER_STORAGE,
|
|
8607
9671
|
SUPABASE_QUEUE_STORAGE,
|
|
@@ -8610,6 +9674,7 @@ export {
|
|
|
8610
9674
|
SQLITE_RATE_LIMITER_STORAGE,
|
|
8611
9675
|
SQLITE_QUEUE_STORAGE,
|
|
8612
9676
|
SQLITE_KV_REPOSITORY,
|
|
9677
|
+
SHARED_IN_MEMORY_TABULAR_REPOSITORY,
|
|
8613
9678
|
RATE_LIMITER_STORAGE,
|
|
8614
9679
|
QUEUE_STORAGE,
|
|
8615
9680
|
PostgresVectorStorage,
|
|
@@ -8658,9 +9723,10 @@ export {
|
|
|
8658
9723
|
EncryptedKvCredentialStore,
|
|
8659
9724
|
DefaultKeyValueSchema,
|
|
8660
9725
|
DefaultKeyValueKey,
|
|
9726
|
+
CoveringIndexMissingError,
|
|
8661
9727
|
CachedTabularStorage,
|
|
8662
9728
|
CACHED_TABULAR_REPOSITORY,
|
|
8663
9729
|
BaseTabularStorage
|
|
8664
9730
|
};
|
|
8665
9731
|
|
|
8666
|
-
//# debugId=
|
|
9732
|
+
//# debugId=992341815F57641864756E2164756E21
|