@workglow/postgres 0.2.30 → 0.2.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -0
- package/dist/job-queue/PostgresQueueStorage.d.ts +12 -19
- package/dist/job-queue/PostgresQueueStorage.d.ts.map +1 -1
- package/dist/job-queue/PostgresRateLimiterStorage.d.ts +9 -19
- package/dist/job-queue/PostgresRateLimiterStorage.d.ts.map +1 -1
- package/dist/job-queue/browser.js +331 -165
- package/dist/job-queue/browser.js.map +8 -5
- package/dist/job-queue/common.d.ts +3 -0
- package/dist/job-queue/common.d.ts.map +1 -1
- package/dist/job-queue/node.js +331 -165
- package/dist/job-queue/node.js.map +8 -5
- package/dist/migrations/PostgresMigrationRunner.d.ts +31 -0
- package/dist/migrations/PostgresMigrationRunner.d.ts.map +1 -0
- package/dist/migrations/common.d.ts +9 -0
- package/dist/migrations/common.d.ts.map +1 -0
- package/dist/migrations/postgresQueueMigrations.d.ts +18 -0
- package/dist/migrations/postgresQueueMigrations.d.ts.map +1 -0
- package/dist/migrations/postgresRateLimiterMigrations.d.ts +11 -0
- package/dist/migrations/postgresRateLimiterMigrations.d.ts.map +1 -0
- package/dist/storage/PostgresTabularStorage.d.ts +175 -10
- package/dist/storage/PostgresTabularStorage.d.ts.map +1 -1
- package/dist/storage/PostgresVectorStorage.d.ts +52 -2
- package/dist/storage/PostgresVectorStorage.d.ts.map +1 -1
- package/dist/storage/browser.js +454 -75
- package/dist/storage/browser.js.map +4 -4
- package/dist/storage/common.d.ts +1 -0
- package/dist/storage/common.d.ts.map +1 -1
- package/dist/storage/node.js +564 -75
- package/dist/storage/node.js.map +6 -5
- package/package.json +7 -7
package/dist/storage/browser.js
CHANGED
|
@@ -147,26 +147,55 @@ import { createServiceToken as createServiceToken2 } from "@workglow/util";
|
|
|
147
147
|
import { createServiceToken } from "@workglow/util";
|
|
148
148
|
import {
|
|
149
149
|
BaseSqlTabularStorage,
|
|
150
|
-
|
|
150
|
+
buildSearchWhere,
|
|
151
|
+
MIGRATIONS_TABLE,
|
|
152
|
+
PostgresDialect,
|
|
153
|
+
SqlTabularMigrationApplier,
|
|
151
154
|
pickCoveringIndex
|
|
152
155
|
} from "@workglow/storage";
|
|
153
156
|
var POSTGRES_TABULAR_REPOSITORY = createServiceToken("storage.tabularRepository.postgres");
|
|
157
|
+
function assertPositiveInt(value, label) {
|
|
158
|
+
if (value === undefined)
|
|
159
|
+
return;
|
|
160
|
+
if (!Number.isFinite(value) || !Number.isInteger(value) || value <= 0) {
|
|
161
|
+
throw new Error(`VectorIndexOptions.${label} must be a positive integer; received ${String(value)}`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
154
164
|
|
|
155
165
|
class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
156
166
|
db;
|
|
157
|
-
constructor(db, table = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
|
|
158
|
-
super(table, schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
167
|
+
constructor(db, table = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing", tabularMigrations) {
|
|
168
|
+
super(table, schema, primaryKeyNames, indexes, clientProvidedKeys, tabularMigrations, table);
|
|
159
169
|
this.db = db;
|
|
160
170
|
}
|
|
161
171
|
async setupDatabase() {
|
|
172
|
+
if (this.tabularMigrations && this.tabularMigrations.length > 0) {
|
|
173
|
+
const exists = await this.tableExistsAsync();
|
|
174
|
+
await this.createTableAndIndexes();
|
|
175
|
+
await this.applyTabularMigrations({ freshTable: !exists });
|
|
176
|
+
if (exists) {
|
|
177
|
+
await this.createDeclaredIndexes();
|
|
178
|
+
}
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
await this.createTableAndIndexes();
|
|
182
|
+
}
|
|
183
|
+
async tableExistsAsync() {
|
|
184
|
+
const r = await this.db.query(`SELECT 1 AS x FROM information_schema.tables WHERE table_name = $1 LIMIT 1`, [this.table]);
|
|
185
|
+
return r.rows.length > 0;
|
|
186
|
+
}
|
|
187
|
+
async createTableAndIndexes() {
|
|
162
188
|
const sql = `
|
|
163
189
|
CREATE TABLE IF NOT EXISTS "${this.table}" (
|
|
164
190
|
${this.constructPrimaryKeyColumns('"')} ${this.constructValueColumns('"')},
|
|
165
|
-
PRIMARY KEY (${this.primaryKeyColumnList()})
|
|
191
|
+
PRIMARY KEY (${this.primaryKeyColumnList()})
|
|
166
192
|
)
|
|
167
193
|
`;
|
|
168
194
|
await this.db.query(sql);
|
|
169
195
|
await this.createVectorIndexes();
|
|
196
|
+
await this.createDeclaredIndexes();
|
|
197
|
+
}
|
|
198
|
+
async createDeclaredIndexes() {
|
|
170
199
|
const pkColumns = this.primaryKeyColumns();
|
|
171
200
|
const createdIndexes = new Set;
|
|
172
201
|
for (const columns of this.indexes) {
|
|
@@ -190,6 +219,15 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
190
219
|
}
|
|
191
220
|
}
|
|
192
221
|
}
|
|
222
|
+
async runMigrationDdl(sql) {
|
|
223
|
+
await this.db.query(sql);
|
|
224
|
+
}
|
|
225
|
+
async recordMigrationApplied(component, version, description) {
|
|
226
|
+
await this.db.query(`INSERT INTO ${MIGRATIONS_TABLE}(component, version, description) VALUES ($1, $2, $3)`, [component, version, description]);
|
|
227
|
+
}
|
|
228
|
+
getMigrationApplier() {
|
|
229
|
+
return new PostgresTabularMigrationApplierImpl(this);
|
|
230
|
+
}
|
|
193
231
|
isVectorFormat(format) {
|
|
194
232
|
if (!format)
|
|
195
233
|
return false;
|
|
@@ -401,32 +439,73 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
401
439
|
}
|
|
402
440
|
return vectorColumns;
|
|
403
441
|
}
|
|
442
|
+
getVectorIndexOptions() {
|
|
443
|
+
return {};
|
|
444
|
+
}
|
|
404
445
|
async createVectorIndexes() {
|
|
405
446
|
const vectorColumns = this.getVectorColumns();
|
|
406
447
|
if (vectorColumns.length === 0) {
|
|
407
448
|
return;
|
|
408
449
|
}
|
|
450
|
+
const opts = this.getVectorIndexOptions();
|
|
451
|
+
if (opts.hnsw && opts.ivfflat) {
|
|
452
|
+
throw new Error("VectorIndexOptions: only one of `hnsw` or `ivfflat` may be set; received both.");
|
|
453
|
+
}
|
|
454
|
+
assertPositiveInt(opts.hnsw?.m, "hnsw.m");
|
|
455
|
+
assertPositiveInt(opts.hnsw?.efConstruction, "hnsw.efConstruction");
|
|
456
|
+
assertPositiveInt(opts.hnsw?.efSearch, "hnsw.efSearch");
|
|
457
|
+
assertPositiveInt(opts.ivfflat?.lists, "ivfflat.lists");
|
|
458
|
+
assertPositiveInt(opts.ivfflat?.probes, "ivfflat.probes");
|
|
409
459
|
try {
|
|
410
460
|
await this.db.query("CREATE EXTENSION IF NOT EXISTS vector");
|
|
411
461
|
} catch (error) {
|
|
412
462
|
console.warn("pgvector extension not available, vector columns will use TEXT fallback:", error);
|
|
413
463
|
return;
|
|
414
464
|
}
|
|
465
|
+
const distance = opts.distance ?? "cosine";
|
|
466
|
+
const opClass = distance === "l2" ? "vector_l2_ops" : distance === "ip" ? "vector_ip_ops" : "vector_cosine_ops";
|
|
467
|
+
const tableId = PostgresDialect.quoteId(this.table);
|
|
468
|
+
if (opts.ivfflat) {
|
|
469
|
+
const { lists } = opts.ivfflat;
|
|
470
|
+
for (const { column } of vectorColumns) {
|
|
471
|
+
const indexId = PostgresDialect.quoteId(`${this.table}_${column}_ivfflat_idx`);
|
|
472
|
+
const columnId = PostgresDialect.quoteId(column);
|
|
473
|
+
try {
|
|
474
|
+
await this.db.query(`
|
|
475
|
+
CREATE INDEX IF NOT EXISTS ${indexId}
|
|
476
|
+
ON ${tableId}
|
|
477
|
+
USING ivfflat (${columnId} ${opClass})
|
|
478
|
+
WITH (lists = ${lists})
|
|
479
|
+
`);
|
|
480
|
+
} catch (error) {
|
|
481
|
+
console.warn(`Failed to create IVFFlat index on ${column}:`, error);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
return;
|
|
485
|
+
}
|
|
486
|
+
const hnsw = opts.hnsw ?? {};
|
|
487
|
+
const buildParams = [];
|
|
488
|
+
if (typeof hnsw.m === "number")
|
|
489
|
+
buildParams.push(`m = ${hnsw.m}`);
|
|
490
|
+
if (typeof hnsw.efConstruction === "number") {
|
|
491
|
+
buildParams.push(`ef_construction = ${hnsw.efConstruction}`);
|
|
492
|
+
}
|
|
493
|
+
const withClause = buildParams.length > 0 ? ` WITH (${buildParams.join(", ")})` : "";
|
|
415
494
|
for (const { column } of vectorColumns) {
|
|
416
|
-
const
|
|
495
|
+
const indexId = PostgresDialect.quoteId(`${this.table}_${column}_hnsw_idx`);
|
|
496
|
+
const columnId = PostgresDialect.quoteId(column);
|
|
417
497
|
try {
|
|
418
498
|
await this.db.query(`
|
|
419
|
-
CREATE INDEX IF NOT EXISTS
|
|
420
|
-
ON
|
|
421
|
-
USING hnsw (
|
|
499
|
+
CREATE INDEX IF NOT EXISTS ${indexId}
|
|
500
|
+
ON ${tableId}
|
|
501
|
+
USING hnsw (${columnId} ${opClass})${withClause}
|
|
422
502
|
`);
|
|
423
503
|
} catch (error) {
|
|
424
504
|
console.warn(`Failed to create HNSW index on ${column}:`, error);
|
|
425
505
|
}
|
|
426
506
|
}
|
|
427
507
|
}
|
|
428
|
-
|
|
429
|
-
const db = this.db;
|
|
508
|
+
buildPutSql(entity) {
|
|
430
509
|
const columnsToInsert = [];
|
|
431
510
|
const paramsToInsert = [];
|
|
432
511
|
const pkColumns = this.primaryKeyColumns();
|
|
@@ -468,7 +547,7 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
468
547
|
const placeholders = columnsToInsert.map((_, i) => `$${i + 1}`).join(", ");
|
|
469
548
|
const conflictClause = valueColumns.length > 0 ? `
|
|
470
549
|
ON CONFLICT (${this.primaryKeyColumnList('"')}) DO UPDATE
|
|
471
|
-
SET
|
|
550
|
+
SET
|
|
472
551
|
${valueColumns.map((col) => {
|
|
473
552
|
const colIdx = columnsToInsert.indexOf(String(col));
|
|
474
553
|
return `"${col}" = $${colIdx + 1}`;
|
|
@@ -480,22 +559,177 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
480
559
|
${conflictClause}
|
|
481
560
|
RETURNING *
|
|
482
561
|
`;
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
const
|
|
562
|
+
return { sql, params: paramsToInsert };
|
|
563
|
+
}
|
|
564
|
+
hydrateRow(row) {
|
|
565
|
+
const entity = row;
|
|
566
|
+
const record = entity;
|
|
487
567
|
for (const key in this.schema.properties) {
|
|
488
|
-
|
|
568
|
+
record[key] = this.sqlToJsValue(key, record[key]);
|
|
569
|
+
}
|
|
570
|
+
return entity;
|
|
571
|
+
}
|
|
572
|
+
async acquireConnection() {
|
|
573
|
+
const supportsConnect = typeof this.db.connect === "function";
|
|
574
|
+
if (supportsConnect) {
|
|
575
|
+
return await this.db.connect();
|
|
576
|
+
}
|
|
577
|
+
const dbAny = this.db;
|
|
578
|
+
const ctorName = dbAny.constructor?.name;
|
|
579
|
+
const looksLikePGlite = typeof dbAny.exec === "function" && dbAny.waitReady !== undefined;
|
|
580
|
+
const looksLikePGLitePool = ctorName === "PGLitePool";
|
|
581
|
+
if (!looksLikePGlite && !looksLikePGLitePool) {
|
|
582
|
+
throw new Error(`PostgresTabularStorage.putBulk requires a pg.Pool with connect() or a known single-connection wrapper (PGLitePool, PGlite); got ${ctorName ?? typeof this.db}. A multi-connection pool without connect() would dispatch BEGIN and the bracketed INSERTs to different sessions, breaking atomicity.`);
|
|
583
|
+
}
|
|
584
|
+
return {
|
|
585
|
+
query: this.db.query.bind(this.db),
|
|
586
|
+
release: () => {}
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
mutexChain = Promise.resolve();
|
|
590
|
+
get serializeOps() {
|
|
591
|
+
return typeof this.db.connect !== "function";
|
|
592
|
+
}
|
|
593
|
+
async mutex(fn) {
|
|
594
|
+
if (!this.serializeOps)
|
|
595
|
+
return fn();
|
|
596
|
+
const prev = this.mutexChain;
|
|
597
|
+
let release;
|
|
598
|
+
this.mutexChain = new Promise((resolve) => {
|
|
599
|
+
release = resolve;
|
|
600
|
+
});
|
|
601
|
+
await prev;
|
|
602
|
+
try {
|
|
603
|
+
return await fn();
|
|
604
|
+
} finally {
|
|
605
|
+
release();
|
|
489
606
|
}
|
|
490
|
-
|
|
607
|
+
}
|
|
608
|
+
inTransaction = false;
|
|
609
|
+
emitPut(entity) {
|
|
610
|
+
this.events.emit("put", entity);
|
|
611
|
+
}
|
|
612
|
+
async put(entity) {
|
|
613
|
+
return this.mutex(() => this._putInternal(entity));
|
|
614
|
+
}
|
|
615
|
+
async _putInternal(entity) {
|
|
616
|
+
const { sql, params } = this.buildPutSql(entity);
|
|
617
|
+
const result = await this.db.query(sql, params);
|
|
618
|
+
const updatedEntity = this.hydrateRow(result.rows[0]);
|
|
619
|
+
this.emitPut(updatedEntity);
|
|
491
620
|
return updatedEntity;
|
|
492
621
|
}
|
|
493
622
|
async putBulk(entities) {
|
|
623
|
+
return this.mutex(() => this._putBulkInternal(entities));
|
|
624
|
+
}
|
|
625
|
+
async _putBulkInternal(entities) {
|
|
494
626
|
if (entities.length === 0)
|
|
495
627
|
return [];
|
|
496
|
-
|
|
628
|
+
if (this.inTransaction) {
|
|
629
|
+
const updated = [];
|
|
630
|
+
for (const entity of entities) {
|
|
631
|
+
const { sql, params } = this.buildPutSql(entity);
|
|
632
|
+
const result = await this.db.query(sql, params);
|
|
633
|
+
updated.push(this.hydrateRow(result.rows[0]));
|
|
634
|
+
}
|
|
635
|
+
for (const entity of updated)
|
|
636
|
+
this.emitPut(entity);
|
|
637
|
+
return updated;
|
|
638
|
+
}
|
|
639
|
+
const conn = await this.acquireConnection();
|
|
640
|
+
const updatedEntities = [];
|
|
641
|
+
try {
|
|
642
|
+
await conn.query("BEGIN");
|
|
643
|
+
try {
|
|
644
|
+
for (const entity of entities) {
|
|
645
|
+
const { sql, params } = this.buildPutSql(entity);
|
|
646
|
+
const result = await conn.query(sql, params);
|
|
647
|
+
updatedEntities.push(this.hydrateRow(result.rows[0]));
|
|
648
|
+
}
|
|
649
|
+
await conn.query("COMMIT");
|
|
650
|
+
} catch (err) {
|
|
651
|
+
try {
|
|
652
|
+
await conn.query("ROLLBACK");
|
|
653
|
+
} catch {}
|
|
654
|
+
throw err;
|
|
655
|
+
}
|
|
656
|
+
} finally {
|
|
657
|
+
conn.release();
|
|
658
|
+
}
|
|
659
|
+
for (const entity of updatedEntities)
|
|
660
|
+
this.emitPut(entity);
|
|
661
|
+
return updatedEntities;
|
|
662
|
+
}
|
|
663
|
+
createTxView(txDb, deferredPutEvents) {
|
|
664
|
+
const target = this;
|
|
665
|
+
return new Proxy(target, {
|
|
666
|
+
get(t, prop, receiver) {
|
|
667
|
+
if (prop === "withTransaction") {
|
|
668
|
+
return () => {
|
|
669
|
+
throw new Error("PostgresTabularStorage.withTransaction does not support nesting. " + "Use SAVEPOINT directly or refactor to a single transaction.");
|
|
670
|
+
};
|
|
671
|
+
}
|
|
672
|
+
if (prop === "db")
|
|
673
|
+
return txDb;
|
|
674
|
+
if (prop === "inTransaction")
|
|
675
|
+
return true;
|
|
676
|
+
if (prop === "emitPut") {
|
|
677
|
+
return (entity) => deferredPutEvents.push(entity);
|
|
678
|
+
}
|
|
679
|
+
if (typeof prop === "string") {
|
|
680
|
+
const internal = t[`_${prop}Internal`];
|
|
681
|
+
if (typeof internal === "function") {
|
|
682
|
+
return (...args) => internal.apply(receiver, args);
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
const value = Reflect.get(t, prop, receiver);
|
|
686
|
+
return typeof value === "function" ? value.bind(receiver) : value;
|
|
687
|
+
}
|
|
688
|
+
});
|
|
689
|
+
}
|
|
690
|
+
async withTransaction(fn) {
|
|
691
|
+
const supportsConnect = typeof this.db.connect === "function";
|
|
692
|
+
if (supportsConnect) {
|
|
693
|
+
const client = await this.db.connect();
|
|
694
|
+
try {
|
|
695
|
+
return await this.runInTransaction(fn, { query: client.query.bind(client) });
|
|
696
|
+
} finally {
|
|
697
|
+
client.release();
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
if (this.inTransaction) {
|
|
701
|
+
throw new Error("PostgresTabularStorage.withTransaction does not support nesting. " + "Use SAVEPOINT directly or refactor to a single transaction.");
|
|
702
|
+
}
|
|
703
|
+
return this.mutex(async () => {
|
|
704
|
+
this.inTransaction = true;
|
|
705
|
+
try {
|
|
706
|
+
return await this.runInTransaction(fn, { query: this.db.query.bind(this.db) });
|
|
707
|
+
} finally {
|
|
708
|
+
this.inTransaction = false;
|
|
709
|
+
}
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
async runInTransaction(fn, txDb) {
|
|
713
|
+
const deferredPutEvents = [];
|
|
714
|
+
await txDb.query("BEGIN");
|
|
715
|
+
let result;
|
|
716
|
+
try {
|
|
717
|
+
result = await fn(this.createTxView(txDb, deferredPutEvents));
|
|
718
|
+
await txDb.query("COMMIT");
|
|
719
|
+
} catch (err) {
|
|
720
|
+
try {
|
|
721
|
+
await txDb.query("ROLLBACK");
|
|
722
|
+
} catch {}
|
|
723
|
+
throw err;
|
|
724
|
+
}
|
|
725
|
+
for (const entity of deferredPutEvents)
|
|
726
|
+
this.events.emit("put", entity);
|
|
727
|
+
return result;
|
|
497
728
|
}
|
|
498
729
|
async get(key) {
|
|
730
|
+
return this.mutex(() => this._getInternal(key));
|
|
731
|
+
}
|
|
732
|
+
async _getInternal(key) {
|
|
499
733
|
const db = this.db;
|
|
500
734
|
const whereClauses = this.primaryKeyColumns().map((discriminatorKey, i) => `"${discriminatorKey}" = $${i + 1}`).join(" AND ");
|
|
501
735
|
const sql = `SELECT * FROM "${this.table}" WHERE ${whereClauses}`;
|
|
@@ -515,6 +749,9 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
515
749
|
return val;
|
|
516
750
|
}
|
|
517
751
|
async delete(value) {
|
|
752
|
+
return this.mutex(() => this._deleteInternal(value));
|
|
753
|
+
}
|
|
754
|
+
async _deleteInternal(value) {
|
|
518
755
|
const db = this.db;
|
|
519
756
|
const { key } = this.separateKeyValueFromCombined(value);
|
|
520
757
|
const whereClauses = this.primaryKeyColumns().map((key2, i) => `${key2} = $${i + 1}`).join(" AND ");
|
|
@@ -523,6 +760,9 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
523
760
|
this.events.emit("delete", key);
|
|
524
761
|
}
|
|
525
762
|
async getAll(options) {
|
|
763
|
+
return this.mutex(() => this._getAllInternal(options));
|
|
764
|
+
}
|
|
765
|
+
async _getAllInternal(options) {
|
|
526
766
|
this.validateGetAllOptions(options);
|
|
527
767
|
const db = this.db;
|
|
528
768
|
let sql = `SELECT * FROM "${this.table}"`;
|
|
@@ -552,18 +792,27 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
552
792
|
return;
|
|
553
793
|
}
|
|
554
794
|
async deleteAll() {
|
|
795
|
+
return this.mutex(() => this._deleteAllInternal());
|
|
796
|
+
}
|
|
797
|
+
async _deleteAllInternal() {
|
|
555
798
|
const db = this.db;
|
|
556
799
|
await db.query(`DELETE FROM "${this.table}"`);
|
|
557
800
|
this.events.emit("clearall");
|
|
558
801
|
}
|
|
559
802
|
async size() {
|
|
803
|
+
return this.mutex(() => this._sizeInternal());
|
|
804
|
+
}
|
|
805
|
+
async _sizeInternal() {
|
|
560
806
|
const db = this.db;
|
|
561
807
|
const result = await db.query(`SELECT COUNT(*) FROM "${this.table}"`);
|
|
562
808
|
return parseInt(result.rows[0].count, 10);
|
|
563
809
|
}
|
|
564
810
|
async count(criteria) {
|
|
811
|
+
return this.mutex(() => this._countInternal(criteria));
|
|
812
|
+
}
|
|
813
|
+
async _countInternal(criteria) {
|
|
565
814
|
if (!criteria || Object.keys(criteria).length === 0) {
|
|
566
|
-
return await this.
|
|
815
|
+
return await this._sizeInternal();
|
|
567
816
|
}
|
|
568
817
|
this.validateQueryParams(criteria);
|
|
569
818
|
const db = this.db;
|
|
@@ -572,6 +821,9 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
572
821
|
return parseInt(result.rows[0].count, 10);
|
|
573
822
|
}
|
|
574
823
|
async getBulk(offset, limit) {
|
|
824
|
+
return this.mutex(() => this._getBulkInternal(offset, limit));
|
|
825
|
+
}
|
|
826
|
+
async _getBulkInternal(offset, limit) {
|
|
575
827
|
const db = this.db;
|
|
576
828
|
const orderByClause = this.primaryKeyColumns().map((col) => `"${String(col)}"`).join(", ");
|
|
577
829
|
const result = await db.query(`SELECT * FROM "${this.table}" ORDER BY ${orderByClause} LIMIT $1 OFFSET $2`, [limit, offset]);
|
|
@@ -587,32 +839,52 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
587
839
|
return result.rows;
|
|
588
840
|
}
|
|
589
841
|
buildDeleteSearchWhere(criteria) {
|
|
590
|
-
const
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
params.push(this.jsToSqlValue(column, value));
|
|
608
|
-
paramIndex++;
|
|
609
|
-
}
|
|
842
|
+
const built = buildSearchWhere(PostgresDialect, criteria, this.schema.properties, (column, value) => this.jsToSqlValue(column, value));
|
|
843
|
+
return { whereClause: built.whereClause, params: built.params };
|
|
844
|
+
}
|
|
845
|
+
async getPage(request = {}) {
|
|
846
|
+
return this.mutex(() => this._getPageInternal(request));
|
|
847
|
+
}
|
|
848
|
+
async _getPageInternal(request) {
|
|
849
|
+
return this.runSqlPage(undefined, request, this.postgresDialect());
|
|
850
|
+
}
|
|
851
|
+
async queryPage(criteria, request = {}) {
|
|
852
|
+
return this.mutex(() => this._queryPageInternal(criteria, request));
|
|
853
|
+
}
|
|
854
|
+
async _queryPageInternal(criteria, request) {
|
|
855
|
+
this.validateQueryParams(criteria, undefined);
|
|
856
|
+
return this.runSqlPage(criteria, request, this.postgresDialect());
|
|
857
|
+
}
|
|
858
|
+
postgresDialect() {
|
|
610
859
|
return {
|
|
611
|
-
|
|
612
|
-
|
|
860
|
+
quote: '"',
|
|
861
|
+
placeholder: (index) => `$${index}`,
|
|
862
|
+
buildSearchWhere: (criteria, startIndex) => this.buildSearchWhereWithIndex(criteria, startIndex),
|
|
863
|
+
executeSelect: async (sql, params) => {
|
|
864
|
+
const result = await this.db.query(sql, params);
|
|
865
|
+
const rows = result.rows ?? [];
|
|
866
|
+
for (const row of rows) {
|
|
867
|
+
const record = row;
|
|
868
|
+
for (const k in this.schema.properties) {
|
|
869
|
+
record[k] = this.sqlToJsValue(k, record[k]);
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
return rows;
|
|
873
|
+
}
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
buildSearchWhereWithIndex(criteria, startIndex) {
|
|
877
|
+
const built = buildSearchWhere(PostgresDialect, criteria, this.schema.properties, (column, value) => this.jsToSqlValue(column, value), startIndex);
|
|
878
|
+
return {
|
|
879
|
+
whereClause: built.whereClause,
|
|
880
|
+
params: built.params,
|
|
881
|
+
nextIndex: startIndex + built.params.length
|
|
613
882
|
};
|
|
614
883
|
}
|
|
615
884
|
async deleteSearch(criteria) {
|
|
885
|
+
return this.mutex(() => this._deleteSearchInternal(criteria));
|
|
886
|
+
}
|
|
887
|
+
async _deleteSearchInternal(criteria) {
|
|
616
888
|
const criteriaKeys = Object.keys(criteria);
|
|
617
889
|
if (criteriaKeys.length === 0) {
|
|
618
890
|
return;
|
|
@@ -623,6 +895,9 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
623
895
|
this.events.emit("delete", criteriaKeys[0]);
|
|
624
896
|
}
|
|
625
897
|
async query(criteria, options) {
|
|
898
|
+
return this.mutex(() => this._queryInternal(criteria, options));
|
|
899
|
+
}
|
|
900
|
+
async _queryInternal(criteria, options) {
|
|
626
901
|
this.validateQueryParams(criteria, options);
|
|
627
902
|
const db = this.db;
|
|
628
903
|
let sql = `SELECT * FROM "${this.table}"`;
|
|
@@ -655,6 +930,9 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
655
930
|
return;
|
|
656
931
|
}
|
|
657
932
|
async queryIndex(criteria, options) {
|
|
933
|
+
return this.mutex(() => this._queryIndexInternal(criteria, options));
|
|
934
|
+
}
|
|
935
|
+
async _queryIndexInternal(criteria, options) {
|
|
658
936
|
this.validateSelect(options);
|
|
659
937
|
this.validateQueryParams(criteria, options);
|
|
660
938
|
const registered = this.indexes.map((cols2, i) => {
|
|
@@ -705,6 +983,51 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
705
983
|
}
|
|
706
984
|
}
|
|
707
985
|
|
|
986
|
+
class PostgresTabularMigrationApplierImpl extends SqlTabularMigrationApplier {
|
|
987
|
+
host;
|
|
988
|
+
constructor(host) {
|
|
989
|
+
super();
|
|
990
|
+
this.host = host;
|
|
991
|
+
}
|
|
992
|
+
dialectName() {
|
|
993
|
+
return "postgres";
|
|
994
|
+
}
|
|
995
|
+
table() {
|
|
996
|
+
return this.host.table;
|
|
997
|
+
}
|
|
998
|
+
storage() {
|
|
999
|
+
return this.host;
|
|
1000
|
+
}
|
|
1001
|
+
mapTypeToSQL(typeDef) {
|
|
1002
|
+
return this.host.mapTypeToSQL(typeDef);
|
|
1003
|
+
}
|
|
1004
|
+
isNullableSchema(typeDef) {
|
|
1005
|
+
return this.host.isNullable(typeDef);
|
|
1006
|
+
}
|
|
1007
|
+
async executeSql(sql) {
|
|
1008
|
+
await this.host.db.query(sql);
|
|
1009
|
+
}
|
|
1010
|
+
async executeSqlTx(sql, tx) {
|
|
1011
|
+
await tx.runMigrationDdl(sql);
|
|
1012
|
+
}
|
|
1013
|
+
async recordAppliedTx(component, version, description, tx) {
|
|
1014
|
+
await tx.recordMigrationApplied(component, version, description ?? null);
|
|
1015
|
+
}
|
|
1016
|
+
async recordApplied(component, version, description) {
|
|
1017
|
+
await this.host.db.query(`INSERT INTO ${MIGRATIONS_TABLE}(component, version, description) VALUES ($1, $2, $3)`, [component, version, description ?? null]);
|
|
1018
|
+
}
|
|
1019
|
+
async queryAppliedVersions(component) {
|
|
1020
|
+
const r = await this.host.db.query(`SELECT version FROM ${MIGRATIONS_TABLE} WHERE component = $1`, [component]);
|
|
1021
|
+
return new Set(r.rows.map((row) => Number(row.version)));
|
|
1022
|
+
}
|
|
1023
|
+
async probeTableExists() {
|
|
1024
|
+
const r = await this.host.db.query(`SELECT 1 FROM information_schema.tables WHERE table_name = $1 LIMIT 1`, [
|
|
1025
|
+
this.host.table
|
|
1026
|
+
]);
|
|
1027
|
+
return r.rows.length > 0;
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
|
|
708
1031
|
// src/storage/PostgresKvStorage.ts
|
|
709
1032
|
import {
|
|
710
1033
|
DefaultKeyValueKey,
|
|
@@ -726,7 +1049,12 @@ class PostgresKvStorage extends KvViaTabularStorage {
|
|
|
726
1049
|
}
|
|
727
1050
|
// src/storage/PostgresVectorStorage.ts
|
|
728
1051
|
import { cosineSimilarity } from "@workglow/util/schema";
|
|
729
|
-
import {
|
|
1052
|
+
import {
|
|
1053
|
+
PostgresDialect as PostgresDialect2,
|
|
1054
|
+
StorageValidationError,
|
|
1055
|
+
getMetadataProperty,
|
|
1056
|
+
getVectorProperty
|
|
1057
|
+
} from "@workglow/storage";
|
|
730
1058
|
var SAFE_IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
731
1059
|
|
|
732
1060
|
class PostgresVectorStorage extends PostgresTabularStorage {
|
|
@@ -734,10 +1062,12 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
734
1062
|
vectorCtor;
|
|
735
1063
|
vectorPropertyName;
|
|
736
1064
|
metadataPropertyName;
|
|
737
|
-
|
|
1065
|
+
indexOptions;
|
|
1066
|
+
constructor(db, table, schema, primaryKeyNames, indexes = [], dimensions, vectorCtor = Float32Array, indexOptions = {}) {
|
|
738
1067
|
super(db, table, schema, primaryKeyNames, indexes);
|
|
739
1068
|
this.vectorDimensions = dimensions;
|
|
740
1069
|
this.vectorCtor = vectorCtor;
|
|
1070
|
+
this.indexOptions = indexOptions;
|
|
741
1071
|
const vectorProp = getVectorProperty(schema);
|
|
742
1072
|
if (!vectorProp) {
|
|
743
1073
|
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
@@ -748,20 +1078,58 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
748
1078
|
getVectorDimensions() {
|
|
749
1079
|
return this.vectorDimensions;
|
|
750
1080
|
}
|
|
1081
|
+
getVectorIndexOptions() {
|
|
1082
|
+
return this.indexOptions;
|
|
1083
|
+
}
|
|
1084
|
+
get distance() {
|
|
1085
|
+
return this.indexOptions.distance ?? "cosine";
|
|
1086
|
+
}
|
|
1087
|
+
get distanceOperator() {
|
|
1088
|
+
switch (this.distance) {
|
|
1089
|
+
case "l2":
|
|
1090
|
+
return "<->";
|
|
1091
|
+
case "ip":
|
|
1092
|
+
return "<#>";
|
|
1093
|
+
default:
|
|
1094
|
+
return "<=>";
|
|
1095
|
+
}
|
|
1096
|
+
}
|
|
1097
|
+
buildScoreExpr(vectorExpr, queryParam) {
|
|
1098
|
+
const op = this.distanceOperator;
|
|
1099
|
+
switch (this.distance) {
|
|
1100
|
+
case "l2":
|
|
1101
|
+
return `(1.0 / (1.0 + (${vectorExpr} ${op} ${queryParam}::vector)))`;
|
|
1102
|
+
case "ip":
|
|
1103
|
+
return `(-1.0 * (${vectorExpr} ${op} ${queryParam}::vector))`;
|
|
1104
|
+
default:
|
|
1105
|
+
return `(1 - (${vectorExpr} ${op} ${queryParam}::vector))`;
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
getQueryTuning() {
|
|
1109
|
+
return {
|
|
1110
|
+
efSearch: this.indexOptions.hnsw?.efSearch,
|
|
1111
|
+
probes: this.indexOptions.ivfflat?.probes
|
|
1112
|
+
};
|
|
1113
|
+
}
|
|
751
1114
|
async similaritySearch(query, options = {}) {
|
|
752
1115
|
const { topK = 10, filter, scoreThreshold = 0 } = options;
|
|
753
1116
|
try {
|
|
754
1117
|
const queryVector = `[${Array.from(query).join(",")}]`;
|
|
755
|
-
const
|
|
756
|
-
const
|
|
1118
|
+
const vectorColRaw = String(this.vectorPropertyName);
|
|
1119
|
+
const vectorCol = PostgresDialect2.quoteId(vectorColRaw);
|
|
1120
|
+
const metadataColRaw = this.metadataPropertyName ? String(this.metadataPropertyName) : null;
|
|
1121
|
+
const metadataCol = metadataColRaw ? PostgresDialect2.quoteId(metadataColRaw) : null;
|
|
1122
|
+
const distOp = this.distanceOperator;
|
|
1123
|
+
const scoreExpr = this.buildScoreExpr(vectorCol, "$1");
|
|
757
1124
|
let sql = `
|
|
758
|
-
SELECT
|
|
1125
|
+
SELECT
|
|
759
1126
|
*,
|
|
760
|
-
|
|
1127
|
+
${scoreExpr} as score
|
|
761
1128
|
FROM "${this.table}"
|
|
762
1129
|
`;
|
|
763
1130
|
const params = [queryVector];
|
|
764
1131
|
let paramIndex = 2;
|
|
1132
|
+
let hasWhere = false;
|
|
765
1133
|
if (filter && Object.keys(filter).length > 0 && metadataCol) {
|
|
766
1134
|
const conditions = [];
|
|
767
1135
|
for (const [key, value] of Object.entries(filter)) {
|
|
@@ -772,21 +1140,22 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
772
1140
|
params.push(String(value));
|
|
773
1141
|
paramIndex++;
|
|
774
1142
|
}
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
sql += ` (1 - (${vectorCol} <=> $1::vector)) >= $${paramIndex}`;
|
|
780
|
-
params.push(scoreThreshold);
|
|
781
|
-
paramIndex++;
|
|
1143
|
+
if (conditions.length > 0) {
|
|
1144
|
+
sql += ` WHERE ${conditions.join(" AND ")}`;
|
|
1145
|
+
hasWhere = true;
|
|
1146
|
+
}
|
|
782
1147
|
}
|
|
783
|
-
sql +=
|
|
1148
|
+
sql += hasWhere ? " AND" : " WHERE";
|
|
1149
|
+
sql += ` ${scoreExpr} >= $${paramIndex}`;
|
|
1150
|
+
params.push(scoreThreshold);
|
|
1151
|
+
paramIndex++;
|
|
1152
|
+
sql += ` ORDER BY ${vectorCol} ${distOp} $1::vector LIMIT $${paramIndex}`;
|
|
784
1153
|
params.push(topK);
|
|
785
1154
|
const result = await this.db.query(sql, params);
|
|
786
1155
|
const results = [];
|
|
787
1156
|
for (const row of result.rows) {
|
|
788
1157
|
const vectorResult = await this.db.query(`SELECT ${vectorCol}::text FROM "${this.table}" WHERE ${this.getPrimaryKeyWhereClause()}`, this.getPrimaryKeyValues(row));
|
|
789
|
-
const vectorStr = vectorResult.rows[0]?.[
|
|
1158
|
+
const vectorStr = vectorResult.rows[0]?.[vectorColRaw] || "[]";
|
|
790
1159
|
const vectorArray = JSON.parse(vectorStr);
|
|
791
1160
|
results.push({
|
|
792
1161
|
...row,
|
|
@@ -811,19 +1180,24 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
811
1180
|
try {
|
|
812
1181
|
const queryVector = `[${Array.from(query).join(",")}]`;
|
|
813
1182
|
const tsQueryText = textQuery;
|
|
814
|
-
const
|
|
815
|
-
const
|
|
1183
|
+
const vectorColRaw = String(this.vectorPropertyName);
|
|
1184
|
+
const vectorCol = PostgresDialect2.quoteId(vectorColRaw);
|
|
1185
|
+
const metadataColRaw = this.metadataPropertyName ? String(this.metadataPropertyName) : null;
|
|
1186
|
+
const metadataCol = metadataColRaw ? PostgresDialect2.quoteId(metadataColRaw) : null;
|
|
1187
|
+
const vectorScoreExpr = this.buildScoreExpr(vectorCol, "$1");
|
|
1188
|
+
const combinedScoreExpr = `(
|
|
1189
|
+
$2 * ${vectorScoreExpr} +
|
|
1190
|
+
$3 * ts_rank(to_tsvector('english', ${metadataCol || "''"}::text), plainto_tsquery('english', $4))
|
|
1191
|
+
)`;
|
|
816
1192
|
let sql = `
|
|
817
|
-
SELECT
|
|
1193
|
+
SELECT
|
|
818
1194
|
*,
|
|
819
|
-
|
|
820
|
-
$2 * (1 - (${vectorCol} <=> $1::vector)) +
|
|
821
|
-
$3 * ts_rank(to_tsvector('english', ${metadataCol || "''"}::text), plainto_tsquery('english', $4))
|
|
822
|
-
) as score
|
|
1195
|
+
${combinedScoreExpr} as score
|
|
823
1196
|
FROM "${this.table}"
|
|
824
1197
|
`;
|
|
825
1198
|
const params = [queryVector, vectorWeight, 1 - vectorWeight, tsQueryText];
|
|
826
1199
|
let paramIndex = 5;
|
|
1200
|
+
let hasWhere = false;
|
|
827
1201
|
if (filter && Object.keys(filter).length > 0 && metadataCol) {
|
|
828
1202
|
const conditions = [];
|
|
829
1203
|
for (const [key, value] of Object.entries(filter)) {
|
|
@@ -834,24 +1208,22 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
834
1208
|
params.push(String(value));
|
|
835
1209
|
paramIndex++;
|
|
836
1210
|
}
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
sql += ` (
|
|
842
|
-
$2 * (1 - (${vectorCol} <=> $1::vector)) +
|
|
843
|
-
$3 * ts_rank(to_tsvector('english', ${metadataCol || "''"}::text), plainto_tsquery('english', $4))
|
|
844
|
-
) >= $${paramIndex}`;
|
|
845
|
-
params.push(scoreThreshold);
|
|
846
|
-
paramIndex++;
|
|
1211
|
+
if (conditions.length > 0) {
|
|
1212
|
+
sql += ` WHERE ${conditions.join(" AND ")}`;
|
|
1213
|
+
hasWhere = true;
|
|
1214
|
+
}
|
|
847
1215
|
}
|
|
1216
|
+
sql += hasWhere ? " AND" : " WHERE";
|
|
1217
|
+
sql += ` ${combinedScoreExpr} >= $${paramIndex}`;
|
|
1218
|
+
params.push(scoreThreshold);
|
|
1219
|
+
paramIndex++;
|
|
848
1220
|
sql += ` ORDER BY score DESC LIMIT $${paramIndex}`;
|
|
849
1221
|
params.push(topK);
|
|
850
1222
|
const result = await this.db.query(sql, params);
|
|
851
1223
|
const results = [];
|
|
852
1224
|
for (const row of result.rows) {
|
|
853
1225
|
const vectorResult = await this.db.query(`SELECT ${vectorCol}::text FROM "${this.table}" WHERE ${this.getPrimaryKeyWhereClause()}`, this.getPrimaryKeyValues(row));
|
|
854
|
-
const vectorStr = vectorResult.rows[0]?.[
|
|
1226
|
+
const vectorStr = vectorResult.rows[0]?.[vectorColRaw] || "[]";
|
|
855
1227
|
const vectorArray = JSON.parse(vectorStr);
|
|
856
1228
|
results.push({
|
|
857
1229
|
...row,
|
|
@@ -868,7 +1240,13 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
868
1240
|
return this.hybridSearchFallback(query, options);
|
|
869
1241
|
}
|
|
870
1242
|
}
|
|
1243
|
+
assertFallbackSupportsDistance() {
|
|
1244
|
+
if (this.distance !== "cosine") {
|
|
1245
|
+
throw new Error(`PostgresVectorStorage: pgvector is unavailable and the in-memory ` + `fallback only supports cosine distance (configured: "${this.distance}"). ` + `Install pgvector or switch the storage to cosine distance.`);
|
|
1246
|
+
}
|
|
1247
|
+
}
|
|
871
1248
|
async searchFallback(query, options) {
|
|
1249
|
+
this.assertFallbackSupportsDistance();
|
|
872
1250
|
const { topK = 10, filter, scoreThreshold = 0 } = options;
|
|
873
1251
|
const allRows = await this.getAll() || [];
|
|
874
1252
|
const results = [];
|
|
@@ -888,6 +1266,7 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
888
1266
|
return topResults;
|
|
889
1267
|
}
|
|
890
1268
|
async hybridSearchFallback(query, options) {
|
|
1269
|
+
this.assertFallbackSupportsDistance();
|
|
891
1270
|
const { topK = 10, filter, scoreThreshold = 0, textQuery, vectorWeight = 0.7 } = options;
|
|
892
1271
|
const allRows = await this.getAll() || [];
|
|
893
1272
|
const results = [];
|
|
@@ -948,4 +1327,4 @@ export {
|
|
|
948
1327
|
POSTGRES_KV_REPOSITORY
|
|
949
1328
|
};
|
|
950
1329
|
|
|
951
|
-
//# debugId=
|
|
1330
|
+
//# debugId=9A981229758C242364756E2164756E21
|