@workglow/storage 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +89 -95
- package/dist/browser.js.map +14 -14
- package/dist/bun.js +127 -122
- package/dist/bun.js.map +20 -20
- package/dist/kv/KvViaTabularStorage.d.ts +3 -8
- package/dist/kv/KvViaTabularStorage.d.ts.map +1 -1
- package/dist/node.js +127 -122
- package/dist/node.js.map +20 -20
- package/dist/queue/PostgresQueueStorage.d.ts.map +1 -1
- package/dist/tabular/BaseSqlTabularStorage.d.ts +6 -1
- package/dist/tabular/BaseSqlTabularStorage.d.ts.map +1 -1
- package/dist/tabular/BaseTabularStorage.d.ts +1 -1
- package/dist/tabular/BaseTabularStorage.d.ts.map +1 -1
- package/dist/tabular/CachedTabularStorage.d.ts +1 -1
- package/dist/tabular/CachedTabularStorage.d.ts.map +1 -1
- package/dist/tabular/FsFolderTabularStorage.d.ts +1 -1
- package/dist/tabular/FsFolderTabularStorage.d.ts.map +1 -1
- package/dist/tabular/HuggingFaceTabularStorage.d.ts.map +1 -1
- package/dist/tabular/InMemoryTabularStorage.d.ts +1 -1
- package/dist/tabular/InMemoryTabularStorage.d.ts.map +1 -1
- package/dist/tabular/IndexedDbTabularStorage.d.ts +1 -1
- package/dist/tabular/IndexedDbTabularStorage.d.ts.map +1 -1
- package/dist/tabular/PostgresTabularStorage.d.ts +2 -2
- package/dist/tabular/PostgresTabularStorage.d.ts.map +1 -1
- package/dist/tabular/SharedInMemoryTabularStorage.d.ts +1 -1
- package/dist/tabular/SharedInMemoryTabularStorage.d.ts.map +1 -1
- package/dist/tabular/SqliteTabularStorage.d.ts +1 -1
- package/dist/tabular/SqliteTabularStorage.d.ts.map +1 -1
- package/dist/tabular/SupabaseTabularStorage.d.ts +1 -1
- package/dist/tabular/SupabaseTabularStorage.d.ts.map +1 -1
- package/dist/util/IndexedDbTable.d.ts.map +1 -1
- package/dist/vector/InMemoryVectorStorage.d.ts +8 -6
- package/dist/vector/InMemoryVectorStorage.d.ts.map +1 -1
- package/dist/vector/IndexedDbVectorStorage.d.ts +7 -9
- package/dist/vector/IndexedDbVectorStorage.d.ts.map +1 -1
- package/dist/vector/PostgresVectorStorage.d.ts +5 -5
- package/dist/vector/PostgresVectorStorage.d.ts.map +1 -1
- package/dist/vector/SqliteAiVectorStorage.d.ts +6 -6
- package/dist/vector/SqliteAiVectorStorage.d.ts.map +1 -1
- package/dist/vector/SqliteVectorStorage.d.ts +11 -7
- package/dist/vector/SqliteVectorStorage.d.ts.map +1 -1
- package/package.json +5 -5
package/dist/bun.js
CHANGED
|
@@ -398,30 +398,36 @@ class InMemoryTabularStorage extends BaseTabularStorage {
|
|
|
398
398
|
}
|
|
399
399
|
async put(value) {
|
|
400
400
|
let entityToStore = value;
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
shouldGenerate =
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
401
|
+
const savedCounter = this.autoIncrementCounter;
|
|
402
|
+
try {
|
|
403
|
+
if (this.hasAutoGeneratedKey() && this.autoGeneratedKeyName) {
|
|
404
|
+
const keyName = this.autoGeneratedKeyName;
|
|
405
|
+
const clientProvidedValue = value[keyName];
|
|
406
|
+
const hasClientValue = clientProvidedValue !== undefined && clientProvidedValue !== null;
|
|
407
|
+
let shouldGenerate = false;
|
|
408
|
+
if (this.clientProvidedKeys === "never") {
|
|
409
|
+
shouldGenerate = true;
|
|
410
|
+
} else if (this.clientProvidedKeys === "always") {
|
|
411
|
+
if (!hasClientValue) {
|
|
412
|
+
throw new Error(`Auto-generated key "${keyName}" is required when clientProvidedKeys is "always"`);
|
|
413
|
+
}
|
|
414
|
+
shouldGenerate = false;
|
|
415
|
+
} else {
|
|
416
|
+
shouldGenerate = !hasClientValue;
|
|
417
|
+
}
|
|
418
|
+
if (shouldGenerate) {
|
|
419
|
+
const generatedValue = this.generateKeyValue(keyName, this.autoGeneratedKeyStrategy);
|
|
420
|
+
entityToStore = { ...value, [keyName]: generatedValue };
|
|
411
421
|
}
|
|
412
|
-
shouldGenerate = false;
|
|
413
|
-
} else {
|
|
414
|
-
shouldGenerate = !hasClientValue;
|
|
415
|
-
}
|
|
416
|
-
if (shouldGenerate) {
|
|
417
|
-
const generatedValue = this.generateKeyValue(keyName, this.autoGeneratedKeyStrategy);
|
|
418
|
-
entityToStore = { ...value, [keyName]: generatedValue };
|
|
419
422
|
}
|
|
423
|
+
const { key } = this.separateKeyValueFromCombined(entityToStore);
|
|
424
|
+
const id = await makeFingerprint2(key);
|
|
425
|
+
this._lastPutWasInsert = !this.values.has(id);
|
|
426
|
+
this.values.set(id, entityToStore);
|
|
427
|
+
} catch (e) {
|
|
428
|
+
this.autoIncrementCounter = savedCounter;
|
|
429
|
+
throw e;
|
|
420
430
|
}
|
|
421
|
-
const { key } = this.separateKeyValueFromCombined(entityToStore);
|
|
422
|
-
const id = await makeFingerprint2(key);
|
|
423
|
-
this._lastPutWasInsert = !this.values.has(id);
|
|
424
|
-
this.values.set(id, entityToStore);
|
|
425
431
|
this.events.emit("put", entityToStore);
|
|
426
432
|
return entityToStore;
|
|
427
433
|
}
|
|
@@ -1277,46 +1283,42 @@ class KvStorage {
|
|
|
1277
1283
|
}
|
|
1278
1284
|
|
|
1279
1285
|
// src/kv/KvViaTabularStorage.ts
|
|
1286
|
+
var PRIMITIVE_SCHEMA_TYPES = new Set(["number", "boolean", "string", "blob"]);
|
|
1287
|
+
|
|
1280
1288
|
class KvViaTabularStorage extends KvStorage {
|
|
1289
|
+
get needsJsonSerialization() {
|
|
1290
|
+
if (this._needsJsonSerialization === undefined) {
|
|
1291
|
+
const schemaType = typeof this.valueSchema === "object" && this.valueSchema !== null && "type" in this.valueSchema ? this.valueSchema.type : undefined;
|
|
1292
|
+
this._needsJsonSerialization = !PRIMITIVE_SCHEMA_TYPES.has(schemaType);
|
|
1293
|
+
}
|
|
1294
|
+
return this._needsJsonSerialization;
|
|
1295
|
+
}
|
|
1296
|
+
_needsJsonSerialization;
|
|
1281
1297
|
async setupDatabase() {
|
|
1282
1298
|
await this.tabularRepository.setupDatabase?.();
|
|
1283
1299
|
}
|
|
1284
1300
|
async put(key, value) {
|
|
1285
|
-
|
|
1286
|
-
const shouldStringify = !["number", "boolean", "string", "blob"].includes(schemaType);
|
|
1287
|
-
if (shouldStringify) {
|
|
1301
|
+
if (this.needsJsonSerialization) {
|
|
1288
1302
|
value = JSON.stringify(value);
|
|
1289
1303
|
}
|
|
1290
1304
|
await this.tabularRepository.put({ key, value });
|
|
1291
1305
|
}
|
|
1292
1306
|
async putBulk(items) {
|
|
1293
|
-
const
|
|
1294
|
-
const shouldStringify = !["number", "boolean", "string", "blob"].includes(schemaType);
|
|
1295
|
-
const entities = items.map(({ key, value }) => {
|
|
1296
|
-
if (shouldStringify) {
|
|
1297
|
-
value = JSON.stringify(value);
|
|
1298
|
-
}
|
|
1299
|
-
return { key, value };
|
|
1300
|
-
});
|
|
1307
|
+
const entities = this.needsJsonSerialization ? items.map(({ key, value }) => ({ key, value: JSON.stringify(value) })) : items;
|
|
1301
1308
|
await this.tabularRepository.putBulk(entities);
|
|
1302
1309
|
}
|
|
1303
1310
|
async get(key) {
|
|
1304
1311
|
const result = await this.tabularRepository.get({ key });
|
|
1305
|
-
if (result)
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
} catch (e) {
|
|
1312
|
-
return result.value;
|
|
1313
|
-
}
|
|
1314
|
-
} else {
|
|
1312
|
+
if (!result)
|
|
1313
|
+
return;
|
|
1314
|
+
if (this.needsJsonSerialization) {
|
|
1315
|
+
try {
|
|
1316
|
+
return JSON.parse(result.value);
|
|
1317
|
+
} catch (e) {
|
|
1315
1318
|
return result.value;
|
|
1316
1319
|
}
|
|
1317
|
-
} else {
|
|
1318
|
-
return;
|
|
1319
1320
|
}
|
|
1321
|
+
return result.value;
|
|
1320
1322
|
}
|
|
1321
1323
|
async delete(key) {
|
|
1322
1324
|
return await this.tabularRepository.delete({ key });
|
|
@@ -1327,9 +1329,7 @@ class KvViaTabularStorage extends KvStorage {
|
|
|
1327
1329
|
return values.map((value) => ({
|
|
1328
1330
|
key: value.key,
|
|
1329
1331
|
value: (() => {
|
|
1330
|
-
|
|
1331
|
-
const shouldParse = !["number", "boolean", "string"].includes(schemaType);
|
|
1332
|
-
if (shouldParse && typeof value.value === "string") {
|
|
1332
|
+
if (this.needsJsonSerialization && typeof value.value === "string") {
|
|
1333
1333
|
try {
|
|
1334
1334
|
return JSON.parse(value.value);
|
|
1335
1335
|
} catch (e) {
|
|
@@ -2069,13 +2069,11 @@ function textRelevance(text, query) {
|
|
|
2069
2069
|
|
|
2070
2070
|
class InMemoryVectorStorage extends InMemoryTabularStorage {
|
|
2071
2071
|
vectorDimensions;
|
|
2072
|
-
VectorType;
|
|
2073
2072
|
vectorPropertyName;
|
|
2074
2073
|
metadataPropertyName;
|
|
2075
|
-
constructor(schema, primaryKeyNames, indexes = [], dimensions,
|
|
2074
|
+
constructor(schema, primaryKeyNames, indexes = [], dimensions, _vectorCtor = Float32Array) {
|
|
2076
2075
|
super(schema, primaryKeyNames, indexes);
|
|
2077
2076
|
this.vectorDimensions = dimensions;
|
|
2078
|
-
this.VectorType = VectorType;
|
|
2079
2077
|
const vectorProp = getVectorProperty(schema);
|
|
2080
2078
|
if (!vectorProp) {
|
|
2081
2079
|
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
@@ -2505,31 +2503,40 @@ import { createServiceToken as createServiceToken13 } from "@workglow/util";
|
|
|
2505
2503
|
// src/tabular/BaseSqlTabularStorage.ts
|
|
2506
2504
|
class BaseSqlTabularStorage extends BaseTabularStorage {
|
|
2507
2505
|
table;
|
|
2506
|
+
_pkColsCache = new Map;
|
|
2507
|
+
_valColsCache = new Map;
|
|
2508
|
+
_pkColListCache = new Map;
|
|
2509
|
+
_valColListCache = new Map;
|
|
2508
2510
|
constructor(table = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
|
|
2509
2511
|
super(schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
2510
2512
|
this.table = table;
|
|
2511
2513
|
this.validateTableAndSchema();
|
|
2512
2514
|
}
|
|
2513
2515
|
constructPrimaryKeyColumns($delimiter = "") {
|
|
2514
|
-
|
|
2515
|
-
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2516
|
+
let cached = this._pkColsCache.get($delimiter);
|
|
2517
|
+
if (cached === undefined) {
|
|
2518
|
+
cached = Object.entries(this.primaryKeySchema.properties).map(([key, typeDef]) => {
|
|
2519
|
+
const sqlType = this.mapTypeToSQL(typeDef);
|
|
2520
|
+
return `${$delimiter}${key}${$delimiter} ${sqlType} NOT NULL`;
|
|
2521
|
+
}).join(", ");
|
|
2522
|
+
this._pkColsCache.set($delimiter, cached);
|
|
2523
|
+
}
|
|
2524
|
+
return cached;
|
|
2519
2525
|
}
|
|
2520
2526
|
constructValueColumns($delimiter = "") {
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
const
|
|
2524
|
-
const
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
}
|
|
2527
|
+
let cached = this._valColsCache.get($delimiter);
|
|
2528
|
+
if (cached === undefined) {
|
|
2529
|
+
const requiredSet = new Set(this.valueSchema.required ?? []);
|
|
2530
|
+
const cols = Object.entries(this.valueSchema.properties).map(([key, typeDef]) => {
|
|
2531
|
+
const sqlType = this.mapTypeToSQL(typeDef);
|
|
2532
|
+
const isRequired = requiredSet.has(key);
|
|
2533
|
+
const nullable = !isRequired || this.isNullable(typeDef);
|
|
2534
|
+
return `${$delimiter}${key}${$delimiter} ${sqlType}${nullable ? " NULL" : " NOT NULL"}`;
|
|
2535
|
+
}).join(", ");
|
|
2536
|
+
cached = cols.length > 0 ? `, ${cols}` : "";
|
|
2537
|
+
this._valColsCache.set($delimiter, cached);
|
|
2538
|
+
}
|
|
2539
|
+
return cached;
|
|
2533
2540
|
}
|
|
2534
2541
|
isNullable(typeDef) {
|
|
2535
2542
|
if (typeof typeDef === "boolean")
|
|
@@ -2549,10 +2556,20 @@ class BaseSqlTabularStorage extends BaseTabularStorage {
|
|
|
2549
2556
|
return false;
|
|
2550
2557
|
}
|
|
2551
2558
|
primaryKeyColumnList($delimiter = "") {
|
|
2552
|
-
|
|
2559
|
+
let cached = this._pkColListCache.get($delimiter);
|
|
2560
|
+
if (cached === undefined) {
|
|
2561
|
+
cached = $delimiter + this.primaryKeyColumns().join(`${$delimiter}, ${$delimiter}`) + $delimiter;
|
|
2562
|
+
this._pkColListCache.set($delimiter, cached);
|
|
2563
|
+
}
|
|
2564
|
+
return cached;
|
|
2553
2565
|
}
|
|
2554
2566
|
valueColumnList($delimiter = "") {
|
|
2555
|
-
|
|
2567
|
+
let cached = this._valColListCache.get($delimiter);
|
|
2568
|
+
if (cached === undefined) {
|
|
2569
|
+
cached = $delimiter + this.valueColumns().join(`${$delimiter}, ${$delimiter}`) + $delimiter;
|
|
2570
|
+
this._valColListCache.set($delimiter, cached);
|
|
2571
|
+
}
|
|
2572
|
+
return cached;
|
|
2556
2573
|
}
|
|
2557
2574
|
getNonNullType(typeDef) {
|
|
2558
2575
|
if (typeof typeDef === "boolean")
|
|
@@ -2963,7 +2980,6 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
2963
2980
|
const db = this.db;
|
|
2964
2981
|
const columnsToInsert = [];
|
|
2965
2982
|
const paramsToInsert = [];
|
|
2966
|
-
let paramIndex = 1;
|
|
2967
2983
|
const pkColumns = this.primaryKeyColumns();
|
|
2968
2984
|
const entityRecord = entity;
|
|
2969
2985
|
for (const col of pkColumns) {
|
|
@@ -4288,6 +4304,7 @@ class SupabaseKvStorage extends KvViaTabularStorage {
|
|
|
4288
4304
|
// src/queue/PostgresQueueStorage.ts
|
|
4289
4305
|
import { createServiceToken as createServiceToken21, makeFingerprint as makeFingerprint6, uuid4 as uuid45 } from "@workglow/util";
|
|
4290
4306
|
var POSTGRES_QUEUE_STORAGE = createServiceToken21("jobqueue.storage.postgres");
|
|
4307
|
+
var SAFE_IDENTIFIER = /^[a-zA-Z][a-zA-Z0-9_]*$/;
|
|
4291
4308
|
|
|
4292
4309
|
class PostgresQueueStorage {
|
|
4293
4310
|
db;
|
|
@@ -4300,6 +4317,11 @@ class PostgresQueueStorage {
|
|
|
4300
4317
|
this.queueName = queueName;
|
|
4301
4318
|
this.prefixes = options?.prefixes ?? [];
|
|
4302
4319
|
this.prefixValues = options?.prefixValues ?? {};
|
|
4320
|
+
for (const prefix of this.prefixes) {
|
|
4321
|
+
if (!SAFE_IDENTIFIER.test(prefix.name)) {
|
|
4322
|
+
throw new Error(`Prefix column name must start with a letter and contain only letters, digits, and underscores, got: ${prefix.name}`);
|
|
4323
|
+
}
|
|
4324
|
+
}
|
|
4303
4325
|
if (this.prefixes.length > 0) {
|
|
4304
4326
|
const prefixNames = this.prefixes.map((p) => p.name).join("_");
|
|
4305
4327
|
this.tableName = `job_queue_${prefixNames}`;
|
|
@@ -4334,7 +4356,13 @@ class PostgresQueueStorage {
|
|
|
4334
4356
|
async setupDatabase() {
|
|
4335
4357
|
let sql;
|
|
4336
4358
|
try {
|
|
4337
|
-
|
|
4359
|
+
const enumValues = Object.values(JobStatus);
|
|
4360
|
+
for (const v of enumValues) {
|
|
4361
|
+
if (!SAFE_IDENTIFIER.test(v)) {
|
|
4362
|
+
throw new Error(`Invalid JobStatus enum value: ${v}`);
|
|
4363
|
+
}
|
|
4364
|
+
}
|
|
4365
|
+
sql = `CREATE TYPE job_status AS ENUM (${enumValues.map((v) => `'${v}'`).join(",")})`;
|
|
4338
4366
|
await this.db.query(sql);
|
|
4339
4367
|
} catch (e) {
|
|
4340
4368
|
if (e.code !== "42710")
|
|
@@ -5901,13 +5929,13 @@ var SAFE_IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
|
5901
5929
|
|
|
5902
5930
|
class PostgresVectorStorage extends PostgresTabularStorage {
|
|
5903
5931
|
vectorDimensions;
|
|
5904
|
-
|
|
5932
|
+
vectorCtor;
|
|
5905
5933
|
vectorPropertyName;
|
|
5906
5934
|
metadataPropertyName;
|
|
5907
|
-
constructor(db, table, schema, primaryKeyNames, indexes = [], dimensions,
|
|
5935
|
+
constructor(db, table, schema, primaryKeyNames, indexes = [], dimensions, vectorCtor = Float32Array) {
|
|
5908
5936
|
super(db, table, schema, primaryKeyNames, indexes);
|
|
5909
5937
|
this.vectorDimensions = dimensions;
|
|
5910
|
-
this.
|
|
5938
|
+
this.vectorCtor = vectorCtor;
|
|
5911
5939
|
const vectorProp = getVectorProperty(schema);
|
|
5912
5940
|
if (!vectorProp) {
|
|
5913
5941
|
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
@@ -5955,12 +5983,12 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
5955
5983
|
const result = await this.db.query(sql, params);
|
|
5956
5984
|
const results = [];
|
|
5957
5985
|
for (const row of result.rows) {
|
|
5958
|
-
const vectorResult = await this.db.query(`SELECT ${vectorCol}::text FROM "${this.table}" WHERE ${this.getPrimaryKeyWhereClause(
|
|
5986
|
+
const vectorResult = await this.db.query(`SELECT ${vectorCol}::text FROM "${this.table}" WHERE ${this.getPrimaryKeyWhereClause()}`, this.getPrimaryKeyValues(row));
|
|
5959
5987
|
const vectorStr = vectorResult.rows[0]?.[vectorCol] || "[]";
|
|
5960
5988
|
const vectorArray = JSON.parse(vectorStr);
|
|
5961
5989
|
results.push({
|
|
5962
5990
|
...row,
|
|
5963
|
-
[this.vectorPropertyName]: new this.
|
|
5991
|
+
[this.vectorPropertyName]: new this.vectorCtor(vectorArray),
|
|
5964
5992
|
score: parseFloat(row.score)
|
|
5965
5993
|
});
|
|
5966
5994
|
}
|
|
@@ -6020,12 +6048,12 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
6020
6048
|
const result = await this.db.query(sql, params);
|
|
6021
6049
|
const results = [];
|
|
6022
6050
|
for (const row of result.rows) {
|
|
6023
|
-
const vectorResult = await this.db.query(`SELECT ${vectorCol}::text FROM "${this.table}" WHERE ${this.getPrimaryKeyWhereClause(
|
|
6051
|
+
const vectorResult = await this.db.query(`SELECT ${vectorCol}::text FROM "${this.table}" WHERE ${this.getPrimaryKeyWhereClause()}`, this.getPrimaryKeyValues(row));
|
|
6024
6052
|
const vectorStr = vectorResult.rows[0]?.[vectorCol] || "[]";
|
|
6025
6053
|
const vectorArray = JSON.parse(vectorStr);
|
|
6026
6054
|
results.push({
|
|
6027
6055
|
...row,
|
|
6028
|
-
[this.vectorPropertyName]: new this.
|
|
6056
|
+
[this.vectorPropertyName]: new this.vectorCtor(vectorArray),
|
|
6029
6057
|
score: parseFloat(row.score)
|
|
6030
6058
|
});
|
|
6031
6059
|
}
|
|
@@ -6090,7 +6118,7 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
6090
6118
|
const topResults = results.slice(0, topK);
|
|
6091
6119
|
return topResults;
|
|
6092
6120
|
}
|
|
6093
|
-
getPrimaryKeyWhereClause(
|
|
6121
|
+
getPrimaryKeyWhereClause() {
|
|
6094
6122
|
const conditions = this.primaryKeyNames.map((key, idx) => `${String(key)} = $${idx + 1}`);
|
|
6095
6123
|
return conditions.join(" AND ");
|
|
6096
6124
|
}
|
|
@@ -6119,13 +6147,13 @@ function matchesFilter2(metadata, filter) {
|
|
|
6119
6147
|
|
|
6120
6148
|
class SqliteVectorStorage extends SqliteTabularStorage {
|
|
6121
6149
|
vectorDimensions;
|
|
6122
|
-
|
|
6150
|
+
vectorCtor;
|
|
6123
6151
|
vectorPropertyName;
|
|
6124
6152
|
metadataPropertyName;
|
|
6125
|
-
constructor(dbOrPath, table = "vectors", schema, primaryKeyNames, indexes = [], dimensions,
|
|
6153
|
+
constructor(dbOrPath, table = "vectors", schema, primaryKeyNames, indexes = [], dimensions, vectorCtor = Float32Array) {
|
|
6126
6154
|
super(dbOrPath, table, schema, primaryKeyNames, indexes);
|
|
6127
6155
|
this.vectorDimensions = dimensions;
|
|
6128
|
-
this.
|
|
6156
|
+
this.vectorCtor = vectorCtor;
|
|
6129
6157
|
const vectorProp = getVectorProperty(schema);
|
|
6130
6158
|
if (!vectorProp) {
|
|
6131
6159
|
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
@@ -6138,7 +6166,7 @@ class SqliteVectorStorage extends SqliteTabularStorage {
|
|
|
6138
6166
|
}
|
|
6139
6167
|
deserializeVector(vectorJson) {
|
|
6140
6168
|
const array = JSON.parse(vectorJson);
|
|
6141
|
-
return new this.
|
|
6169
|
+
return new this.vectorCtor(array);
|
|
6142
6170
|
}
|
|
6143
6171
|
async similaritySearch(query, options = {}) {
|
|
6144
6172
|
const { topK = 10, filter, scoreThreshold = 0 } = options;
|
|
@@ -6215,10 +6243,10 @@ var VECTOR_TYPE_MAP = {
|
|
|
6215
6243
|
Uint8Array: "u8",
|
|
6216
6244
|
Int16Array: "f16"
|
|
6217
6245
|
};
|
|
6218
|
-
function getVectorTypeSuffix(
|
|
6219
|
-
return VECTOR_TYPE_MAP[
|
|
6246
|
+
function getVectorTypeSuffix(vectorCtor) {
|
|
6247
|
+
return VECTOR_TYPE_MAP[vectorCtor.name] || "f32";
|
|
6220
6248
|
}
|
|
6221
|
-
function getVectorTypeOption(
|
|
6249
|
+
function getVectorTypeOption(vectorCtor) {
|
|
6222
6250
|
const typeMap = {
|
|
6223
6251
|
Float32Array: "FLOAT32",
|
|
6224
6252
|
Float64Array: "FLOAT32",
|
|
@@ -6226,7 +6254,7 @@ function getVectorTypeOption(VectorType) {
|
|
|
6226
6254
|
Uint8Array: "UINT8",
|
|
6227
6255
|
Int16Array: "FLOAT16"
|
|
6228
6256
|
};
|
|
6229
|
-
return typeMap[
|
|
6257
|
+
return typeMap[vectorCtor.name] || "FLOAT32";
|
|
6230
6258
|
}
|
|
6231
6259
|
function matchesFilter3(metadata, filter) {
|
|
6232
6260
|
for (const [key, value] of Object.entries(filter)) {
|
|
@@ -6242,16 +6270,16 @@ function escapeIdentifier(name) {
|
|
|
6242
6270
|
|
|
6243
6271
|
class SqliteAiVectorStorage extends SqliteTabularStorage {
|
|
6244
6272
|
vectorDimensions;
|
|
6245
|
-
|
|
6273
|
+
vectorCtor;
|
|
6246
6274
|
vectorPropertyName;
|
|
6247
6275
|
metadataPropertyName;
|
|
6248
6276
|
vectorTypeSuffix;
|
|
6249
6277
|
extensionLoaded = false;
|
|
6250
|
-
constructor(dbOrPath, table = "vectors", schema, primaryKeyNames, indexes = [], dimensions,
|
|
6278
|
+
constructor(dbOrPath, table = "vectors", schema, primaryKeyNames, indexes = [], dimensions, vectorCtor = Float32Array) {
|
|
6251
6279
|
super(dbOrPath, table, schema, primaryKeyNames, indexes);
|
|
6252
6280
|
this.vectorDimensions = dimensions;
|
|
6253
|
-
this.
|
|
6254
|
-
this.vectorTypeSuffix = getVectorTypeSuffix(
|
|
6281
|
+
this.vectorCtor = vectorCtor;
|
|
6282
|
+
this.vectorTypeSuffix = getVectorTypeSuffix(vectorCtor);
|
|
6255
6283
|
const vectorProp = getVectorProperty(schema);
|
|
6256
6284
|
if (!vectorProp) {
|
|
6257
6285
|
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
@@ -6278,7 +6306,7 @@ class SqliteAiVectorStorage extends SqliteTabularStorage {
|
|
|
6278
6306
|
}
|
|
6279
6307
|
if (this.extensionLoaded) {
|
|
6280
6308
|
const vectorCol = String(this.vectorPropertyName);
|
|
6281
|
-
const vectorType = getVectorTypeOption(this.
|
|
6309
|
+
const vectorType = getVectorTypeOption(this.vectorCtor);
|
|
6282
6310
|
try {
|
|
6283
6311
|
this.database.prepare("SELECT vector_init(?, ?, ?)").run(this.table, vectorCol, `dimension=${this.vectorDimensions},type=${vectorType},distance=COSINE`);
|
|
6284
6312
|
} catch {}
|
|
@@ -6290,18 +6318,18 @@ class SqliteAiVectorStorage extends SqliteTabularStorage {
|
|
|
6290
6318
|
decodeVector(raw) {
|
|
6291
6319
|
if (raw instanceof Uint8Array || typeof Buffer !== "undefined" && raw instanceof Buffer) {
|
|
6292
6320
|
const view = raw instanceof Uint8Array ? raw : new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength);
|
|
6293
|
-
if (this.
|
|
6321
|
+
if (this.vectorCtor.name === "Float32Array" || this.vectorCtor === Float32Array) {
|
|
6294
6322
|
return new Float32Array(view.buffer, view.byteOffset, this.vectorDimensions);
|
|
6295
6323
|
}
|
|
6296
6324
|
const f32 = new Float32Array(view.buffer, view.byteOffset, this.vectorDimensions);
|
|
6297
|
-
return new this.
|
|
6325
|
+
return new this.vectorCtor(Array.from(f32));
|
|
6298
6326
|
}
|
|
6299
6327
|
if (typeof raw === "string") {
|
|
6300
6328
|
const array = JSON.parse(raw);
|
|
6301
|
-
return new this.
|
|
6329
|
+
return new this.vectorCtor(array);
|
|
6302
6330
|
}
|
|
6303
6331
|
if (Array.isArray(raw)) {
|
|
6304
|
-
return new this.
|
|
6332
|
+
return new this.vectorCtor(raw);
|
|
6305
6333
|
}
|
|
6306
6334
|
throw new Error(`Cannot decode vector from type: ${typeof raw}`);
|
|
6307
6335
|
}
|
|
@@ -6612,24 +6640,6 @@ async function saveSchemaMetadata(db, tableName, snapshot) {
|
|
|
6612
6640
|
}
|
|
6613
6641
|
});
|
|
6614
6642
|
}
|
|
6615
|
-
async function loadSchemaMetadata(db, tableName) {
|
|
6616
|
-
return new Promise((resolve) => {
|
|
6617
|
-
try {
|
|
6618
|
-
if (!db.objectStoreNames.contains(METADATA_STORE_NAME)) {
|
|
6619
|
-
resolve(null);
|
|
6620
|
-
return;
|
|
6621
|
-
}
|
|
6622
|
-
const transaction = db.transaction(METADATA_STORE_NAME, "readonly");
|
|
6623
|
-
const store = transaction.objectStore(METADATA_STORE_NAME);
|
|
6624
|
-
const request = store.get(tableName);
|
|
6625
|
-
request.onsuccess = () => resolve(request.result || null);
|
|
6626
|
-
request.onerror = () => resolve(null);
|
|
6627
|
-
transaction.onerror = () => resolve(null);
|
|
6628
|
-
} catch (err) {
|
|
6629
|
-
resolve(null);
|
|
6630
|
-
}
|
|
6631
|
-
});
|
|
6632
|
-
}
|
|
6633
6643
|
async function openIndexedDbTable(tableName, version, upgradeNeededCallback) {
|
|
6634
6644
|
return new Promise((resolve, reject) => {
|
|
6635
6645
|
const openRequest = indexedDB.open(tableName, version);
|
|
@@ -6721,7 +6731,6 @@ async function performIncrementalMigration(db, tableName, diff, options = {}) {
|
|
|
6721
6731
|
db.close();
|
|
6722
6732
|
options.onMigrationProgress?.(`Migrating ${tableName} from version ${currentVersion} to ${newVersion}...`, 0);
|
|
6723
6733
|
return openIndexedDbTable(tableName, newVersion, (event) => {
|
|
6724
|
-
const db2 = event.target.result;
|
|
6725
6734
|
const transaction = event.target.transaction;
|
|
6726
6735
|
const store = transaction.objectStore(tableName);
|
|
6727
6736
|
for (const indexName of diff.indexesToRemove) {
|
|
@@ -6783,7 +6792,6 @@ async function performDestructiveMigration(db, tableName, primaryKey, expectedIn
|
|
|
6783
6792
|
options.onMigrationProgress?.(`Recreating object store...`, 0.75);
|
|
6784
6793
|
const newDb = await openIndexedDbTable(tableName, newVersion, (event) => {
|
|
6785
6794
|
const db2 = event.target.result;
|
|
6786
|
-
const transaction = event.target.transaction;
|
|
6787
6795
|
if (db2.objectStoreNames.contains(tableName)) {
|
|
6788
6796
|
db2.deleteObjectStore(tableName);
|
|
6789
6797
|
}
|
|
@@ -6884,7 +6892,6 @@ async function ensureIndexedDbTable(tableName, primaryKey, expectedIndexes = [],
|
|
|
6884
6892
|
}
|
|
6885
6893
|
});
|
|
6886
6894
|
}
|
|
6887
|
-
const metadata = await loadSchemaMetadata(db, tableName);
|
|
6888
6895
|
if (!db.objectStoreNames.contains(tableName)) {
|
|
6889
6896
|
options.onMigrationProgress?.(`Object store ${tableName} does not exist, creating...`, 0);
|
|
6890
6897
|
db.close();
|
|
@@ -8174,8 +8181,8 @@ class IndexedDbQueueStorage {
|
|
|
8174
8181
|
}
|
|
8175
8182
|
}
|
|
8176
8183
|
// src/vector/IndexedDbVectorStorage.ts
|
|
8177
|
-
import { cosineSimilarity as cosineSimilarity5 } from "@workglow/util/schema";
|
|
8178
8184
|
import { createServiceToken as createServiceToken31 } from "@workglow/util";
|
|
8185
|
+
import { cosineSimilarity as cosineSimilarity5 } from "@workglow/util/schema";
|
|
8179
8186
|
var IDB_VECTOR_REPOSITORY = createServiceToken31("storage.vectorRepository.indexedDb");
|
|
8180
8187
|
function matchesFilter4(metadata, filter) {
|
|
8181
8188
|
for (const [key, value] of Object.entries(filter)) {
|
|
@@ -8203,13 +8210,11 @@ function textRelevance2(text, query) {
|
|
|
8203
8210
|
|
|
8204
8211
|
class IndexedDbVectorStorage extends IndexedDbTabularStorage {
|
|
8205
8212
|
vectorDimensions;
|
|
8206
|
-
VectorType;
|
|
8207
8213
|
vectorPropertyName;
|
|
8208
8214
|
metadataPropertyName;
|
|
8209
|
-
constructor(table = "vectors", schema, primaryKeyNames, indexes = [], dimensions,
|
|
8215
|
+
constructor(table = "vectors", schema, primaryKeyNames, indexes = [], dimensions, _vectorCtor = Float32Array, migrationOptions = {}, clientProvidedKeys = "if-missing") {
|
|
8210
8216
|
super(table, schema, primaryKeyNames, indexes, migrationOptions, clientProvidedKeys);
|
|
8211
8217
|
this.vectorDimensions = dimensions;
|
|
8212
|
-
this.VectorType = VectorType;
|
|
8213
8218
|
const vectorProp = getVectorProperty(schema);
|
|
8214
8219
|
if (!vectorProp) {
|
|
8215
8220
|
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
@@ -8366,4 +8371,4 @@ export {
|
|
|
8366
8371
|
BaseTabularStorage
|
|
8367
8372
|
};
|
|
8368
8373
|
|
|
8369
|
-
//# debugId=
|
|
8374
|
+
//# debugId=738960D72FA07CB864756E2164756E21
|