@workglow/storage 0.2.5 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +4 -29
- package/dist/browser.js.map +6 -6
- package/dist/bun.js +29 -55
- package/dist/bun.js.map +11 -11
- package/dist/node.js +29 -55
- package/dist/node.js.map +11 -11
- package/dist/tabular/HuggingFaceTabularStorage.d.ts.map +1 -1
- package/dist/tabular/PostgresTabularStorage.d.ts +1 -1
- package/dist/tabular/PostgresTabularStorage.d.ts.map +1 -1
- package/dist/tabular/SqliteTabularStorage.d.ts.map +1 -1
- package/dist/util/IndexedDbTable.d.ts.map +1 -1
- package/dist/vector/InMemoryVectorStorage.d.ts +8 -6
- package/dist/vector/InMemoryVectorStorage.d.ts.map +1 -1
- package/dist/vector/IndexedDbVectorStorage.d.ts +7 -9
- package/dist/vector/IndexedDbVectorStorage.d.ts.map +1 -1
- package/dist/vector/PostgresVectorStorage.d.ts +5 -5
- package/dist/vector/PostgresVectorStorage.d.ts.map +1 -1
- package/dist/vector/SqliteAiVectorStorage.d.ts +6 -6
- package/dist/vector/SqliteAiVectorStorage.d.ts.map +1 -1
- package/dist/vector/SqliteVectorStorage.d.ts +11 -7
- package/dist/vector/SqliteVectorStorage.d.ts.map +1 -1
- package/package.json +5 -5
package/dist/bun.js
CHANGED
|
@@ -2069,13 +2069,11 @@ function textRelevance(text, query) {
|
|
|
2069
2069
|
|
|
2070
2070
|
class InMemoryVectorStorage extends InMemoryTabularStorage {
|
|
2071
2071
|
vectorDimensions;
|
|
2072
|
-
VectorType;
|
|
2073
2072
|
vectorPropertyName;
|
|
2074
2073
|
metadataPropertyName;
|
|
2075
|
-
constructor(schema, primaryKeyNames, indexes = [], dimensions,
|
|
2074
|
+
constructor(schema, primaryKeyNames, indexes = [], dimensions, _vectorCtor = Float32Array) {
|
|
2076
2075
|
super(schema, primaryKeyNames, indexes);
|
|
2077
2076
|
this.vectorDimensions = dimensions;
|
|
2078
|
-
this.VectorType = VectorType;
|
|
2079
2077
|
const vectorProp = getVectorProperty(schema);
|
|
2080
2078
|
if (!vectorProp) {
|
|
2081
2079
|
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
@@ -2963,7 +2961,6 @@ class PostgresTabularStorage extends BaseSqlTabularStorage {
|
|
|
2963
2961
|
const db = this.db;
|
|
2964
2962
|
const columnsToInsert = [];
|
|
2965
2963
|
const paramsToInsert = [];
|
|
2966
|
-
let paramIndex = 1;
|
|
2967
2964
|
const pkColumns = this.primaryKeyColumns();
|
|
2968
2965
|
const entityRecord = entity;
|
|
2969
2966
|
for (const col of pkColumns) {
|
|
@@ -5901,13 +5898,13 @@ var SAFE_IDENTIFIER_RE = /^[a-zA-Z_][a-zA-Z0-9_]*$/;
|
|
|
5901
5898
|
|
|
5902
5899
|
class PostgresVectorStorage extends PostgresTabularStorage {
|
|
5903
5900
|
vectorDimensions;
|
|
5904
|
-
|
|
5901
|
+
vectorCtor;
|
|
5905
5902
|
vectorPropertyName;
|
|
5906
5903
|
metadataPropertyName;
|
|
5907
|
-
constructor(db, table, schema, primaryKeyNames, indexes = [], dimensions,
|
|
5904
|
+
constructor(db, table, schema, primaryKeyNames, indexes = [], dimensions, vectorCtor = Float32Array) {
|
|
5908
5905
|
super(db, table, schema, primaryKeyNames, indexes);
|
|
5909
5906
|
this.vectorDimensions = dimensions;
|
|
5910
|
-
this.
|
|
5907
|
+
this.vectorCtor = vectorCtor;
|
|
5911
5908
|
const vectorProp = getVectorProperty(schema);
|
|
5912
5909
|
if (!vectorProp) {
|
|
5913
5910
|
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
@@ -5955,12 +5952,12 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
5955
5952
|
const result = await this.db.query(sql, params);
|
|
5956
5953
|
const results = [];
|
|
5957
5954
|
for (const row of result.rows) {
|
|
5958
|
-
const vectorResult = await this.db.query(`SELECT ${vectorCol}::text FROM "${this.table}" WHERE ${this.getPrimaryKeyWhereClause(
|
|
5955
|
+
const vectorResult = await this.db.query(`SELECT ${vectorCol}::text FROM "${this.table}" WHERE ${this.getPrimaryKeyWhereClause()}`, this.getPrimaryKeyValues(row));
|
|
5959
5956
|
const vectorStr = vectorResult.rows[0]?.[vectorCol] || "[]";
|
|
5960
5957
|
const vectorArray = JSON.parse(vectorStr);
|
|
5961
5958
|
results.push({
|
|
5962
5959
|
...row,
|
|
5963
|
-
[this.vectorPropertyName]: new this.
|
|
5960
|
+
[this.vectorPropertyName]: new this.vectorCtor(vectorArray),
|
|
5964
5961
|
score: parseFloat(row.score)
|
|
5965
5962
|
});
|
|
5966
5963
|
}
|
|
@@ -6020,12 +6017,12 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
6020
6017
|
const result = await this.db.query(sql, params);
|
|
6021
6018
|
const results = [];
|
|
6022
6019
|
for (const row of result.rows) {
|
|
6023
|
-
const vectorResult = await this.db.query(`SELECT ${vectorCol}::text FROM "${this.table}" WHERE ${this.getPrimaryKeyWhereClause(
|
|
6020
|
+
const vectorResult = await this.db.query(`SELECT ${vectorCol}::text FROM "${this.table}" WHERE ${this.getPrimaryKeyWhereClause()}`, this.getPrimaryKeyValues(row));
|
|
6024
6021
|
const vectorStr = vectorResult.rows[0]?.[vectorCol] || "[]";
|
|
6025
6022
|
const vectorArray = JSON.parse(vectorStr);
|
|
6026
6023
|
results.push({
|
|
6027
6024
|
...row,
|
|
6028
|
-
[this.vectorPropertyName]: new this.
|
|
6025
|
+
[this.vectorPropertyName]: new this.vectorCtor(vectorArray),
|
|
6029
6026
|
score: parseFloat(row.score)
|
|
6030
6027
|
});
|
|
6031
6028
|
}
|
|
@@ -6090,7 +6087,7 @@ class PostgresVectorStorage extends PostgresTabularStorage {
|
|
|
6090
6087
|
const topResults = results.slice(0, topK);
|
|
6091
6088
|
return topResults;
|
|
6092
6089
|
}
|
|
6093
|
-
getPrimaryKeyWhereClause(
|
|
6090
|
+
getPrimaryKeyWhereClause() {
|
|
6094
6091
|
const conditions = this.primaryKeyNames.map((key, idx) => `${String(key)} = $${idx + 1}`);
|
|
6095
6092
|
return conditions.join(" AND ");
|
|
6096
6093
|
}
|
|
@@ -6119,13 +6116,13 @@ function matchesFilter2(metadata, filter) {
|
|
|
6119
6116
|
|
|
6120
6117
|
class SqliteVectorStorage extends SqliteTabularStorage {
|
|
6121
6118
|
vectorDimensions;
|
|
6122
|
-
|
|
6119
|
+
vectorCtor;
|
|
6123
6120
|
vectorPropertyName;
|
|
6124
6121
|
metadataPropertyName;
|
|
6125
|
-
constructor(dbOrPath, table = "vectors", schema, primaryKeyNames, indexes = [], dimensions,
|
|
6122
|
+
constructor(dbOrPath, table = "vectors", schema, primaryKeyNames, indexes = [], dimensions, vectorCtor = Float32Array) {
|
|
6126
6123
|
super(dbOrPath, table, schema, primaryKeyNames, indexes);
|
|
6127
6124
|
this.vectorDimensions = dimensions;
|
|
6128
|
-
this.
|
|
6125
|
+
this.vectorCtor = vectorCtor;
|
|
6129
6126
|
const vectorProp = getVectorProperty(schema);
|
|
6130
6127
|
if (!vectorProp) {
|
|
6131
6128
|
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
@@ -6138,7 +6135,7 @@ class SqliteVectorStorage extends SqliteTabularStorage {
|
|
|
6138
6135
|
}
|
|
6139
6136
|
deserializeVector(vectorJson) {
|
|
6140
6137
|
const array = JSON.parse(vectorJson);
|
|
6141
|
-
return new this.
|
|
6138
|
+
return new this.vectorCtor(array);
|
|
6142
6139
|
}
|
|
6143
6140
|
async similaritySearch(query, options = {}) {
|
|
6144
6141
|
const { topK = 10, filter, scoreThreshold = 0 } = options;
|
|
@@ -6215,10 +6212,10 @@ var VECTOR_TYPE_MAP = {
|
|
|
6215
6212
|
Uint8Array: "u8",
|
|
6216
6213
|
Int16Array: "f16"
|
|
6217
6214
|
};
|
|
6218
|
-
function getVectorTypeSuffix(
|
|
6219
|
-
return VECTOR_TYPE_MAP[
|
|
6215
|
+
function getVectorTypeSuffix(vectorCtor) {
|
|
6216
|
+
return VECTOR_TYPE_MAP[vectorCtor.name] || "f32";
|
|
6220
6217
|
}
|
|
6221
|
-
function getVectorTypeOption(
|
|
6218
|
+
function getVectorTypeOption(vectorCtor) {
|
|
6222
6219
|
const typeMap = {
|
|
6223
6220
|
Float32Array: "FLOAT32",
|
|
6224
6221
|
Float64Array: "FLOAT32",
|
|
@@ -6226,7 +6223,7 @@ function getVectorTypeOption(VectorType) {
|
|
|
6226
6223
|
Uint8Array: "UINT8",
|
|
6227
6224
|
Int16Array: "FLOAT16"
|
|
6228
6225
|
};
|
|
6229
|
-
return typeMap[
|
|
6226
|
+
return typeMap[vectorCtor.name] || "FLOAT32";
|
|
6230
6227
|
}
|
|
6231
6228
|
function matchesFilter3(metadata, filter) {
|
|
6232
6229
|
for (const [key, value] of Object.entries(filter)) {
|
|
@@ -6242,16 +6239,16 @@ function escapeIdentifier(name) {
|
|
|
6242
6239
|
|
|
6243
6240
|
class SqliteAiVectorStorage extends SqliteTabularStorage {
|
|
6244
6241
|
vectorDimensions;
|
|
6245
|
-
|
|
6242
|
+
vectorCtor;
|
|
6246
6243
|
vectorPropertyName;
|
|
6247
6244
|
metadataPropertyName;
|
|
6248
6245
|
vectorTypeSuffix;
|
|
6249
6246
|
extensionLoaded = false;
|
|
6250
|
-
constructor(dbOrPath, table = "vectors", schema, primaryKeyNames, indexes = [], dimensions,
|
|
6247
|
+
constructor(dbOrPath, table = "vectors", schema, primaryKeyNames, indexes = [], dimensions, vectorCtor = Float32Array) {
|
|
6251
6248
|
super(dbOrPath, table, schema, primaryKeyNames, indexes);
|
|
6252
6249
|
this.vectorDimensions = dimensions;
|
|
6253
|
-
this.
|
|
6254
|
-
this.vectorTypeSuffix = getVectorTypeSuffix(
|
|
6250
|
+
this.vectorCtor = vectorCtor;
|
|
6251
|
+
this.vectorTypeSuffix = getVectorTypeSuffix(vectorCtor);
|
|
6255
6252
|
const vectorProp = getVectorProperty(schema);
|
|
6256
6253
|
if (!vectorProp) {
|
|
6257
6254
|
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
@@ -6278,7 +6275,7 @@ class SqliteAiVectorStorage extends SqliteTabularStorage {
|
|
|
6278
6275
|
}
|
|
6279
6276
|
if (this.extensionLoaded) {
|
|
6280
6277
|
const vectorCol = String(this.vectorPropertyName);
|
|
6281
|
-
const vectorType = getVectorTypeOption(this.
|
|
6278
|
+
const vectorType = getVectorTypeOption(this.vectorCtor);
|
|
6282
6279
|
try {
|
|
6283
6280
|
this.database.prepare("SELECT vector_init(?, ?, ?)").run(this.table, vectorCol, `dimension=${this.vectorDimensions},type=${vectorType},distance=COSINE`);
|
|
6284
6281
|
} catch {}
|
|
@@ -6290,18 +6287,18 @@ class SqliteAiVectorStorage extends SqliteTabularStorage {
|
|
|
6290
6287
|
decodeVector(raw) {
|
|
6291
6288
|
if (raw instanceof Uint8Array || typeof Buffer !== "undefined" && raw instanceof Buffer) {
|
|
6292
6289
|
const view = raw instanceof Uint8Array ? raw : new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength);
|
|
6293
|
-
if (this.
|
|
6290
|
+
if (this.vectorCtor.name === "Float32Array" || this.vectorCtor === Float32Array) {
|
|
6294
6291
|
return new Float32Array(view.buffer, view.byteOffset, this.vectorDimensions);
|
|
6295
6292
|
}
|
|
6296
6293
|
const f32 = new Float32Array(view.buffer, view.byteOffset, this.vectorDimensions);
|
|
6297
|
-
return new this.
|
|
6294
|
+
return new this.vectorCtor(Array.from(f32));
|
|
6298
6295
|
}
|
|
6299
6296
|
if (typeof raw === "string") {
|
|
6300
6297
|
const array = JSON.parse(raw);
|
|
6301
|
-
return new this.
|
|
6298
|
+
return new this.vectorCtor(array);
|
|
6302
6299
|
}
|
|
6303
6300
|
if (Array.isArray(raw)) {
|
|
6304
|
-
return new this.
|
|
6301
|
+
return new this.vectorCtor(raw);
|
|
6305
6302
|
}
|
|
6306
6303
|
throw new Error(`Cannot decode vector from type: ${typeof raw}`);
|
|
6307
6304
|
}
|
|
@@ -6612,24 +6609,6 @@ async function saveSchemaMetadata(db, tableName, snapshot) {
|
|
|
6612
6609
|
}
|
|
6613
6610
|
});
|
|
6614
6611
|
}
|
|
6615
|
-
async function loadSchemaMetadata(db, tableName) {
|
|
6616
|
-
return new Promise((resolve) => {
|
|
6617
|
-
try {
|
|
6618
|
-
if (!db.objectStoreNames.contains(METADATA_STORE_NAME)) {
|
|
6619
|
-
resolve(null);
|
|
6620
|
-
return;
|
|
6621
|
-
}
|
|
6622
|
-
const transaction = db.transaction(METADATA_STORE_NAME, "readonly");
|
|
6623
|
-
const store = transaction.objectStore(METADATA_STORE_NAME);
|
|
6624
|
-
const request = store.get(tableName);
|
|
6625
|
-
request.onsuccess = () => resolve(request.result || null);
|
|
6626
|
-
request.onerror = () => resolve(null);
|
|
6627
|
-
transaction.onerror = () => resolve(null);
|
|
6628
|
-
} catch (err) {
|
|
6629
|
-
resolve(null);
|
|
6630
|
-
}
|
|
6631
|
-
});
|
|
6632
|
-
}
|
|
6633
6612
|
async function openIndexedDbTable(tableName, version, upgradeNeededCallback) {
|
|
6634
6613
|
return new Promise((resolve, reject) => {
|
|
6635
6614
|
const openRequest = indexedDB.open(tableName, version);
|
|
@@ -6721,7 +6700,6 @@ async function performIncrementalMigration(db, tableName, diff, options = {}) {
|
|
|
6721
6700
|
db.close();
|
|
6722
6701
|
options.onMigrationProgress?.(`Migrating ${tableName} from version ${currentVersion} to ${newVersion}...`, 0);
|
|
6723
6702
|
return openIndexedDbTable(tableName, newVersion, (event) => {
|
|
6724
|
-
const db2 = event.target.result;
|
|
6725
6703
|
const transaction = event.target.transaction;
|
|
6726
6704
|
const store = transaction.objectStore(tableName);
|
|
6727
6705
|
for (const indexName of diff.indexesToRemove) {
|
|
@@ -6783,7 +6761,6 @@ async function performDestructiveMigration(db, tableName, primaryKey, expectedIn
|
|
|
6783
6761
|
options.onMigrationProgress?.(`Recreating object store...`, 0.75);
|
|
6784
6762
|
const newDb = await openIndexedDbTable(tableName, newVersion, (event) => {
|
|
6785
6763
|
const db2 = event.target.result;
|
|
6786
|
-
const transaction = event.target.transaction;
|
|
6787
6764
|
if (db2.objectStoreNames.contains(tableName)) {
|
|
6788
6765
|
db2.deleteObjectStore(tableName);
|
|
6789
6766
|
}
|
|
@@ -6884,7 +6861,6 @@ async function ensureIndexedDbTable(tableName, primaryKey, expectedIndexes = [],
|
|
|
6884
6861
|
}
|
|
6885
6862
|
});
|
|
6886
6863
|
}
|
|
6887
|
-
const metadata = await loadSchemaMetadata(db, tableName);
|
|
6888
6864
|
if (!db.objectStoreNames.contains(tableName)) {
|
|
6889
6865
|
options.onMigrationProgress?.(`Object store ${tableName} does not exist, creating...`, 0);
|
|
6890
6866
|
db.close();
|
|
@@ -8174,8 +8150,8 @@ class IndexedDbQueueStorage {
|
|
|
8174
8150
|
}
|
|
8175
8151
|
}
|
|
8176
8152
|
// src/vector/IndexedDbVectorStorage.ts
|
|
8177
|
-
import { cosineSimilarity as cosineSimilarity5 } from "@workglow/util/schema";
|
|
8178
8153
|
import { createServiceToken as createServiceToken31 } from "@workglow/util";
|
|
8154
|
+
import { cosineSimilarity as cosineSimilarity5 } from "@workglow/util/schema";
|
|
8179
8155
|
var IDB_VECTOR_REPOSITORY = createServiceToken31("storage.vectorRepository.indexedDb");
|
|
8180
8156
|
function matchesFilter4(metadata, filter) {
|
|
8181
8157
|
for (const [key, value] of Object.entries(filter)) {
|
|
@@ -8203,13 +8179,11 @@ function textRelevance2(text, query) {
|
|
|
8203
8179
|
|
|
8204
8180
|
class IndexedDbVectorStorage extends IndexedDbTabularStorage {
|
|
8205
8181
|
vectorDimensions;
|
|
8206
|
-
VectorType;
|
|
8207
8182
|
vectorPropertyName;
|
|
8208
8183
|
metadataPropertyName;
|
|
8209
|
-
constructor(table = "vectors", schema, primaryKeyNames, indexes = [], dimensions,
|
|
8184
|
+
constructor(table = "vectors", schema, primaryKeyNames, indexes = [], dimensions, _vectorCtor = Float32Array, migrationOptions = {}, clientProvidedKeys = "if-missing") {
|
|
8210
8185
|
super(table, schema, primaryKeyNames, indexes, migrationOptions, clientProvidedKeys);
|
|
8211
8186
|
this.vectorDimensions = dimensions;
|
|
8212
|
-
this.VectorType = VectorType;
|
|
8213
8187
|
const vectorProp = getVectorProperty(schema);
|
|
8214
8188
|
if (!vectorProp) {
|
|
8215
8189
|
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
@@ -8366,4 +8340,4 @@ export {
|
|
|
8366
8340
|
BaseTabularStorage
|
|
8367
8341
|
};
|
|
8368
8342
|
|
|
8369
|
-
//# debugId=
|
|
8343
|
+
//# debugId=F54F1115EFC431E364756E2164756E21
|