@workglow/storage 0.0.115 → 0.0.116
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +224 -1
- package/dist/browser.js.map +9 -4
- package/dist/bun.js +620 -7
- package/dist/bun.js.map +11 -5
- package/dist/common-server.d.ts +1 -0
- package/dist/common-server.d.ts.map +1 -1
- package/dist/common.d.ts +5 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/kv/TelemetryKvStorage.d.ts +36 -0
- package/dist/kv/TelemetryKvStorage.d.ts.map +1 -0
- package/dist/node.js +621 -7
- package/dist/node.js.map +11 -5
- package/dist/queue/TelemetryQueueStorage.d.ts +31 -0
- package/dist/queue/TelemetryQueueStorage.d.ts.map +1 -0
- package/dist/tabular/SqliteTabularStorage.d.ts +2 -0
- package/dist/tabular/SqliteTabularStorage.d.ts.map +1 -1
- package/dist/tabular/TelemetryTabularStorage.d.ts +39 -0
- package/dist/tabular/TelemetryTabularStorage.d.ts.map +1 -0
- package/dist/util/traced.d.ts +10 -0
- package/dist/util/traced.d.ts.map +1 -0
- package/dist/vector/IVectorStorage.d.ts +2 -2
- package/dist/vector/IVectorStorage.d.ts.map +1 -1
- package/dist/vector/SqliteAiVectorStorage.d.ts +106 -0
- package/dist/vector/SqliteAiVectorStorage.d.ts.map +1 -0
- package/dist/vector/TelemetryVectorStorage.d.ts +25 -0
- package/dist/vector/TelemetryVectorStorage.d.ts.map +1 -0
- package/package.json +10 -6
package/dist/bun.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
// @bun
|
|
2
|
+
var __require = import.meta.require;
|
|
3
|
+
|
|
2
4
|
// src/tabular/BaseTabularStorage.ts
|
|
3
5
|
import {
|
|
4
6
|
createServiceToken,
|
|
@@ -1139,6 +1141,102 @@ function resolveRepositoryFromRegistry(id, format, registry) {
|
|
|
1139
1141
|
return repo;
|
|
1140
1142
|
}
|
|
1141
1143
|
registerInputResolver("storage:tabular", resolveRepositoryFromRegistry);
|
|
1144
|
+
// src/util/traced.ts
|
|
1145
|
+
import { getTelemetryProvider, SpanStatusCode } from "@workglow/util";
|
|
1146
|
+
async function traced(spanName, storageName, fn) {
|
|
1147
|
+
const telemetry = getTelemetryProvider();
|
|
1148
|
+
if (!telemetry.isEnabled)
|
|
1149
|
+
return fn();
|
|
1150
|
+
const span = telemetry.startSpan(spanName, {
|
|
1151
|
+
attributes: { "workglow.storage.name": storageName }
|
|
1152
|
+
});
|
|
1153
|
+
try {
|
|
1154
|
+
const result = await fn();
|
|
1155
|
+
span.setStatus(SpanStatusCode.OK);
|
|
1156
|
+
return result;
|
|
1157
|
+
} catch (err) {
|
|
1158
|
+
span.setStatus(SpanStatusCode.ERROR, err instanceof Error ? err.message : String(err));
|
|
1159
|
+
throw err;
|
|
1160
|
+
} finally {
|
|
1161
|
+
span.end();
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
// src/tabular/TelemetryTabularStorage.ts
|
|
1166
|
+
class TelemetryTabularStorage {
|
|
1167
|
+
storageName;
|
|
1168
|
+
inner;
|
|
1169
|
+
constructor(storageName, inner) {
|
|
1170
|
+
this.storageName = storageName;
|
|
1171
|
+
this.inner = inner;
|
|
1172
|
+
}
|
|
1173
|
+
put(value) {
|
|
1174
|
+
return traced("workglow.storage.tabular.put", this.storageName, () => this.inner.put(value));
|
|
1175
|
+
}
|
|
1176
|
+
putBulk(values) {
|
|
1177
|
+
return traced("workglow.storage.tabular.putBulk", this.storageName, () => this.inner.putBulk(values));
|
|
1178
|
+
}
|
|
1179
|
+
get(key) {
|
|
1180
|
+
return traced("workglow.storage.tabular.get", this.storageName, () => this.inner.get(key));
|
|
1181
|
+
}
|
|
1182
|
+
delete(key) {
|
|
1183
|
+
return traced("workglow.storage.tabular.delete", this.storageName, () => this.inner.delete(key));
|
|
1184
|
+
}
|
|
1185
|
+
getAll(options) {
|
|
1186
|
+
return traced("workglow.storage.tabular.getAll", this.storageName, () => this.inner.getAll(options));
|
|
1187
|
+
}
|
|
1188
|
+
deleteAll() {
|
|
1189
|
+
return traced("workglow.storage.tabular.deleteAll", this.storageName, () => this.inner.deleteAll());
|
|
1190
|
+
}
|
|
1191
|
+
size() {
|
|
1192
|
+
return traced("workglow.storage.tabular.size", this.storageName, () => this.inner.size());
|
|
1193
|
+
}
|
|
1194
|
+
deleteSearch(criteria) {
|
|
1195
|
+
return traced("workglow.storage.tabular.deleteSearch", this.storageName, () => this.inner.deleteSearch(criteria));
|
|
1196
|
+
}
|
|
1197
|
+
getBulk(offset, limit) {
|
|
1198
|
+
return traced("workglow.storage.tabular.getBulk", this.storageName, () => this.inner.getBulk(offset, limit));
|
|
1199
|
+
}
|
|
1200
|
+
query(criteria, options) {
|
|
1201
|
+
return traced("workglow.storage.tabular.query", this.storageName, () => this.inner.query(criteria, options));
|
|
1202
|
+
}
|
|
1203
|
+
records(pageSize) {
|
|
1204
|
+
return this.inner.records(pageSize);
|
|
1205
|
+
}
|
|
1206
|
+
pages(pageSize) {
|
|
1207
|
+
return this.inner.pages(pageSize);
|
|
1208
|
+
}
|
|
1209
|
+
subscribeToChanges(callback, options) {
|
|
1210
|
+
return this.inner.subscribeToChanges(callback, options);
|
|
1211
|
+
}
|
|
1212
|
+
setupDatabase() {
|
|
1213
|
+
return this.inner.setupDatabase();
|
|
1214
|
+
}
|
|
1215
|
+
destroy() {
|
|
1216
|
+
return this.inner.destroy();
|
|
1217
|
+
}
|
|
1218
|
+
[Symbol.dispose]() {
|
|
1219
|
+
return this.inner[Symbol.dispose]();
|
|
1220
|
+
}
|
|
1221
|
+
[Symbol.asyncDispose]() {
|
|
1222
|
+
return this.inner[Symbol.asyncDispose]();
|
|
1223
|
+
}
|
|
1224
|
+
on(name, fn) {
|
|
1225
|
+
this.inner.on(name, fn);
|
|
1226
|
+
}
|
|
1227
|
+
off(name, fn) {
|
|
1228
|
+
this.inner.off(name, fn);
|
|
1229
|
+
}
|
|
1230
|
+
emit(name, ...args) {
|
|
1231
|
+
this.inner.emit(name, ...args);
|
|
1232
|
+
}
|
|
1233
|
+
once(name, fn) {
|
|
1234
|
+
this.inner.once(name, fn);
|
|
1235
|
+
}
|
|
1236
|
+
waitOn(name) {
|
|
1237
|
+
return this.inner.waitOn(name);
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1142
1240
|
// src/kv/IKvStorage.ts
|
|
1143
1241
|
var DefaultKeyValueSchema = {
|
|
1144
1242
|
type: "object",
|
|
@@ -1270,6 +1368,54 @@ class InMemoryKvStorage extends KvViaTabularStorage {
|
|
|
1270
1368
|
this.tabularRepository = new InMemoryTabularStorage(DefaultKeyValueSchema, DefaultKeyValueKey);
|
|
1271
1369
|
}
|
|
1272
1370
|
}
|
|
1371
|
+
// src/kv/TelemetryKvStorage.ts
|
|
1372
|
+
class TelemetryKvStorage {
|
|
1373
|
+
storageName;
|
|
1374
|
+
inner;
|
|
1375
|
+
constructor(storageName, inner) {
|
|
1376
|
+
this.storageName = storageName;
|
|
1377
|
+
this.inner = inner;
|
|
1378
|
+
}
|
|
1379
|
+
put(key, value) {
|
|
1380
|
+
return traced("workglow.storage.kv.put", this.storageName, () => this.inner.put(key, value));
|
|
1381
|
+
}
|
|
1382
|
+
putBulk(items) {
|
|
1383
|
+
return traced("workglow.storage.kv.putBulk", this.storageName, () => this.inner.putBulk(items));
|
|
1384
|
+
}
|
|
1385
|
+
get(key) {
|
|
1386
|
+
return traced("workglow.storage.kv.get", this.storageName, () => this.inner.get(key));
|
|
1387
|
+
}
|
|
1388
|
+
delete(key) {
|
|
1389
|
+
return traced("workglow.storage.kv.delete", this.storageName, () => this.inner.delete(key));
|
|
1390
|
+
}
|
|
1391
|
+
getAll() {
|
|
1392
|
+
return traced("workglow.storage.kv.getAll", this.storageName, () => this.inner.getAll());
|
|
1393
|
+
}
|
|
1394
|
+
deleteAll() {
|
|
1395
|
+
return traced("workglow.storage.kv.deleteAll", this.storageName, () => this.inner.deleteAll());
|
|
1396
|
+
}
|
|
1397
|
+
size() {
|
|
1398
|
+
return traced("workglow.storage.kv.size", this.storageName, () => this.inner.size());
|
|
1399
|
+
}
|
|
1400
|
+
getObjectAsIdString(object) {
|
|
1401
|
+
return this.inner.getObjectAsIdString(object);
|
|
1402
|
+
}
|
|
1403
|
+
on(name, fn) {
|
|
1404
|
+
this.inner.on(name, fn);
|
|
1405
|
+
}
|
|
1406
|
+
off(name, fn) {
|
|
1407
|
+
this.inner.off(name, fn);
|
|
1408
|
+
}
|
|
1409
|
+
emit(name, ...args) {
|
|
1410
|
+
this.inner.emit(name, ...args);
|
|
1411
|
+
}
|
|
1412
|
+
once(name, fn) {
|
|
1413
|
+
this.inner.once(name, fn);
|
|
1414
|
+
}
|
|
1415
|
+
waitOn(name) {
|
|
1416
|
+
return this.inner.waitOn(name);
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1273
1419
|
// src/queue/InMemoryQueueStorage.ts
|
|
1274
1420
|
import {
|
|
1275
1421
|
createServiceToken as createServiceToken9,
|
|
@@ -1481,6 +1627,60 @@ class InMemoryQueueStorage {
|
|
|
1481
1627
|
return this.events.subscribe("change", filteredCallback);
|
|
1482
1628
|
}
|
|
1483
1629
|
}
|
|
1630
|
+
// src/queue/TelemetryQueueStorage.ts
|
|
1631
|
+
class TelemetryQueueStorage {
|
|
1632
|
+
storageName;
|
|
1633
|
+
inner;
|
|
1634
|
+
constructor(storageName, inner) {
|
|
1635
|
+
this.storageName = storageName;
|
|
1636
|
+
this.inner = inner;
|
|
1637
|
+
}
|
|
1638
|
+
add(job) {
|
|
1639
|
+
return traced("workglow.storage.queue.add", this.storageName, () => this.inner.add(job));
|
|
1640
|
+
}
|
|
1641
|
+
get(id) {
|
|
1642
|
+
return traced("workglow.storage.queue.get", this.storageName, () => this.inner.get(id));
|
|
1643
|
+
}
|
|
1644
|
+
next(workerId) {
|
|
1645
|
+
return traced("workglow.storage.queue.next", this.storageName, () => this.inner.next(workerId));
|
|
1646
|
+
}
|
|
1647
|
+
peek(status, num) {
|
|
1648
|
+
return traced("workglow.storage.queue.peek", this.storageName, () => this.inner.peek(status, num));
|
|
1649
|
+
}
|
|
1650
|
+
size(status) {
|
|
1651
|
+
return traced("workglow.storage.queue.size", this.storageName, () => this.inner.size(status));
|
|
1652
|
+
}
|
|
1653
|
+
complete(job) {
|
|
1654
|
+
return traced("workglow.storage.queue.complete", this.storageName, () => this.inner.complete(job));
|
|
1655
|
+
}
|
|
1656
|
+
deleteAll() {
|
|
1657
|
+
return traced("workglow.storage.queue.deleteAll", this.storageName, () => this.inner.deleteAll());
|
|
1658
|
+
}
|
|
1659
|
+
outputForInput(input) {
|
|
1660
|
+
return traced("workglow.storage.queue.outputForInput", this.storageName, () => this.inner.outputForInput(input));
|
|
1661
|
+
}
|
|
1662
|
+
abort(id) {
|
|
1663
|
+
return traced("workglow.storage.queue.abort", this.storageName, () => this.inner.abort(id));
|
|
1664
|
+
}
|
|
1665
|
+
getByRunId(runId) {
|
|
1666
|
+
return traced("workglow.storage.queue.getByRunId", this.storageName, () => this.inner.getByRunId(runId));
|
|
1667
|
+
}
|
|
1668
|
+
saveProgress(id, progress, message, details) {
|
|
1669
|
+
return traced("workglow.storage.queue.saveProgress", this.storageName, () => this.inner.saveProgress(id, progress, message, details));
|
|
1670
|
+
}
|
|
1671
|
+
delete(id) {
|
|
1672
|
+
return traced("workglow.storage.queue.delete", this.storageName, () => this.inner.delete(id));
|
|
1673
|
+
}
|
|
1674
|
+
deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
1675
|
+
return traced("workglow.storage.queue.deleteJobsByStatusAndAge", this.storageName, () => this.inner.deleteJobsByStatusAndAge(status, olderThanMs));
|
|
1676
|
+
}
|
|
1677
|
+
setupDatabase() {
|
|
1678
|
+
return this.inner.setupDatabase();
|
|
1679
|
+
}
|
|
1680
|
+
subscribeToChanges(callback, options) {
|
|
1681
|
+
return this.inner.subscribeToChanges(callback, options);
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1484
1684
|
// src/queue-limiter/InMemoryRateLimiterStorage.ts
|
|
1485
1685
|
import { createServiceToken as createServiceToken10, sleep as sleep2 } from "@workglow/util";
|
|
1486
1686
|
var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken10("ratelimiter.storage.inMemory");
|
|
@@ -1934,6 +2134,26 @@ class InMemoryVectorStorage extends InMemoryTabularStorage {
|
|
|
1934
2134
|
return topResults;
|
|
1935
2135
|
}
|
|
1936
2136
|
}
|
|
2137
|
+
// src/vector/TelemetryVectorStorage.ts
|
|
2138
|
+
class TelemetryVectorStorage extends TelemetryTabularStorage {
|
|
2139
|
+
vectorInner;
|
|
2140
|
+
constructor(storageName, inner) {
|
|
2141
|
+
super(storageName, inner);
|
|
2142
|
+
this.vectorInner = inner;
|
|
2143
|
+
}
|
|
2144
|
+
getVectorDimensions() {
|
|
2145
|
+
return this.vectorInner.getVectorDimensions();
|
|
2146
|
+
}
|
|
2147
|
+
similaritySearch(query, options) {
|
|
2148
|
+
return traced("workglow.storage.vector.similaritySearch", this.storageName, () => this.vectorInner.similaritySearch(query, options));
|
|
2149
|
+
}
|
|
2150
|
+
hybridSearch(query, options) {
|
|
2151
|
+
if (!this.vectorInner.hybridSearch) {
|
|
2152
|
+
throw new Error("hybridSearch is not supported by the underlying storage implementation");
|
|
2153
|
+
}
|
|
2154
|
+
return traced("workglow.storage.vector.hybridSearch", this.storageName, () => this.vectorInner.hybridSearch(query, options));
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
1937
2157
|
// src/credentials/EncryptedKvCredentialStore.ts
|
|
1938
2158
|
import { decrypt, encrypt } from "@workglow/util";
|
|
1939
2159
|
|
|
@@ -2913,6 +3133,9 @@ var Database = Sqlite.Database;
|
|
|
2913
3133
|
|
|
2914
3134
|
class SqliteTabularStorage extends BaseSqlTabularStorage {
|
|
2915
3135
|
db;
|
|
3136
|
+
get database() {
|
|
3137
|
+
return this.db;
|
|
3138
|
+
}
|
|
2916
3139
|
constructor(dbOrPath, table = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
|
|
2917
3140
|
super(table, schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
2918
3141
|
if (typeof dbOrPath === "string") {
|
|
@@ -5897,6 +6120,390 @@ class SqliteVectorStorage extends SqliteTabularStorage {
|
|
|
5897
6120
|
return topResults;
|
|
5898
6121
|
}
|
|
5899
6122
|
}
|
|
6123
|
+
// src/vector/SqliteAiVectorStorage.ts
|
|
6124
|
+
import { cosineSimilarity as cosineSimilarity4 } from "@workglow/util";
|
|
6125
|
+
var VECTOR_TYPE_MAP = {
|
|
6126
|
+
Float32Array: "f32",
|
|
6127
|
+
Float64Array: "f32",
|
|
6128
|
+
Int8Array: "i8",
|
|
6129
|
+
Uint8Array: "u8",
|
|
6130
|
+
Int16Array: "f16"
|
|
6131
|
+
};
|
|
6132
|
+
function getVectorTypeSuffix(VectorType) {
|
|
6133
|
+
return VECTOR_TYPE_MAP[VectorType.name] || "f32";
|
|
6134
|
+
}
|
|
6135
|
+
function getVectorTypeOption(VectorType) {
|
|
6136
|
+
const typeMap = {
|
|
6137
|
+
Float32Array: "FLOAT32",
|
|
6138
|
+
Float64Array: "FLOAT32",
|
|
6139
|
+
Int8Array: "INT8",
|
|
6140
|
+
Uint8Array: "UINT8",
|
|
6141
|
+
Int16Array: "FLOAT16"
|
|
6142
|
+
};
|
|
6143
|
+
return typeMap[VectorType.name] || "FLOAT32";
|
|
6144
|
+
}
|
|
6145
|
+
function matchesFilter3(metadata, filter) {
|
|
6146
|
+
for (const [key, value] of Object.entries(filter)) {
|
|
6147
|
+
if (metadata[key] !== value) {
|
|
6148
|
+
return false;
|
|
6149
|
+
}
|
|
6150
|
+
}
|
|
6151
|
+
return true;
|
|
6152
|
+
}
|
|
6153
|
+
function escapeIdentifier(name) {
|
|
6154
|
+
return "`" + name.replace(/`/g, "``") + "`";
|
|
6155
|
+
}
|
|
6156
|
+
|
|
6157
|
+
class SqliteAiVectorStorage extends SqliteTabularStorage {
|
|
6158
|
+
vectorDimensions;
|
|
6159
|
+
VectorType;
|
|
6160
|
+
vectorPropertyName;
|
|
6161
|
+
metadataPropertyName;
|
|
6162
|
+
vectorTypeSuffix;
|
|
6163
|
+
extensionLoaded = false;
|
|
6164
|
+
constructor(dbOrPath, table = "vectors", schema, primaryKeyNames, indexes = [], dimensions, VectorType = Float32Array) {
|
|
6165
|
+
super(dbOrPath, table, schema, primaryKeyNames, indexes);
|
|
6166
|
+
this.vectorDimensions = dimensions;
|
|
6167
|
+
this.VectorType = VectorType;
|
|
6168
|
+
this.vectorTypeSuffix = getVectorTypeSuffix(VectorType);
|
|
6169
|
+
const vectorProp = getVectorProperty(schema);
|
|
6170
|
+
if (!vectorProp) {
|
|
6171
|
+
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
6172
|
+
}
|
|
6173
|
+
this.vectorPropertyName = vectorProp;
|
|
6174
|
+
this.metadataPropertyName = getMetadataProperty(schema);
|
|
6175
|
+
}
|
|
6176
|
+
getVectorDimensions() {
|
|
6177
|
+
return this.vectorDimensions;
|
|
6178
|
+
}
|
|
6179
|
+
async setupDatabase() {
|
|
6180
|
+
await super.setupDatabase();
|
|
6181
|
+
if (!this.extensionLoaded) {
|
|
6182
|
+
try {
|
|
6183
|
+
const { getExtensionPath } = await import("@sqliteai/sqlite-vector");
|
|
6184
|
+
this.database.loadExtension(getExtensionPath());
|
|
6185
|
+
this.extensionLoaded = true;
|
|
6186
|
+
} catch {
|
|
6187
|
+
try {
|
|
6188
|
+
this.database.exec("SELECT vector_version()");
|
|
6189
|
+
this.extensionLoaded = true;
|
|
6190
|
+
} catch {}
|
|
6191
|
+
}
|
|
6192
|
+
}
|
|
6193
|
+
if (this.extensionLoaded) {
|
|
6194
|
+
const vectorCol = String(this.vectorPropertyName);
|
|
6195
|
+
const vectorType = getVectorTypeOption(this.VectorType);
|
|
6196
|
+
try {
|
|
6197
|
+
this.database.prepare("SELECT vector_init(?, ?, ?)").run(this.table, vectorCol, `dimension=${this.vectorDimensions},type=${vectorType},distance=COSINE`);
|
|
6198
|
+
} catch {}
|
|
6199
|
+
}
|
|
6200
|
+
}
|
|
6201
|
+
encodeVectorJson(vector) {
|
|
6202
|
+
return `[${Array.from(vector).join(",")}]`;
|
|
6203
|
+
}
|
|
6204
|
+
decodeVector(raw) {
|
|
6205
|
+
if (raw instanceof Uint8Array || typeof Buffer !== "undefined" && raw instanceof Buffer) {
|
|
6206
|
+
const view = raw instanceof Uint8Array ? raw : new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength);
|
|
6207
|
+
if (this.VectorType === Float32Array || this.VectorType.name === "Float32Array") {
|
|
6208
|
+
return new Float32Array(view.buffer, view.byteOffset, this.vectorDimensions);
|
|
6209
|
+
}
|
|
6210
|
+
const f32 = new Float32Array(view.buffer, view.byteOffset, this.vectorDimensions);
|
|
6211
|
+
return new this.VectorType(Array.from(f32));
|
|
6212
|
+
}
|
|
6213
|
+
if (typeof raw === "string") {
|
|
6214
|
+
const array = JSON.parse(raw);
|
|
6215
|
+
return new this.VectorType(array);
|
|
6216
|
+
}
|
|
6217
|
+
if (Array.isArray(raw)) {
|
|
6218
|
+
return new this.VectorType(raw);
|
|
6219
|
+
}
|
|
6220
|
+
throw new Error(`Cannot decode vector from type: ${typeof raw}`);
|
|
6221
|
+
}
|
|
6222
|
+
jsToSqlValue(column, value) {
|
|
6223
|
+
if (column === String(this.vectorPropertyName) && value != null) {
|
|
6224
|
+
const vector = value;
|
|
6225
|
+
return this.encodeVectorJson(vector);
|
|
6226
|
+
}
|
|
6227
|
+
return super.jsToSqlValue(column, value);
|
|
6228
|
+
}
|
|
6229
|
+
sqlToJsValue(column, value) {
|
|
6230
|
+
if (column === String(this.vectorPropertyName) && value != null) {
|
|
6231
|
+
return this.decodeVector(value);
|
|
6232
|
+
}
|
|
6233
|
+
return super.sqlToJsValue(column, value);
|
|
6234
|
+
}
|
|
6235
|
+
mapTypeToSQL(typeDef) {
|
|
6236
|
+
if (typeof typeDef !== "boolean" && typeDef.type === "array") {
|
|
6237
|
+
const format = typeDef.format;
|
|
6238
|
+
if (format === "TypedArray" || format?.startsWith("TypedArray:")) {
|
|
6239
|
+
return "BLOB";
|
|
6240
|
+
}
|
|
6241
|
+
}
|
|
6242
|
+
return super.mapTypeToSQL(typeDef);
|
|
6243
|
+
}
|
|
6244
|
+
async put(entity) {
|
|
6245
|
+
if (!this.extensionLoaded) {
|
|
6246
|
+
return super.put(entity);
|
|
6247
|
+
}
|
|
6248
|
+
const db = this.database;
|
|
6249
|
+
const vectorCol = String(this.vectorPropertyName);
|
|
6250
|
+
let entityToInsert = entity;
|
|
6251
|
+
if (this.hasAutoGeneratedKey() && this.autoGeneratedKeyName) {
|
|
6252
|
+
const keyName = String(this.autoGeneratedKeyName);
|
|
6253
|
+
const clientProvidedValue = entity[keyName];
|
|
6254
|
+
const hasClientValue = clientProvidedValue !== undefined && clientProvidedValue !== null;
|
|
6255
|
+
const clientProvidedKeys = this.clientProvidedKeys;
|
|
6256
|
+
const autoGeneratedKeyStrategy = this.autoGeneratedKeyStrategy;
|
|
6257
|
+
if (autoGeneratedKeyStrategy === "uuid" && !hasClientValue && clientProvidedKeys !== "always") {
|
|
6258
|
+
const generatedValue = this.generateKeyValue(keyName, "uuid");
|
|
6259
|
+
entityToInsert = { ...entity, [keyName]: generatedValue };
|
|
6260
|
+
}
|
|
6261
|
+
}
|
|
6262
|
+
const allColumns = [];
|
|
6263
|
+
const placeholders = [];
|
|
6264
|
+
const params = [];
|
|
6265
|
+
const pkColumns = this.primaryKeyColumns();
|
|
6266
|
+
for (const col of pkColumns) {
|
|
6267
|
+
const autoGeneratedKeyStrategy = this.autoGeneratedKeyStrategy;
|
|
6268
|
+
const isAutoKey = this.isAutoGeneratedKey(col);
|
|
6269
|
+
if (isAutoKey && autoGeneratedKeyStrategy === "autoincrement") {
|
|
6270
|
+
const clientProvidedKeys = this.clientProvidedKeys;
|
|
6271
|
+
const clientValue = entityToInsert[col];
|
|
6272
|
+
if (clientProvidedKeys === "if-missing" && clientValue != null) {
|
|
6273
|
+
allColumns.push(col);
|
|
6274
|
+
placeholders.push("?");
|
|
6275
|
+
params.push(this.jsToSqlValue(col, clientValue));
|
|
6276
|
+
}
|
|
6277
|
+
continue;
|
|
6278
|
+
}
|
|
6279
|
+
allColumns.push(col);
|
|
6280
|
+
placeholders.push("?");
|
|
6281
|
+
params.push(this.jsToSqlValue(col, entityToInsert[col]));
|
|
6282
|
+
}
|
|
6283
|
+
const valueColumns = this.valueColumns();
|
|
6284
|
+
for (const col of valueColumns) {
|
|
6285
|
+
allColumns.push(col);
|
|
6286
|
+
const value = entityToInsert[col];
|
|
6287
|
+
if (col === vectorCol && value != null) {
|
|
6288
|
+
placeholders.push(`vector_as_${this.vectorTypeSuffix}(?)`);
|
|
6289
|
+
params.push(this.encodeVectorJson(value));
|
|
6290
|
+
} else {
|
|
6291
|
+
placeholders.push("?");
|
|
6292
|
+
params.push(this.jsToSqlValue(col, value));
|
|
6293
|
+
}
|
|
6294
|
+
}
|
|
6295
|
+
const columnList = allColumns.map((c) => `\`${c}\``).join(", ");
|
|
6296
|
+
const placeholderList = placeholders.join(", ");
|
|
6297
|
+
const sql = `
|
|
6298
|
+
INSERT OR REPLACE INTO ${escapeIdentifier(this.table)} (${columnList})
|
|
6299
|
+
VALUES (${placeholderList})
|
|
6300
|
+
RETURNING *
|
|
6301
|
+
`;
|
|
6302
|
+
for (let i = 0;i < params.length; i++) {
|
|
6303
|
+
if (params[i] === undefined) {
|
|
6304
|
+
params[i] = null;
|
|
6305
|
+
} else if (params[i] !== null && typeof params[i] === "object") {
|
|
6306
|
+
const p = params[i];
|
|
6307
|
+
if (!(p instanceof Uint8Array) && (typeof Buffer === "undefined" || !(p instanceof Buffer))) {
|
|
6308
|
+
params[i] = JSON.stringify(p);
|
|
6309
|
+
}
|
|
6310
|
+
}
|
|
6311
|
+
}
|
|
6312
|
+
const stmt = db.prepare(sql);
|
|
6313
|
+
const updatedEntity = stmt.get(...params);
|
|
6314
|
+
const updatedRecord = updatedEntity;
|
|
6315
|
+
for (const k in this.schema.properties) {
|
|
6316
|
+
updatedRecord[k] = this.sqlToJsValue(k, updatedRecord[k]);
|
|
6317
|
+
}
|
|
6318
|
+
this.events.emit("put", updatedEntity);
|
|
6319
|
+
return updatedEntity;
|
|
6320
|
+
}
|
|
6321
|
+
async similaritySearch(query, options = {}) {
|
|
6322
|
+
if (!this.extensionLoaded) {
|
|
6323
|
+
return this.searchFallback(query, options);
|
|
6324
|
+
}
|
|
6325
|
+
const { topK = 10, filter, scoreThreshold = 0 } = options;
|
|
6326
|
+
const db = this.database;
|
|
6327
|
+
const tableName = this.table;
|
|
6328
|
+
const vectorCol = String(this.vectorPropertyName);
|
|
6329
|
+
const metadataCol = this.metadataPropertyName ? String(this.metadataPropertyName) : null;
|
|
6330
|
+
try {
|
|
6331
|
+
const queryJson = this.encodeVectorJson(query);
|
|
6332
|
+
const queryBlob = db.prepare(`SELECT vector_as_${this.vectorTypeSuffix}(?) as v`).get(queryJson);
|
|
6333
|
+
if (filter && Object.keys(filter).length > 0) {
|
|
6334
|
+
const sql2 = `
|
|
6335
|
+
SELECT t.*, v.distance
|
|
6336
|
+
FROM ${escapeIdentifier(tableName)} AS t
|
|
6337
|
+
JOIN vector_full_scan(?, ?, ?) AS v
|
|
6338
|
+
ON t.rowid = v.rowid
|
|
6339
|
+
ORDER BY v.distance ASC
|
|
6340
|
+
`;
|
|
6341
|
+
const stmt2 = db.prepare(sql2);
|
|
6342
|
+
const rows2 = stmt2.all(tableName, vectorCol, queryBlob.v);
|
|
6343
|
+
const results2 = [];
|
|
6344
|
+
for (const row of rows2) {
|
|
6345
|
+
const score = 1 - row.distance;
|
|
6346
|
+
if (score < scoreThreshold) {
|
|
6347
|
+
continue;
|
|
6348
|
+
}
|
|
6349
|
+
const entity = { ...row };
|
|
6350
|
+
delete entity.distance;
|
|
6351
|
+
for (const k in this.schema.properties) {
|
|
6352
|
+
entity[k] = this.sqlToJsValue(k, entity[k]);
|
|
6353
|
+
}
|
|
6354
|
+
const metadata = metadataCol ? entity[metadataCol] : {};
|
|
6355
|
+
if (filter && !matchesFilter3(metadata, filter)) {
|
|
6356
|
+
continue;
|
|
6357
|
+
}
|
|
6358
|
+
results2.push({ ...entity, score });
|
|
6359
|
+
if (results2.length >= topK) {
|
|
6360
|
+
break;
|
|
6361
|
+
}
|
|
6362
|
+
}
|
|
6363
|
+
results2.sort((a, b) => b.score - a.score);
|
|
6364
|
+
return results2.slice(0, topK);
|
|
6365
|
+
}
|
|
6366
|
+
const sql = `
|
|
6367
|
+
SELECT t.*, v.distance
|
|
6368
|
+
FROM ${escapeIdentifier(tableName)} AS t
|
|
6369
|
+
JOIN vector_full_scan(?, ?, ?, ?) AS v
|
|
6370
|
+
ON t.rowid = v.rowid
|
|
6371
|
+
ORDER BY v.distance ASC
|
|
6372
|
+
`;
|
|
6373
|
+
const stmt = db.prepare(sql);
|
|
6374
|
+
const rows = stmt.all(tableName, vectorCol, queryBlob.v, topK);
|
|
6375
|
+
const results = [];
|
|
6376
|
+
for (const row of rows) {
|
|
6377
|
+
const score = 1 - row.distance;
|
|
6378
|
+
if (score < scoreThreshold) {
|
|
6379
|
+
continue;
|
|
6380
|
+
}
|
|
6381
|
+
const entity = { ...row };
|
|
6382
|
+
delete entity.distance;
|
|
6383
|
+
for (const k in this.schema.properties) {
|
|
6384
|
+
entity[k] = this.sqlToJsValue(k, entity[k]);
|
|
6385
|
+
}
|
|
6386
|
+
results.push({ ...entity, score });
|
|
6387
|
+
}
|
|
6388
|
+
return results;
|
|
6389
|
+
} catch (error) {
|
|
6390
|
+
console.warn("sqlite-vector query failed, falling back to in-memory search:", error);
|
|
6391
|
+
return this.searchFallback(query, options);
|
|
6392
|
+
}
|
|
6393
|
+
}
|
|
6394
|
+
async hybridSearch(query, options) {
|
|
6395
|
+
const { topK = 10, filter, scoreThreshold = 0, textQuery, vectorWeight = 0.7 } = options;
|
|
6396
|
+
if (!textQuery || textQuery.trim().length === 0) {
|
|
6397
|
+
return this.similaritySearch(query, { topK, filter, scoreThreshold });
|
|
6398
|
+
}
|
|
6399
|
+
if (!this.extensionLoaded) {
|
|
6400
|
+
return this.hybridSearchFallback(query, options);
|
|
6401
|
+
}
|
|
6402
|
+
const db = this.database;
|
|
6403
|
+
const tableName = this.table;
|
|
6404
|
+
const vectorCol = String(this.vectorPropertyName);
|
|
6405
|
+
const metadataCol = this.metadataPropertyName ? String(this.metadataPropertyName) : null;
|
|
6406
|
+
try {
|
|
6407
|
+
const queryJson = this.encodeVectorJson(query);
|
|
6408
|
+
const queryBlob = db.prepare(`SELECT vector_as_${this.vectorTypeSuffix}(?) as v`).get(queryJson);
|
|
6409
|
+
const sql = `
|
|
6410
|
+
SELECT t.*, v.distance
|
|
6411
|
+
FROM ${escapeIdentifier(tableName)} AS t
|
|
6412
|
+
JOIN vector_full_scan(?, ?, ?) AS v
|
|
6413
|
+
ON t.rowid = v.rowid
|
|
6414
|
+
ORDER BY v.distance ASC
|
|
6415
|
+
`;
|
|
6416
|
+
const stmt = db.prepare(sql);
|
|
6417
|
+
const rows = stmt.all(tableName, vectorCol, queryBlob.v);
|
|
6418
|
+
const queryLower = textQuery.toLowerCase();
|
|
6419
|
+
const queryWords = queryLower.split(/\s+/).filter((w) => w.length > 0);
|
|
6420
|
+
const results = [];
|
|
6421
|
+
for (const row of rows) {
|
|
6422
|
+
const vectorScore = 1 - row.distance;
|
|
6423
|
+
const entity = { ...row };
|
|
6424
|
+
delete entity.distance;
|
|
6425
|
+
for (const k in this.schema.properties) {
|
|
6426
|
+
entity[k] = this.sqlToJsValue(k, entity[k]);
|
|
6427
|
+
}
|
|
6428
|
+
const metadata = metadataCol ? entity[metadataCol] : {};
|
|
6429
|
+
if (filter && !matchesFilter3(metadata, filter)) {
|
|
6430
|
+
continue;
|
|
6431
|
+
}
|
|
6432
|
+
const metadataText = JSON.stringify(metadata).toLowerCase();
|
|
6433
|
+
let textScore = 0;
|
|
6434
|
+
if (queryWords.length > 0) {
|
|
6435
|
+
let matches = 0;
|
|
6436
|
+
for (const word of queryWords) {
|
|
6437
|
+
if (metadataText.includes(word)) {
|
|
6438
|
+
matches++;
|
|
6439
|
+
}
|
|
6440
|
+
}
|
|
6441
|
+
textScore = matches / queryWords.length;
|
|
6442
|
+
}
|
|
6443
|
+
const combinedScore = vectorWeight * vectorScore + (1 - vectorWeight) * textScore;
|
|
6444
|
+
if (combinedScore < scoreThreshold) {
|
|
6445
|
+
continue;
|
|
6446
|
+
}
|
|
6447
|
+
results.push({ ...entity, score: combinedScore });
|
|
6448
|
+
}
|
|
6449
|
+
results.sort((a, b) => b.score - a.score);
|
|
6450
|
+
return results.slice(0, topK);
|
|
6451
|
+
} catch (error) {
|
|
6452
|
+
console.warn("sqlite-vector hybrid query failed, falling back to in-memory search:", error);
|
|
6453
|
+
return this.hybridSearchFallback(query, options);
|
|
6454
|
+
}
|
|
6455
|
+
}
|
|
6456
|
+
async searchFallback(query, options) {
|
|
6457
|
+
const { topK = 10, filter, scoreThreshold = 0 } = options;
|
|
6458
|
+
const allRows = await this.getAll() || [];
|
|
6459
|
+
const results = [];
|
|
6460
|
+
for (const row of allRows) {
|
|
6461
|
+
const vector = row[this.vectorPropertyName];
|
|
6462
|
+
const metadata = this.metadataPropertyName ? row[this.metadataPropertyName] : {};
|
|
6463
|
+
if (filter && !matchesFilter3(metadata, filter)) {
|
|
6464
|
+
continue;
|
|
6465
|
+
}
|
|
6466
|
+
const score = cosineSimilarity4(query, vector);
|
|
6467
|
+
if (score >= scoreThreshold) {
|
|
6468
|
+
results.push({ ...row, score });
|
|
6469
|
+
}
|
|
6470
|
+
}
|
|
6471
|
+
results.sort((a, b) => b.score - a.score);
|
|
6472
|
+
return results.slice(0, topK);
|
|
6473
|
+
}
|
|
6474
|
+
async hybridSearchFallback(query, options) {
|
|
6475
|
+
const { topK = 10, filter, scoreThreshold = 0, textQuery, vectorWeight = 0.7 } = options;
|
|
6476
|
+
const allRows = await this.getAll() || [];
|
|
6477
|
+
const results = [];
|
|
6478
|
+
const queryLower = textQuery.toLowerCase();
|
|
6479
|
+
const queryWords = queryLower.split(/\s+/).filter((w) => w.length > 0);
|
|
6480
|
+
for (const row of allRows) {
|
|
6481
|
+
const vector = row[this.vectorPropertyName];
|
|
6482
|
+
const metadata = this.metadataPropertyName ? row[this.metadataPropertyName] : {};
|
|
6483
|
+
if (filter && !matchesFilter3(metadata, filter)) {
|
|
6484
|
+
continue;
|
|
6485
|
+
}
|
|
6486
|
+
const vectorScore = cosineSimilarity4(query, vector);
|
|
6487
|
+
const metadataText = JSON.stringify(metadata).toLowerCase();
|
|
6488
|
+
let textScore = 0;
|
|
6489
|
+
if (queryWords.length > 0) {
|
|
6490
|
+
let matches = 0;
|
|
6491
|
+
for (const word of queryWords) {
|
|
6492
|
+
if (metadataText.includes(word)) {
|
|
6493
|
+
matches++;
|
|
6494
|
+
}
|
|
6495
|
+
}
|
|
6496
|
+
textScore = matches / queryWords.length;
|
|
6497
|
+
}
|
|
6498
|
+
const combinedScore = vectorWeight * vectorScore + (1 - vectorWeight) * textScore;
|
|
6499
|
+
if (combinedScore >= scoreThreshold) {
|
|
6500
|
+
results.push({ ...row, score: combinedScore });
|
|
6501
|
+
}
|
|
6502
|
+
}
|
|
6503
|
+
results.sort((a, b) => b.score - a.score);
|
|
6504
|
+
return results.slice(0, topK);
|
|
6505
|
+
}
|
|
6506
|
+
}
|
|
5900
6507
|
// src/kv/IndexedDbKvStorage.ts
|
|
5901
6508
|
import { createServiceToken as createServiceToken28 } from "@workglow/util";
|
|
5902
6509
|
|
|
@@ -7485,9 +8092,9 @@ class IndexedDbQueueStorage {
|
|
|
7485
8092
|
}
|
|
7486
8093
|
}
|
|
7487
8094
|
// src/vector/IndexedDbVectorStorage.ts
|
|
7488
|
-
import { cosineSimilarity as
|
|
8095
|
+
import { cosineSimilarity as cosineSimilarity5, createServiceToken as createServiceToken31 } from "@workglow/util";
|
|
7489
8096
|
var IDB_VECTOR_REPOSITORY = createServiceToken31("storage.vectorRepository.indexedDb");
|
|
7490
|
-
function
|
|
8097
|
+
function matchesFilter4(metadata, filter) {
|
|
7491
8098
|
for (const [key, value] of Object.entries(filter)) {
|
|
7492
8099
|
if (metadata[key] !== value) {
|
|
7493
8100
|
return false;
|
|
@@ -7537,10 +8144,10 @@ class IndexedDbVectorStorage extends IndexedDbTabularStorage {
|
|
|
7537
8144
|
for (const entity of allEntities) {
|
|
7538
8145
|
const vector = entity[this.vectorPropertyName];
|
|
7539
8146
|
const metadata = this.metadataPropertyName ? entity[this.metadataPropertyName] : {};
|
|
7540
|
-
if (filter && !
|
|
8147
|
+
if (filter && !matchesFilter4(metadata, filter)) {
|
|
7541
8148
|
continue;
|
|
7542
8149
|
}
|
|
7543
|
-
const score =
|
|
8150
|
+
const score = cosineSimilarity5(query, vector);
|
|
7544
8151
|
if (score < scoreThreshold) {
|
|
7545
8152
|
continue;
|
|
7546
8153
|
}
|
|
@@ -7563,10 +8170,10 @@ class IndexedDbVectorStorage extends IndexedDbTabularStorage {
|
|
|
7563
8170
|
for (const entity of allEntities) {
|
|
7564
8171
|
const vector = entity[this.vectorPropertyName];
|
|
7565
8172
|
const metadata = this.metadataPropertyName ? entity[this.metadataPropertyName] : {};
|
|
7566
|
-
if (filter && !
|
|
8173
|
+
if (filter && !matchesFilter4(metadata, filter)) {
|
|
7567
8174
|
continue;
|
|
7568
8175
|
}
|
|
7569
|
-
const vectorScore =
|
|
8176
|
+
const vectorScore = cosineSimilarity5(query, vector);
|
|
7570
8177
|
const metadataText = Object.values(metadata).join(" ").toLowerCase();
|
|
7571
8178
|
const textScore = textRelevance2(metadataText, textQuery);
|
|
7572
8179
|
const combinedScore = vectorWeight * vectorScore + (1 - vectorWeight) * textScore;
|
|
@@ -7584,6 +8191,7 @@ class IndexedDbVectorStorage extends IndexedDbTabularStorage {
|
|
|
7584
8191
|
}
|
|
7585
8192
|
}
|
|
7586
8193
|
export {
|
|
8194
|
+
traced,
|
|
7587
8195
|
registerTabularRepository,
|
|
7588
8196
|
isSearchCondition,
|
|
7589
8197
|
getVectorProperty,
|
|
@@ -7592,6 +8200,10 @@ export {
|
|
|
7592
8200
|
getGlobalTabularRepositories,
|
|
7593
8201
|
ensureIndexedDbTable,
|
|
7594
8202
|
dropIndexedDbTable,
|
|
8203
|
+
TelemetryVectorStorage,
|
|
8204
|
+
TelemetryTabularStorage,
|
|
8205
|
+
TelemetryQueueStorage,
|
|
8206
|
+
TelemetryKvStorage,
|
|
7595
8207
|
TABULAR_REPOSITORY,
|
|
7596
8208
|
TABULAR_REPOSITORIES,
|
|
7597
8209
|
SupabaseTabularStorage,
|
|
@@ -7609,6 +8221,7 @@ export {
|
|
|
7609
8221
|
SqliteRateLimiterStorage,
|
|
7610
8222
|
SqliteQueueStorage,
|
|
7611
8223
|
SqliteKvStorage,
|
|
8224
|
+
SqliteAiVectorStorage,
|
|
7612
8225
|
SUPABASE_TABULAR_REPOSITORY,
|
|
7613
8226
|
SUPABASE_RATE_LIMITER_STORAGE,
|
|
7614
8227
|
SUPABASE_QUEUE_STORAGE,
|
|
@@ -7669,4 +8282,4 @@ export {
|
|
|
7669
8282
|
BaseTabularStorage
|
|
7670
8283
|
};
|
|
7671
8284
|
|
|
7672
|
-
//# debugId=
|
|
8285
|
+
//# debugId=9A5BEC0841EC01AF64756E2164756E21
|