@workglow/storage 0.0.115 → 0.0.117
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser.js +224 -1
- package/dist/browser.js.map +9 -4
- package/dist/bun.js +620 -7
- package/dist/bun.js.map +11 -5
- package/dist/common-server.d.ts +1 -0
- package/dist/common-server.d.ts.map +1 -1
- package/dist/common.d.ts +5 -0
- package/dist/common.d.ts.map +1 -1
- package/dist/kv/TelemetryKvStorage.d.ts +36 -0
- package/dist/kv/TelemetryKvStorage.d.ts.map +1 -0
- package/dist/node.js +621 -7
- package/dist/node.js.map +11 -5
- package/dist/queue/TelemetryQueueStorage.d.ts +31 -0
- package/dist/queue/TelemetryQueueStorage.d.ts.map +1 -0
- package/dist/tabular/SqliteTabularStorage.d.ts +2 -0
- package/dist/tabular/SqliteTabularStorage.d.ts.map +1 -1
- package/dist/tabular/TelemetryTabularStorage.d.ts +39 -0
- package/dist/tabular/TelemetryTabularStorage.d.ts.map +1 -0
- package/dist/util/traced.d.ts +10 -0
- package/dist/util/traced.d.ts.map +1 -0
- package/dist/vector/IVectorStorage.d.ts +2 -2
- package/dist/vector/IVectorStorage.d.ts.map +1 -1
- package/dist/vector/SqliteAiVectorStorage.d.ts +106 -0
- package/dist/vector/SqliteAiVectorStorage.d.ts.map +1 -0
- package/dist/vector/TelemetryVectorStorage.d.ts +25 -0
- package/dist/vector/TelemetryVectorStorage.d.ts.map +1 -0
- package/package.json +10 -6
package/dist/node.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
var __require = /* @__PURE__ */ createRequire(import.meta.url);
|
|
3
|
+
|
|
1
4
|
// src/tabular/BaseTabularStorage.ts
|
|
2
5
|
import {
|
|
3
6
|
createServiceToken,
|
|
@@ -1138,6 +1141,102 @@ function resolveRepositoryFromRegistry(id, format, registry) {
|
|
|
1138
1141
|
return repo;
|
|
1139
1142
|
}
|
|
1140
1143
|
registerInputResolver("storage:tabular", resolveRepositoryFromRegistry);
|
|
1144
|
+
// src/util/traced.ts
|
|
1145
|
+
import { getTelemetryProvider, SpanStatusCode } from "@workglow/util";
|
|
1146
|
+
async function traced(spanName, storageName, fn) {
|
|
1147
|
+
const telemetry = getTelemetryProvider();
|
|
1148
|
+
if (!telemetry.isEnabled)
|
|
1149
|
+
return fn();
|
|
1150
|
+
const span = telemetry.startSpan(spanName, {
|
|
1151
|
+
attributes: { "workglow.storage.name": storageName }
|
|
1152
|
+
});
|
|
1153
|
+
try {
|
|
1154
|
+
const result = await fn();
|
|
1155
|
+
span.setStatus(SpanStatusCode.OK);
|
|
1156
|
+
return result;
|
|
1157
|
+
} catch (err) {
|
|
1158
|
+
span.setStatus(SpanStatusCode.ERROR, err instanceof Error ? err.message : String(err));
|
|
1159
|
+
throw err;
|
|
1160
|
+
} finally {
|
|
1161
|
+
span.end();
|
|
1162
|
+
}
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
// src/tabular/TelemetryTabularStorage.ts
|
|
1166
|
+
class TelemetryTabularStorage {
|
|
1167
|
+
storageName;
|
|
1168
|
+
inner;
|
|
1169
|
+
constructor(storageName, inner) {
|
|
1170
|
+
this.storageName = storageName;
|
|
1171
|
+
this.inner = inner;
|
|
1172
|
+
}
|
|
1173
|
+
put(value) {
|
|
1174
|
+
return traced("workglow.storage.tabular.put", this.storageName, () => this.inner.put(value));
|
|
1175
|
+
}
|
|
1176
|
+
putBulk(values) {
|
|
1177
|
+
return traced("workglow.storage.tabular.putBulk", this.storageName, () => this.inner.putBulk(values));
|
|
1178
|
+
}
|
|
1179
|
+
get(key) {
|
|
1180
|
+
return traced("workglow.storage.tabular.get", this.storageName, () => this.inner.get(key));
|
|
1181
|
+
}
|
|
1182
|
+
delete(key) {
|
|
1183
|
+
return traced("workglow.storage.tabular.delete", this.storageName, () => this.inner.delete(key));
|
|
1184
|
+
}
|
|
1185
|
+
getAll(options) {
|
|
1186
|
+
return traced("workglow.storage.tabular.getAll", this.storageName, () => this.inner.getAll(options));
|
|
1187
|
+
}
|
|
1188
|
+
deleteAll() {
|
|
1189
|
+
return traced("workglow.storage.tabular.deleteAll", this.storageName, () => this.inner.deleteAll());
|
|
1190
|
+
}
|
|
1191
|
+
size() {
|
|
1192
|
+
return traced("workglow.storage.tabular.size", this.storageName, () => this.inner.size());
|
|
1193
|
+
}
|
|
1194
|
+
deleteSearch(criteria) {
|
|
1195
|
+
return traced("workglow.storage.tabular.deleteSearch", this.storageName, () => this.inner.deleteSearch(criteria));
|
|
1196
|
+
}
|
|
1197
|
+
getBulk(offset, limit) {
|
|
1198
|
+
return traced("workglow.storage.tabular.getBulk", this.storageName, () => this.inner.getBulk(offset, limit));
|
|
1199
|
+
}
|
|
1200
|
+
query(criteria, options) {
|
|
1201
|
+
return traced("workglow.storage.tabular.query", this.storageName, () => this.inner.query(criteria, options));
|
|
1202
|
+
}
|
|
1203
|
+
records(pageSize) {
|
|
1204
|
+
return this.inner.records(pageSize);
|
|
1205
|
+
}
|
|
1206
|
+
pages(pageSize) {
|
|
1207
|
+
return this.inner.pages(pageSize);
|
|
1208
|
+
}
|
|
1209
|
+
subscribeToChanges(callback, options) {
|
|
1210
|
+
return this.inner.subscribeToChanges(callback, options);
|
|
1211
|
+
}
|
|
1212
|
+
setupDatabase() {
|
|
1213
|
+
return this.inner.setupDatabase();
|
|
1214
|
+
}
|
|
1215
|
+
destroy() {
|
|
1216
|
+
return this.inner.destroy();
|
|
1217
|
+
}
|
|
1218
|
+
[Symbol.dispose]() {
|
|
1219
|
+
return this.inner[Symbol.dispose]();
|
|
1220
|
+
}
|
|
1221
|
+
[Symbol.asyncDispose]() {
|
|
1222
|
+
return this.inner[Symbol.asyncDispose]();
|
|
1223
|
+
}
|
|
1224
|
+
on(name, fn) {
|
|
1225
|
+
this.inner.on(name, fn);
|
|
1226
|
+
}
|
|
1227
|
+
off(name, fn) {
|
|
1228
|
+
this.inner.off(name, fn);
|
|
1229
|
+
}
|
|
1230
|
+
emit(name, ...args) {
|
|
1231
|
+
this.inner.emit(name, ...args);
|
|
1232
|
+
}
|
|
1233
|
+
once(name, fn) {
|
|
1234
|
+
this.inner.once(name, fn);
|
|
1235
|
+
}
|
|
1236
|
+
waitOn(name) {
|
|
1237
|
+
return this.inner.waitOn(name);
|
|
1238
|
+
}
|
|
1239
|
+
}
|
|
1141
1240
|
// src/kv/IKvStorage.ts
|
|
1142
1241
|
var DefaultKeyValueSchema = {
|
|
1143
1242
|
type: "object",
|
|
@@ -1269,6 +1368,54 @@ class InMemoryKvStorage extends KvViaTabularStorage {
|
|
|
1269
1368
|
this.tabularRepository = new InMemoryTabularStorage(DefaultKeyValueSchema, DefaultKeyValueKey);
|
|
1270
1369
|
}
|
|
1271
1370
|
}
|
|
1371
|
+
// src/kv/TelemetryKvStorage.ts
|
|
1372
|
+
class TelemetryKvStorage {
|
|
1373
|
+
storageName;
|
|
1374
|
+
inner;
|
|
1375
|
+
constructor(storageName, inner) {
|
|
1376
|
+
this.storageName = storageName;
|
|
1377
|
+
this.inner = inner;
|
|
1378
|
+
}
|
|
1379
|
+
put(key, value) {
|
|
1380
|
+
return traced("workglow.storage.kv.put", this.storageName, () => this.inner.put(key, value));
|
|
1381
|
+
}
|
|
1382
|
+
putBulk(items) {
|
|
1383
|
+
return traced("workglow.storage.kv.putBulk", this.storageName, () => this.inner.putBulk(items));
|
|
1384
|
+
}
|
|
1385
|
+
get(key) {
|
|
1386
|
+
return traced("workglow.storage.kv.get", this.storageName, () => this.inner.get(key));
|
|
1387
|
+
}
|
|
1388
|
+
delete(key) {
|
|
1389
|
+
return traced("workglow.storage.kv.delete", this.storageName, () => this.inner.delete(key));
|
|
1390
|
+
}
|
|
1391
|
+
getAll() {
|
|
1392
|
+
return traced("workglow.storage.kv.getAll", this.storageName, () => this.inner.getAll());
|
|
1393
|
+
}
|
|
1394
|
+
deleteAll() {
|
|
1395
|
+
return traced("workglow.storage.kv.deleteAll", this.storageName, () => this.inner.deleteAll());
|
|
1396
|
+
}
|
|
1397
|
+
size() {
|
|
1398
|
+
return traced("workglow.storage.kv.size", this.storageName, () => this.inner.size());
|
|
1399
|
+
}
|
|
1400
|
+
getObjectAsIdString(object) {
|
|
1401
|
+
return this.inner.getObjectAsIdString(object);
|
|
1402
|
+
}
|
|
1403
|
+
on(name, fn) {
|
|
1404
|
+
this.inner.on(name, fn);
|
|
1405
|
+
}
|
|
1406
|
+
off(name, fn) {
|
|
1407
|
+
this.inner.off(name, fn);
|
|
1408
|
+
}
|
|
1409
|
+
emit(name, ...args) {
|
|
1410
|
+
this.inner.emit(name, ...args);
|
|
1411
|
+
}
|
|
1412
|
+
once(name, fn) {
|
|
1413
|
+
this.inner.once(name, fn);
|
|
1414
|
+
}
|
|
1415
|
+
waitOn(name) {
|
|
1416
|
+
return this.inner.waitOn(name);
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1272
1419
|
// src/queue/InMemoryQueueStorage.ts
|
|
1273
1420
|
import {
|
|
1274
1421
|
createServiceToken as createServiceToken9,
|
|
@@ -1480,6 +1627,60 @@ class InMemoryQueueStorage {
|
|
|
1480
1627
|
return this.events.subscribe("change", filteredCallback);
|
|
1481
1628
|
}
|
|
1482
1629
|
}
|
|
1630
|
+
// src/queue/TelemetryQueueStorage.ts
|
|
1631
|
+
class TelemetryQueueStorage {
|
|
1632
|
+
storageName;
|
|
1633
|
+
inner;
|
|
1634
|
+
constructor(storageName, inner) {
|
|
1635
|
+
this.storageName = storageName;
|
|
1636
|
+
this.inner = inner;
|
|
1637
|
+
}
|
|
1638
|
+
add(job) {
|
|
1639
|
+
return traced("workglow.storage.queue.add", this.storageName, () => this.inner.add(job));
|
|
1640
|
+
}
|
|
1641
|
+
get(id) {
|
|
1642
|
+
return traced("workglow.storage.queue.get", this.storageName, () => this.inner.get(id));
|
|
1643
|
+
}
|
|
1644
|
+
next(workerId) {
|
|
1645
|
+
return traced("workglow.storage.queue.next", this.storageName, () => this.inner.next(workerId));
|
|
1646
|
+
}
|
|
1647
|
+
peek(status, num) {
|
|
1648
|
+
return traced("workglow.storage.queue.peek", this.storageName, () => this.inner.peek(status, num));
|
|
1649
|
+
}
|
|
1650
|
+
size(status) {
|
|
1651
|
+
return traced("workglow.storage.queue.size", this.storageName, () => this.inner.size(status));
|
|
1652
|
+
}
|
|
1653
|
+
complete(job) {
|
|
1654
|
+
return traced("workglow.storage.queue.complete", this.storageName, () => this.inner.complete(job));
|
|
1655
|
+
}
|
|
1656
|
+
deleteAll() {
|
|
1657
|
+
return traced("workglow.storage.queue.deleteAll", this.storageName, () => this.inner.deleteAll());
|
|
1658
|
+
}
|
|
1659
|
+
outputForInput(input) {
|
|
1660
|
+
return traced("workglow.storage.queue.outputForInput", this.storageName, () => this.inner.outputForInput(input));
|
|
1661
|
+
}
|
|
1662
|
+
abort(id) {
|
|
1663
|
+
return traced("workglow.storage.queue.abort", this.storageName, () => this.inner.abort(id));
|
|
1664
|
+
}
|
|
1665
|
+
getByRunId(runId) {
|
|
1666
|
+
return traced("workglow.storage.queue.getByRunId", this.storageName, () => this.inner.getByRunId(runId));
|
|
1667
|
+
}
|
|
1668
|
+
saveProgress(id, progress, message, details) {
|
|
1669
|
+
return traced("workglow.storage.queue.saveProgress", this.storageName, () => this.inner.saveProgress(id, progress, message, details));
|
|
1670
|
+
}
|
|
1671
|
+
delete(id) {
|
|
1672
|
+
return traced("workglow.storage.queue.delete", this.storageName, () => this.inner.delete(id));
|
|
1673
|
+
}
|
|
1674
|
+
deleteJobsByStatusAndAge(status, olderThanMs) {
|
|
1675
|
+
return traced("workglow.storage.queue.deleteJobsByStatusAndAge", this.storageName, () => this.inner.deleteJobsByStatusAndAge(status, olderThanMs));
|
|
1676
|
+
}
|
|
1677
|
+
setupDatabase() {
|
|
1678
|
+
return this.inner.setupDatabase();
|
|
1679
|
+
}
|
|
1680
|
+
subscribeToChanges(callback, options) {
|
|
1681
|
+
return this.inner.subscribeToChanges(callback, options);
|
|
1682
|
+
}
|
|
1683
|
+
}
|
|
1483
1684
|
// src/queue-limiter/InMemoryRateLimiterStorage.ts
|
|
1484
1685
|
import { createServiceToken as createServiceToken10, sleep as sleep2 } from "@workglow/util";
|
|
1485
1686
|
var IN_MEMORY_RATE_LIMITER_STORAGE = createServiceToken10("ratelimiter.storage.inMemory");
|
|
@@ -1933,6 +2134,26 @@ class InMemoryVectorStorage extends InMemoryTabularStorage {
|
|
|
1933
2134
|
return topResults;
|
|
1934
2135
|
}
|
|
1935
2136
|
}
|
|
2137
|
+
// src/vector/TelemetryVectorStorage.ts
|
|
2138
|
+
class TelemetryVectorStorage extends TelemetryTabularStorage {
|
|
2139
|
+
vectorInner;
|
|
2140
|
+
constructor(storageName, inner) {
|
|
2141
|
+
super(storageName, inner);
|
|
2142
|
+
this.vectorInner = inner;
|
|
2143
|
+
}
|
|
2144
|
+
getVectorDimensions() {
|
|
2145
|
+
return this.vectorInner.getVectorDimensions();
|
|
2146
|
+
}
|
|
2147
|
+
similaritySearch(query, options) {
|
|
2148
|
+
return traced("workglow.storage.vector.similaritySearch", this.storageName, () => this.vectorInner.similaritySearch(query, options));
|
|
2149
|
+
}
|
|
2150
|
+
hybridSearch(query, options) {
|
|
2151
|
+
if (!this.vectorInner.hybridSearch) {
|
|
2152
|
+
throw new Error("hybridSearch is not supported by the underlying storage implementation");
|
|
2153
|
+
}
|
|
2154
|
+
return traced("workglow.storage.vector.hybridSearch", this.storageName, () => this.vectorInner.hybridSearch(query, options));
|
|
2155
|
+
}
|
|
2156
|
+
}
|
|
1936
2157
|
// src/credentials/EncryptedKvCredentialStore.ts
|
|
1937
2158
|
import { decrypt, encrypt } from "@workglow/util";
|
|
1938
2159
|
|
|
@@ -2912,6 +3133,9 @@ var Database = Sqlite.Database;
|
|
|
2912
3133
|
|
|
2913
3134
|
class SqliteTabularStorage extends BaseSqlTabularStorage {
|
|
2914
3135
|
db;
|
|
3136
|
+
get database() {
|
|
3137
|
+
return this.db;
|
|
3138
|
+
}
|
|
2915
3139
|
constructor(dbOrPath, table = "tabular_store", schema, primaryKeyNames, indexes = [], clientProvidedKeys = "if-missing") {
|
|
2916
3140
|
super(table, schema, primaryKeyNames, indexes, clientProvidedKeys);
|
|
2917
3141
|
if (typeof dbOrPath === "string") {
|
|
@@ -5896,6 +6120,390 @@ class SqliteVectorStorage extends SqliteTabularStorage {
|
|
|
5896
6120
|
return topResults;
|
|
5897
6121
|
}
|
|
5898
6122
|
}
|
|
6123
|
+
// src/vector/SqliteAiVectorStorage.ts
|
|
6124
|
+
import { cosineSimilarity as cosineSimilarity4 } from "@workglow/util";
|
|
6125
|
+
var VECTOR_TYPE_MAP = {
|
|
6126
|
+
Float32Array: "f32",
|
|
6127
|
+
Float64Array: "f32",
|
|
6128
|
+
Int8Array: "i8",
|
|
6129
|
+
Uint8Array: "u8",
|
|
6130
|
+
Int16Array: "f16"
|
|
6131
|
+
};
|
|
6132
|
+
function getVectorTypeSuffix(VectorType) {
|
|
6133
|
+
return VECTOR_TYPE_MAP[VectorType.name] || "f32";
|
|
6134
|
+
}
|
|
6135
|
+
function getVectorTypeOption(VectorType) {
|
|
6136
|
+
const typeMap = {
|
|
6137
|
+
Float32Array: "FLOAT32",
|
|
6138
|
+
Float64Array: "FLOAT32",
|
|
6139
|
+
Int8Array: "INT8",
|
|
6140
|
+
Uint8Array: "UINT8",
|
|
6141
|
+
Int16Array: "FLOAT16"
|
|
6142
|
+
};
|
|
6143
|
+
return typeMap[VectorType.name] || "FLOAT32";
|
|
6144
|
+
}
|
|
6145
|
+
function matchesFilter3(metadata, filter) {
|
|
6146
|
+
for (const [key, value] of Object.entries(filter)) {
|
|
6147
|
+
if (metadata[key] !== value) {
|
|
6148
|
+
return false;
|
|
6149
|
+
}
|
|
6150
|
+
}
|
|
6151
|
+
return true;
|
|
6152
|
+
}
|
|
6153
|
+
function escapeIdentifier(name) {
|
|
6154
|
+
return "`" + name.replace(/`/g, "``") + "`";
|
|
6155
|
+
}
|
|
6156
|
+
|
|
6157
|
+
class SqliteAiVectorStorage extends SqliteTabularStorage {
|
|
6158
|
+
vectorDimensions;
|
|
6159
|
+
VectorType;
|
|
6160
|
+
vectorPropertyName;
|
|
6161
|
+
metadataPropertyName;
|
|
6162
|
+
vectorTypeSuffix;
|
|
6163
|
+
extensionLoaded = false;
|
|
6164
|
+
constructor(dbOrPath, table = "vectors", schema, primaryKeyNames, indexes = [], dimensions, VectorType = Float32Array) {
|
|
6165
|
+
super(dbOrPath, table, schema, primaryKeyNames, indexes);
|
|
6166
|
+
this.vectorDimensions = dimensions;
|
|
6167
|
+
this.VectorType = VectorType;
|
|
6168
|
+
this.vectorTypeSuffix = getVectorTypeSuffix(VectorType);
|
|
6169
|
+
const vectorProp = getVectorProperty(schema);
|
|
6170
|
+
if (!vectorProp) {
|
|
6171
|
+
throw new Error("Schema must have a property with type array and format TypedArray");
|
|
6172
|
+
}
|
|
6173
|
+
this.vectorPropertyName = vectorProp;
|
|
6174
|
+
this.metadataPropertyName = getMetadataProperty(schema);
|
|
6175
|
+
}
|
|
6176
|
+
getVectorDimensions() {
|
|
6177
|
+
return this.vectorDimensions;
|
|
6178
|
+
}
|
|
6179
|
+
async setupDatabase() {
|
|
6180
|
+
await super.setupDatabase();
|
|
6181
|
+
if (!this.extensionLoaded) {
|
|
6182
|
+
try {
|
|
6183
|
+
const { getExtensionPath } = await import("@sqliteai/sqlite-vector");
|
|
6184
|
+
this.database.loadExtension(getExtensionPath());
|
|
6185
|
+
this.extensionLoaded = true;
|
|
6186
|
+
} catch {
|
|
6187
|
+
try {
|
|
6188
|
+
this.database.exec("SELECT vector_version()");
|
|
6189
|
+
this.extensionLoaded = true;
|
|
6190
|
+
} catch {}
|
|
6191
|
+
}
|
|
6192
|
+
}
|
|
6193
|
+
if (this.extensionLoaded) {
|
|
6194
|
+
const vectorCol = String(this.vectorPropertyName);
|
|
6195
|
+
const vectorType = getVectorTypeOption(this.VectorType);
|
|
6196
|
+
try {
|
|
6197
|
+
this.database.prepare("SELECT vector_init(?, ?, ?)").run(this.table, vectorCol, `dimension=${this.vectorDimensions},type=${vectorType},distance=COSINE`);
|
|
6198
|
+
} catch {}
|
|
6199
|
+
}
|
|
6200
|
+
}
|
|
6201
|
+
encodeVectorJson(vector) {
|
|
6202
|
+
return `[${Array.from(vector).join(",")}]`;
|
|
6203
|
+
}
|
|
6204
|
+
decodeVector(raw) {
|
|
6205
|
+
if (raw instanceof Uint8Array || typeof Buffer !== "undefined" && raw instanceof Buffer) {
|
|
6206
|
+
const view = raw instanceof Uint8Array ? raw : new Uint8Array(raw.buffer, raw.byteOffset, raw.byteLength);
|
|
6207
|
+
if (this.VectorType === Float32Array || this.VectorType.name === "Float32Array") {
|
|
6208
|
+
return new Float32Array(view.buffer, view.byteOffset, this.vectorDimensions);
|
|
6209
|
+
}
|
|
6210
|
+
const f32 = new Float32Array(view.buffer, view.byteOffset, this.vectorDimensions);
|
|
6211
|
+
return new this.VectorType(Array.from(f32));
|
|
6212
|
+
}
|
|
6213
|
+
if (typeof raw === "string") {
|
|
6214
|
+
const array = JSON.parse(raw);
|
|
6215
|
+
return new this.VectorType(array);
|
|
6216
|
+
}
|
|
6217
|
+
if (Array.isArray(raw)) {
|
|
6218
|
+
return new this.VectorType(raw);
|
|
6219
|
+
}
|
|
6220
|
+
throw new Error(`Cannot decode vector from type: ${typeof raw}`);
|
|
6221
|
+
}
|
|
6222
|
+
jsToSqlValue(column, value) {
|
|
6223
|
+
if (column === String(this.vectorPropertyName) && value != null) {
|
|
6224
|
+
const vector = value;
|
|
6225
|
+
return this.encodeVectorJson(vector);
|
|
6226
|
+
}
|
|
6227
|
+
return super.jsToSqlValue(column, value);
|
|
6228
|
+
}
|
|
6229
|
+
sqlToJsValue(column, value) {
|
|
6230
|
+
if (column === String(this.vectorPropertyName) && value != null) {
|
|
6231
|
+
return this.decodeVector(value);
|
|
6232
|
+
}
|
|
6233
|
+
return super.sqlToJsValue(column, value);
|
|
6234
|
+
}
|
|
6235
|
+
mapTypeToSQL(typeDef) {
|
|
6236
|
+
if (typeof typeDef !== "boolean" && typeDef.type === "array") {
|
|
6237
|
+
const format = typeDef.format;
|
|
6238
|
+
if (format === "TypedArray" || format?.startsWith("TypedArray:")) {
|
|
6239
|
+
return "BLOB";
|
|
6240
|
+
}
|
|
6241
|
+
}
|
|
6242
|
+
return super.mapTypeToSQL(typeDef);
|
|
6243
|
+
}
|
|
6244
|
+
async put(entity) {
|
|
6245
|
+
if (!this.extensionLoaded) {
|
|
6246
|
+
return super.put(entity);
|
|
6247
|
+
}
|
|
6248
|
+
const db = this.database;
|
|
6249
|
+
const vectorCol = String(this.vectorPropertyName);
|
|
6250
|
+
let entityToInsert = entity;
|
|
6251
|
+
if (this.hasAutoGeneratedKey() && this.autoGeneratedKeyName) {
|
|
6252
|
+
const keyName = String(this.autoGeneratedKeyName);
|
|
6253
|
+
const clientProvidedValue = entity[keyName];
|
|
6254
|
+
const hasClientValue = clientProvidedValue !== undefined && clientProvidedValue !== null;
|
|
6255
|
+
const clientProvidedKeys = this.clientProvidedKeys;
|
|
6256
|
+
const autoGeneratedKeyStrategy = this.autoGeneratedKeyStrategy;
|
|
6257
|
+
if (autoGeneratedKeyStrategy === "uuid" && !hasClientValue && clientProvidedKeys !== "always") {
|
|
6258
|
+
const generatedValue = this.generateKeyValue(keyName, "uuid");
|
|
6259
|
+
entityToInsert = { ...entity, [keyName]: generatedValue };
|
|
6260
|
+
}
|
|
6261
|
+
}
|
|
6262
|
+
const allColumns = [];
|
|
6263
|
+
const placeholders = [];
|
|
6264
|
+
const params = [];
|
|
6265
|
+
const pkColumns = this.primaryKeyColumns();
|
|
6266
|
+
for (const col of pkColumns) {
|
|
6267
|
+
const autoGeneratedKeyStrategy = this.autoGeneratedKeyStrategy;
|
|
6268
|
+
const isAutoKey = this.isAutoGeneratedKey(col);
|
|
6269
|
+
if (isAutoKey && autoGeneratedKeyStrategy === "autoincrement") {
|
|
6270
|
+
const clientProvidedKeys = this.clientProvidedKeys;
|
|
6271
|
+
const clientValue = entityToInsert[col];
|
|
6272
|
+
if (clientProvidedKeys === "if-missing" && clientValue != null) {
|
|
6273
|
+
allColumns.push(col);
|
|
6274
|
+
placeholders.push("?");
|
|
6275
|
+
params.push(this.jsToSqlValue(col, clientValue));
|
|
6276
|
+
}
|
|
6277
|
+
continue;
|
|
6278
|
+
}
|
|
6279
|
+
allColumns.push(col);
|
|
6280
|
+
placeholders.push("?");
|
|
6281
|
+
params.push(this.jsToSqlValue(col, entityToInsert[col]));
|
|
6282
|
+
}
|
|
6283
|
+
const valueColumns = this.valueColumns();
|
|
6284
|
+
for (const col of valueColumns) {
|
|
6285
|
+
allColumns.push(col);
|
|
6286
|
+
const value = entityToInsert[col];
|
|
6287
|
+
if (col === vectorCol && value != null) {
|
|
6288
|
+
placeholders.push(`vector_as_${this.vectorTypeSuffix}(?)`);
|
|
6289
|
+
params.push(this.encodeVectorJson(value));
|
|
6290
|
+
} else {
|
|
6291
|
+
placeholders.push("?");
|
|
6292
|
+
params.push(this.jsToSqlValue(col, value));
|
|
6293
|
+
}
|
|
6294
|
+
}
|
|
6295
|
+
const columnList = allColumns.map((c) => `\`${c}\``).join(", ");
|
|
6296
|
+
const placeholderList = placeholders.join(", ");
|
|
6297
|
+
const sql = `
|
|
6298
|
+
INSERT OR REPLACE INTO ${escapeIdentifier(this.table)} (${columnList})
|
|
6299
|
+
VALUES (${placeholderList})
|
|
6300
|
+
RETURNING *
|
|
6301
|
+
`;
|
|
6302
|
+
for (let i = 0;i < params.length; i++) {
|
|
6303
|
+
if (params[i] === undefined) {
|
|
6304
|
+
params[i] = null;
|
|
6305
|
+
} else if (params[i] !== null && typeof params[i] === "object") {
|
|
6306
|
+
const p = params[i];
|
|
6307
|
+
if (!(p instanceof Uint8Array) && (typeof Buffer === "undefined" || !(p instanceof Buffer))) {
|
|
6308
|
+
params[i] = JSON.stringify(p);
|
|
6309
|
+
}
|
|
6310
|
+
}
|
|
6311
|
+
}
|
|
6312
|
+
const stmt = db.prepare(sql);
|
|
6313
|
+
const updatedEntity = stmt.get(...params);
|
|
6314
|
+
const updatedRecord = updatedEntity;
|
|
6315
|
+
for (const k in this.schema.properties) {
|
|
6316
|
+
updatedRecord[k] = this.sqlToJsValue(k, updatedRecord[k]);
|
|
6317
|
+
}
|
|
6318
|
+
this.events.emit("put", updatedEntity);
|
|
6319
|
+
return updatedEntity;
|
|
6320
|
+
}
|
|
6321
|
+
async similaritySearch(query, options = {}) {
|
|
6322
|
+
if (!this.extensionLoaded) {
|
|
6323
|
+
return this.searchFallback(query, options);
|
|
6324
|
+
}
|
|
6325
|
+
const { topK = 10, filter, scoreThreshold = 0 } = options;
|
|
6326
|
+
const db = this.database;
|
|
6327
|
+
const tableName = this.table;
|
|
6328
|
+
const vectorCol = String(this.vectorPropertyName);
|
|
6329
|
+
const metadataCol = this.metadataPropertyName ? String(this.metadataPropertyName) : null;
|
|
6330
|
+
try {
|
|
6331
|
+
const queryJson = this.encodeVectorJson(query);
|
|
6332
|
+
const queryBlob = db.prepare(`SELECT vector_as_${this.vectorTypeSuffix}(?) as v`).get(queryJson);
|
|
6333
|
+
if (filter && Object.keys(filter).length > 0) {
|
|
6334
|
+
const sql2 = `
|
|
6335
|
+
SELECT t.*, v.distance
|
|
6336
|
+
FROM ${escapeIdentifier(tableName)} AS t
|
|
6337
|
+
JOIN vector_full_scan(?, ?, ?) AS v
|
|
6338
|
+
ON t.rowid = v.rowid
|
|
6339
|
+
ORDER BY v.distance ASC
|
|
6340
|
+
`;
|
|
6341
|
+
const stmt2 = db.prepare(sql2);
|
|
6342
|
+
const rows2 = stmt2.all(tableName, vectorCol, queryBlob.v);
|
|
6343
|
+
const results2 = [];
|
|
6344
|
+
for (const row of rows2) {
|
|
6345
|
+
const score = 1 - row.distance;
|
|
6346
|
+
if (score < scoreThreshold) {
|
|
6347
|
+
continue;
|
|
6348
|
+
}
|
|
6349
|
+
const entity = { ...row };
|
|
6350
|
+
delete entity.distance;
|
|
6351
|
+
for (const k in this.schema.properties) {
|
|
6352
|
+
entity[k] = this.sqlToJsValue(k, entity[k]);
|
|
6353
|
+
}
|
|
6354
|
+
const metadata = metadataCol ? entity[metadataCol] : {};
|
|
6355
|
+
if (filter && !matchesFilter3(metadata, filter)) {
|
|
6356
|
+
continue;
|
|
6357
|
+
}
|
|
6358
|
+
results2.push({ ...entity, score });
|
|
6359
|
+
if (results2.length >= topK) {
|
|
6360
|
+
break;
|
|
6361
|
+
}
|
|
6362
|
+
}
|
|
6363
|
+
results2.sort((a, b) => b.score - a.score);
|
|
6364
|
+
return results2.slice(0, topK);
|
|
6365
|
+
}
|
|
6366
|
+
const sql = `
|
|
6367
|
+
SELECT t.*, v.distance
|
|
6368
|
+
FROM ${escapeIdentifier(tableName)} AS t
|
|
6369
|
+
JOIN vector_full_scan(?, ?, ?, ?) AS v
|
|
6370
|
+
ON t.rowid = v.rowid
|
|
6371
|
+
ORDER BY v.distance ASC
|
|
6372
|
+
`;
|
|
6373
|
+
const stmt = db.prepare(sql);
|
|
6374
|
+
const rows = stmt.all(tableName, vectorCol, queryBlob.v, topK);
|
|
6375
|
+
const results = [];
|
|
6376
|
+
for (const row of rows) {
|
|
6377
|
+
const score = 1 - row.distance;
|
|
6378
|
+
if (score < scoreThreshold) {
|
|
6379
|
+
continue;
|
|
6380
|
+
}
|
|
6381
|
+
const entity = { ...row };
|
|
6382
|
+
delete entity.distance;
|
|
6383
|
+
for (const k in this.schema.properties) {
|
|
6384
|
+
entity[k] = this.sqlToJsValue(k, entity[k]);
|
|
6385
|
+
}
|
|
6386
|
+
results.push({ ...entity, score });
|
|
6387
|
+
}
|
|
6388
|
+
return results;
|
|
6389
|
+
} catch (error) {
|
|
6390
|
+
console.warn("sqlite-vector query failed, falling back to in-memory search:", error);
|
|
6391
|
+
return this.searchFallback(query, options);
|
|
6392
|
+
}
|
|
6393
|
+
}
|
|
6394
|
+
async hybridSearch(query, options) {
|
|
6395
|
+
const { topK = 10, filter, scoreThreshold = 0, textQuery, vectorWeight = 0.7 } = options;
|
|
6396
|
+
if (!textQuery || textQuery.trim().length === 0) {
|
|
6397
|
+
return this.similaritySearch(query, { topK, filter, scoreThreshold });
|
|
6398
|
+
}
|
|
6399
|
+
if (!this.extensionLoaded) {
|
|
6400
|
+
return this.hybridSearchFallback(query, options);
|
|
6401
|
+
}
|
|
6402
|
+
const db = this.database;
|
|
6403
|
+
const tableName = this.table;
|
|
6404
|
+
const vectorCol = String(this.vectorPropertyName);
|
|
6405
|
+
const metadataCol = this.metadataPropertyName ? String(this.metadataPropertyName) : null;
|
|
6406
|
+
try {
|
|
6407
|
+
const queryJson = this.encodeVectorJson(query);
|
|
6408
|
+
const queryBlob = db.prepare(`SELECT vector_as_${this.vectorTypeSuffix}(?) as v`).get(queryJson);
|
|
6409
|
+
const sql = `
|
|
6410
|
+
SELECT t.*, v.distance
|
|
6411
|
+
FROM ${escapeIdentifier(tableName)} AS t
|
|
6412
|
+
JOIN vector_full_scan(?, ?, ?) AS v
|
|
6413
|
+
ON t.rowid = v.rowid
|
|
6414
|
+
ORDER BY v.distance ASC
|
|
6415
|
+
`;
|
|
6416
|
+
const stmt = db.prepare(sql);
|
|
6417
|
+
const rows = stmt.all(tableName, vectorCol, queryBlob.v);
|
|
6418
|
+
const queryLower = textQuery.toLowerCase();
|
|
6419
|
+
const queryWords = queryLower.split(/\s+/).filter((w) => w.length > 0);
|
|
6420
|
+
const results = [];
|
|
6421
|
+
for (const row of rows) {
|
|
6422
|
+
const vectorScore = 1 - row.distance;
|
|
6423
|
+
const entity = { ...row };
|
|
6424
|
+
delete entity.distance;
|
|
6425
|
+
for (const k in this.schema.properties) {
|
|
6426
|
+
entity[k] = this.sqlToJsValue(k, entity[k]);
|
|
6427
|
+
}
|
|
6428
|
+
const metadata = metadataCol ? entity[metadataCol] : {};
|
|
6429
|
+
if (filter && !matchesFilter3(metadata, filter)) {
|
|
6430
|
+
continue;
|
|
6431
|
+
}
|
|
6432
|
+
const metadataText = JSON.stringify(metadata).toLowerCase();
|
|
6433
|
+
let textScore = 0;
|
|
6434
|
+
if (queryWords.length > 0) {
|
|
6435
|
+
let matches = 0;
|
|
6436
|
+
for (const word of queryWords) {
|
|
6437
|
+
if (metadataText.includes(word)) {
|
|
6438
|
+
matches++;
|
|
6439
|
+
}
|
|
6440
|
+
}
|
|
6441
|
+
textScore = matches / queryWords.length;
|
|
6442
|
+
}
|
|
6443
|
+
const combinedScore = vectorWeight * vectorScore + (1 - vectorWeight) * textScore;
|
|
6444
|
+
if (combinedScore < scoreThreshold) {
|
|
6445
|
+
continue;
|
|
6446
|
+
}
|
|
6447
|
+
results.push({ ...entity, score: combinedScore });
|
|
6448
|
+
}
|
|
6449
|
+
results.sort((a, b) => b.score - a.score);
|
|
6450
|
+
return results.slice(0, topK);
|
|
6451
|
+
} catch (error) {
|
|
6452
|
+
console.warn("sqlite-vector hybrid query failed, falling back to in-memory search:", error);
|
|
6453
|
+
return this.hybridSearchFallback(query, options);
|
|
6454
|
+
}
|
|
6455
|
+
}
|
|
6456
|
+
async searchFallback(query, options) {
|
|
6457
|
+
const { topK = 10, filter, scoreThreshold = 0 } = options;
|
|
6458
|
+
const allRows = await this.getAll() || [];
|
|
6459
|
+
const results = [];
|
|
6460
|
+
for (const row of allRows) {
|
|
6461
|
+
const vector = row[this.vectorPropertyName];
|
|
6462
|
+
const metadata = this.metadataPropertyName ? row[this.metadataPropertyName] : {};
|
|
6463
|
+
if (filter && !matchesFilter3(metadata, filter)) {
|
|
6464
|
+
continue;
|
|
6465
|
+
}
|
|
6466
|
+
const score = cosineSimilarity4(query, vector);
|
|
6467
|
+
if (score >= scoreThreshold) {
|
|
6468
|
+
results.push({ ...row, score });
|
|
6469
|
+
}
|
|
6470
|
+
}
|
|
6471
|
+
results.sort((a, b) => b.score - a.score);
|
|
6472
|
+
return results.slice(0, topK);
|
|
6473
|
+
}
|
|
6474
|
+
async hybridSearchFallback(query, options) {
|
|
6475
|
+
const { topK = 10, filter, scoreThreshold = 0, textQuery, vectorWeight = 0.7 } = options;
|
|
6476
|
+
const allRows = await this.getAll() || [];
|
|
6477
|
+
const results = [];
|
|
6478
|
+
const queryLower = textQuery.toLowerCase();
|
|
6479
|
+
const queryWords = queryLower.split(/\s+/).filter((w) => w.length > 0);
|
|
6480
|
+
for (const row of allRows) {
|
|
6481
|
+
const vector = row[this.vectorPropertyName];
|
|
6482
|
+
const metadata = this.metadataPropertyName ? row[this.metadataPropertyName] : {};
|
|
6483
|
+
if (filter && !matchesFilter3(metadata, filter)) {
|
|
6484
|
+
continue;
|
|
6485
|
+
}
|
|
6486
|
+
const vectorScore = cosineSimilarity4(query, vector);
|
|
6487
|
+
const metadataText = JSON.stringify(metadata).toLowerCase();
|
|
6488
|
+
let textScore = 0;
|
|
6489
|
+
if (queryWords.length > 0) {
|
|
6490
|
+
let matches = 0;
|
|
6491
|
+
for (const word of queryWords) {
|
|
6492
|
+
if (metadataText.includes(word)) {
|
|
6493
|
+
matches++;
|
|
6494
|
+
}
|
|
6495
|
+
}
|
|
6496
|
+
textScore = matches / queryWords.length;
|
|
6497
|
+
}
|
|
6498
|
+
const combinedScore = vectorWeight * vectorScore + (1 - vectorWeight) * textScore;
|
|
6499
|
+
if (combinedScore >= scoreThreshold) {
|
|
6500
|
+
results.push({ ...row, score: combinedScore });
|
|
6501
|
+
}
|
|
6502
|
+
}
|
|
6503
|
+
results.sort((a, b) => b.score - a.score);
|
|
6504
|
+
return results.slice(0, topK);
|
|
6505
|
+
}
|
|
6506
|
+
}
|
|
5899
6507
|
// src/kv/IndexedDbKvStorage.ts
|
|
5900
6508
|
import { createServiceToken as createServiceToken28 } from "@workglow/util";
|
|
5901
6509
|
|
|
@@ -7484,9 +8092,9 @@ class IndexedDbQueueStorage {
|
|
|
7484
8092
|
}
|
|
7485
8093
|
}
|
|
7486
8094
|
// src/vector/IndexedDbVectorStorage.ts
|
|
7487
|
-
import { cosineSimilarity as
|
|
8095
|
+
import { cosineSimilarity as cosineSimilarity5, createServiceToken as createServiceToken31 } from "@workglow/util";
|
|
7488
8096
|
var IDB_VECTOR_REPOSITORY = createServiceToken31("storage.vectorRepository.indexedDb");
|
|
7489
|
-
function
|
|
8097
|
+
function matchesFilter4(metadata, filter) {
|
|
7490
8098
|
for (const [key, value] of Object.entries(filter)) {
|
|
7491
8099
|
if (metadata[key] !== value) {
|
|
7492
8100
|
return false;
|
|
@@ -7536,10 +8144,10 @@ class IndexedDbVectorStorage extends IndexedDbTabularStorage {
|
|
|
7536
8144
|
for (const entity of allEntities) {
|
|
7537
8145
|
const vector = entity[this.vectorPropertyName];
|
|
7538
8146
|
const metadata = this.metadataPropertyName ? entity[this.metadataPropertyName] : {};
|
|
7539
|
-
if (filter && !
|
|
8147
|
+
if (filter && !matchesFilter4(metadata, filter)) {
|
|
7540
8148
|
continue;
|
|
7541
8149
|
}
|
|
7542
|
-
const score =
|
|
8150
|
+
const score = cosineSimilarity5(query, vector);
|
|
7543
8151
|
if (score < scoreThreshold) {
|
|
7544
8152
|
continue;
|
|
7545
8153
|
}
|
|
@@ -7562,10 +8170,10 @@ class IndexedDbVectorStorage extends IndexedDbTabularStorage {
|
|
|
7562
8170
|
for (const entity of allEntities) {
|
|
7563
8171
|
const vector = entity[this.vectorPropertyName];
|
|
7564
8172
|
const metadata = this.metadataPropertyName ? entity[this.metadataPropertyName] : {};
|
|
7565
|
-
if (filter && !
|
|
8173
|
+
if (filter && !matchesFilter4(metadata, filter)) {
|
|
7566
8174
|
continue;
|
|
7567
8175
|
}
|
|
7568
|
-
const vectorScore =
|
|
8176
|
+
const vectorScore = cosineSimilarity5(query, vector);
|
|
7569
8177
|
const metadataText = Object.values(metadata).join(" ").toLowerCase();
|
|
7570
8178
|
const textScore = textRelevance2(metadataText, textQuery);
|
|
7571
8179
|
const combinedScore = vectorWeight * vectorScore + (1 - vectorWeight) * textScore;
|
|
@@ -7583,6 +8191,7 @@ class IndexedDbVectorStorage extends IndexedDbTabularStorage {
|
|
|
7583
8191
|
}
|
|
7584
8192
|
}
|
|
7585
8193
|
export {
|
|
8194
|
+
traced,
|
|
7586
8195
|
registerTabularRepository,
|
|
7587
8196
|
isSearchCondition,
|
|
7588
8197
|
getVectorProperty,
|
|
@@ -7591,6 +8200,10 @@ export {
|
|
|
7591
8200
|
getGlobalTabularRepositories,
|
|
7592
8201
|
ensureIndexedDbTable,
|
|
7593
8202
|
dropIndexedDbTable,
|
|
8203
|
+
TelemetryVectorStorage,
|
|
8204
|
+
TelemetryTabularStorage,
|
|
8205
|
+
TelemetryQueueStorage,
|
|
8206
|
+
TelemetryKvStorage,
|
|
7594
8207
|
TABULAR_REPOSITORY,
|
|
7595
8208
|
TABULAR_REPOSITORIES,
|
|
7596
8209
|
SupabaseTabularStorage,
|
|
@@ -7608,6 +8221,7 @@ export {
|
|
|
7608
8221
|
SqliteRateLimiterStorage,
|
|
7609
8222
|
SqliteQueueStorage,
|
|
7610
8223
|
SqliteKvStorage,
|
|
8224
|
+
SqliteAiVectorStorage,
|
|
7611
8225
|
SUPABASE_TABULAR_REPOSITORY,
|
|
7612
8226
|
SUPABASE_RATE_LIMITER_STORAGE,
|
|
7613
8227
|
SUPABASE_QUEUE_STORAGE,
|
|
@@ -7668,4 +8282,4 @@ export {
|
|
|
7668
8282
|
BaseTabularStorage
|
|
7669
8283
|
};
|
|
7670
8284
|
|
|
7671
|
-
//# debugId=
|
|
8285
|
+
//# debugId=5B35BE32DD643BE064756E2164756E21
|