@atscript/db-sqlite 0.1.72 → 0.1.74
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/index.cjs +346 -9
- package/dist/index.d.cts +71 -7
- package/dist/index.d.mts +71 -7
- package/dist/index.mjs +346 -9
- package/package.json +7 -3
package/README.md
CHANGED
|
@@ -20,6 +20,9 @@ SQLite adapter for `@atscript/db` with a swappable driver architecture. Ships wi
|
|
|
20
20
|
|
|
21
21
|
```bash
|
|
22
22
|
pnpm add @atscript/db-sqlite better-sqlite3
|
|
23
|
+
|
|
24
|
+
# Optional — enables @db.search.vector via the sqlite-vec extension:
|
|
25
|
+
pnpm add sqlite-vec
|
|
23
26
|
```
|
|
24
27
|
|
|
25
28
|
## Quick Start
|
|
@@ -37,10 +40,11 @@ await users.insertOne({ name: "John", email: "john@example.com" });
|
|
|
37
40
|
## Features
|
|
38
41
|
|
|
39
42
|
- Swappable driver via `TSqliteDriver` interface (5 methods)
|
|
40
|
-
- Built-in `BetterSqlite3Driver` for immediate use
|
|
43
|
+
- Built-in `BetterSqlite3Driver` for immediate use (with optional `{ vector: true }` and `loadExtensions` hooks)
|
|
41
44
|
- MongoDB-style filter translation to parameterized SQL (no injection risk)
|
|
42
45
|
- Automatic schema management from `@db.*` annotations
|
|
43
46
|
- FTS5 full-text search support
|
|
47
|
+
- Vector similarity search via [`sqlite-vec`](https://github.com/asg017/sqlite-vec) — `vec0` shadow tables with AI/AU/AD sync triggers, KNN with partition push-down, threshold control
|
|
44
48
|
- Embedded object flattening and `@db.json` storage
|
|
45
49
|
- Schema sync via `@atscript/db/sync`
|
|
46
50
|
|
package/dist/index.cjs
CHANGED
|
@@ -18,6 +18,9 @@ let _atscript_db_sql_tools = require("@atscript/db-sql-tools");
|
|
|
18
18
|
* // File-based database
|
|
19
19
|
* const driver = new BetterSqlite3Driver('./my-data.db')
|
|
20
20
|
*
|
|
21
|
+
* // With sqlite-vec extension loaded
|
|
22
|
+
* const driver = new BetterSqlite3Driver('./my-data.db', { vector: true })
|
|
23
|
+
*
|
|
21
24
|
* // Pre-created instance
|
|
22
25
|
* import Database from 'better-sqlite3'
|
|
23
26
|
* const db = new Database(':memory:', { verbose: console.log })
|
|
@@ -28,12 +31,27 @@ let _atscript_db_sql_tools = require("@atscript/db-sql-tools");
|
|
|
28
31
|
* ```bash
|
|
29
32
|
* pnpm add better-sqlite3
|
|
30
33
|
* ```
|
|
34
|
+
*
|
|
35
|
+
* Vector search support requires the optional `sqlite-vec` package:
|
|
36
|
+
* ```bash
|
|
37
|
+
* pnpm add sqlite-vec
|
|
38
|
+
* ```
|
|
31
39
|
*/
|
|
32
40
|
var BetterSqlite3Driver = class {
|
|
33
41
|
db;
|
|
42
|
+
hasVectorExt = false;
|
|
34
43
|
constructor(pathOrDb, options) {
|
|
35
|
-
|
|
36
|
-
|
|
44
|
+
const { vector, loadExtensions, ...nativeOptions } = options ?? {};
|
|
45
|
+
const req = (0, node_module.createRequire)(require("url").pathToFileURL(__filename).href);
|
|
46
|
+
if (typeof pathOrDb === "string") {
|
|
47
|
+
const Database = req("better-sqlite3");
|
|
48
|
+
this.db = new Database(pathOrDb, nativeOptions);
|
|
49
|
+
} else this.db = pathOrDb;
|
|
50
|
+
for (const ext of loadExtensions ?? []) this.db.loadExtension(ext);
|
|
51
|
+
if (vector) {
|
|
52
|
+
req("sqlite-vec").load(this.db);
|
|
53
|
+
this.hasVectorExt = true;
|
|
54
|
+
}
|
|
37
55
|
}
|
|
38
56
|
run(sql, params) {
|
|
39
57
|
const stmt = this.db.prepare(sql);
|
|
@@ -216,6 +234,27 @@ function buildAggregateCount(table, where, controls) {
|
|
|
216
234
|
return (0, _atscript_db_sql_tools.buildAggregateCount)(sqliteDialect, table, where, controls);
|
|
217
235
|
}
|
|
218
236
|
/**
|
|
237
|
+
* Converts an Atlas-style similarity score (1 = exact, 0 = orthogonal) to a
|
|
238
|
+
* vec0 distance threshold.
|
|
239
|
+
*
|
|
240
|
+
* - cosine / dotProduct (mapped to cosine in DDL): vec0 distance = 1 - cos_sim,
|
|
241
|
+
* normalized score = (1 + cos_sim) / 2, so distance = 2 * (1 - score).
|
|
242
|
+
* - euclidean (l2): vec0 distance is unbounded; the value is treated as a max
|
|
243
|
+
* distance directly (same convention as pgvector).
|
|
244
|
+
*/
|
|
245
|
+
function thresholdToVecDistance(threshold, similarity) {
|
|
246
|
+
if (similarity === "euclidean") return threshold;
|
|
247
|
+
return 2 * (1 - threshold);
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Maps an Atscript `@db.search.vector` similarity value to a sqlite-vec `vec0` metric.
|
|
251
|
+
* `dotProduct` falls back to `cosine` because vec0 has no dedicated dot-product metric.
|
|
252
|
+
*/
|
|
253
|
+
function similarityToVecMetric(s) {
|
|
254
|
+
if (s === "euclidean") return "l2";
|
|
255
|
+
return "cosine";
|
|
256
|
+
}
|
|
257
|
+
/**
|
|
219
258
|
* Maps Atscript design types to SQLite storage types.
|
|
220
259
|
*/
|
|
221
260
|
function sqliteTypeFromDesignType(designType) {
|
|
@@ -232,12 +271,12 @@ function sqliteTypeFromDesignType(designType) {
|
|
|
232
271
|
* Builds a CREATE TABLE IF NOT EXISTS statement from field descriptors.
|
|
233
272
|
* Uses pre-computed {@link TDbFieldMeta} — no raw type introspection needed.
|
|
234
273
|
*/
|
|
235
|
-
function buildCreateTable(table, fields, foreignKeys) {
|
|
274
|
+
function buildCreateTable(table, fields, foreignKeys, options) {
|
|
236
275
|
const colDefs = [];
|
|
237
276
|
const primaryKeys = fields.filter((f) => f.isPrimaryKey);
|
|
238
277
|
for (const field of fields) {
|
|
239
278
|
if (field.ignored) continue;
|
|
240
|
-
const sqlType = field.isPrimaryKey && (field.designType === "number" || field.designType === "integer") ? "INTEGER" : sqliteTypeFromDesignType(field.designType);
|
|
279
|
+
const sqlType = options?.typeMapper?.(field) ?? (field.isPrimaryKey && (field.designType === "number" || field.designType === "integer") ? "INTEGER" : sqliteTypeFromDesignType(field.designType));
|
|
241
280
|
let def = `"${esc(field.physicalName)}" ${sqlType}`;
|
|
242
281
|
if (field.isPrimaryKey && primaryKeys.length === 1) {
|
|
243
282
|
def += " PRIMARY KEY";
|
|
@@ -302,15 +341,82 @@ function buildPrefixedWhere(alias, filter) {
|
|
|
302
341
|
* const users = new AtscriptDbTable(UsersType, adapter)
|
|
303
342
|
* ```
|
|
304
343
|
*/
|
|
305
|
-
var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
|
|
344
|
+
var SqliteAdapter = class SqliteAdapter extends _atscript_db.BaseDbAdapter {
|
|
306
345
|
supportsNativeValueDefaults() {
|
|
307
346
|
return true;
|
|
308
347
|
}
|
|
348
|
+
/** Whether the SQLite connection has the sqlite-vec extension loaded. */
|
|
349
|
+
_supportsVector;
|
|
350
|
+
/** Vector fields: field path → { dimensions, similarity, indexName }. */
|
|
351
|
+
_vectorFields = /* @__PURE__ */ new Map();
|
|
352
|
+
/** Default similarity thresholds per vector index (from @db.search.vector.threshold). */
|
|
353
|
+
_vectorThresholds = /* @__PURE__ */ new Map();
|
|
354
|
+
/** Partition filter fields per vector index (from @db.search.filter). Field paths. */
|
|
355
|
+
_vectorPartitionFields = /* @__PURE__ */ new Map();
|
|
309
356
|
constructor(driver) {
|
|
310
357
|
super();
|
|
311
358
|
this.driver = driver;
|
|
312
359
|
this.driver.exec("PRAGMA foreign_keys = ON");
|
|
313
360
|
}
|
|
361
|
+
onFieldScanned(field, _type, metadata) {
|
|
362
|
+
const vectorMeta = metadata.get("db.search.vector");
|
|
363
|
+
if (vectorMeta) {
|
|
364
|
+
const indexName = vectorMeta.indexName || field;
|
|
365
|
+
this._vectorFields.set(field, {
|
|
366
|
+
dimensions: vectorMeta.dimensions,
|
|
367
|
+
similarity: vectorMeta.similarity || "cosine",
|
|
368
|
+
indexName
|
|
369
|
+
});
|
|
370
|
+
const threshold = metadata.get("db.search.vector.threshold");
|
|
371
|
+
if (threshold !== void 0) this._vectorThresholds.set(indexName, threshold);
|
|
372
|
+
}
|
|
373
|
+
for (const indexName of metadata.get("db.search.filter") || []) {
|
|
374
|
+
const list = this._vectorPartitionFields.get(indexName);
|
|
375
|
+
if (list) list.push(field);
|
|
376
|
+
else this._vectorPartitionFields.set(indexName, [field]);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
formatValue(field) {
|
|
380
|
+
if (!this._vectorFields.has(field.path)) return;
|
|
381
|
+
if (this._detectVectorSupport()) return {
|
|
382
|
+
toStorage: (value) => Array.isArray(value) ? Buffer.from(new Float32Array(value).buffer) : value,
|
|
383
|
+
fromStorage: (value) => {
|
|
384
|
+
if (value instanceof Buffer || value instanceof Uint8Array) {
|
|
385
|
+
const buf = Buffer.isBuffer(value) ? value : Buffer.from(value);
|
|
386
|
+
const arr = new Float32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4));
|
|
387
|
+
return Array.from(arr);
|
|
388
|
+
}
|
|
389
|
+
return value;
|
|
390
|
+
}
|
|
391
|
+
};
|
|
392
|
+
return {
|
|
393
|
+
toStorage: (value) => Array.isArray(value) ? JSON.stringify(value) : value,
|
|
394
|
+
fromStorage: (value) => {
|
|
395
|
+
if (typeof value === "string") try {
|
|
396
|
+
return JSON.parse(value);
|
|
397
|
+
} catch {
|
|
398
|
+
return value;
|
|
399
|
+
}
|
|
400
|
+
return value;
|
|
401
|
+
}
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
isVectorSearchable() {
|
|
405
|
+
if (this._vectorFields.size === 0) return false;
|
|
406
|
+
return this._detectVectorSupport();
|
|
407
|
+
}
|
|
408
|
+
_detectVectorSupport() {
|
|
409
|
+
if (this._supportsVector !== void 0) return this._supportsVector;
|
|
410
|
+
if (this.driver.hasVectorExt !== void 0) this._supportsVector = this.driver.hasVectorExt;
|
|
411
|
+
else try {
|
|
412
|
+
this.driver.get("SELECT vec_version()");
|
|
413
|
+
this._supportsVector = true;
|
|
414
|
+
} catch {
|
|
415
|
+
this._supportsVector = false;
|
|
416
|
+
}
|
|
417
|
+
if (!this._supportsVector && this._vectorFields.size > 0) this._log("[atscript-db-sqlite] sqlite-vec extension not available — vector fields will be stored as JSON TEXT (no similarity search).");
|
|
418
|
+
return this._supportsVector;
|
|
419
|
+
}
|
|
314
420
|
async _beginTransaction() {
|
|
315
421
|
this._log("BEGIN");
|
|
316
422
|
this.driver.exec("BEGIN");
|
|
@@ -478,7 +584,7 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
|
|
|
478
584
|
}
|
|
479
585
|
async ensureTable() {
|
|
480
586
|
if (this._table instanceof _atscript_db.AtscriptDbView) return this.ensureView();
|
|
481
|
-
const sql = buildCreateTable(this.resolveTableName(), this._table.fieldDescriptors, this._table.foreignKeys);
|
|
587
|
+
const sql = buildCreateTable(this.resolveTableName(), this._table.fieldDescriptors, this._table.foreignKeys, { typeMapper: (field) => this.typeMapper(field) });
|
|
482
588
|
this._log(sql);
|
|
483
589
|
this.driver.exec(sql);
|
|
484
590
|
this._seedIncrementStart();
|
|
@@ -539,10 +645,11 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
|
|
|
539
645
|
const tableName = this.resolveTableName();
|
|
540
646
|
const tempName = `${tableName}__tmp_${Date.now()}`;
|
|
541
647
|
this._dropAllFtsTables(tableName);
|
|
648
|
+
this._dropAllVecTables(tableName);
|
|
542
649
|
this.driver.exec("PRAGMA foreign_keys = OFF");
|
|
543
650
|
this.driver.exec("PRAGMA legacy_alter_table = ON");
|
|
544
651
|
try {
|
|
545
|
-
const createSql = buildCreateTable(tempName, this._table.fieldDescriptors, this._table.foreignKeys);
|
|
652
|
+
const createSql = buildCreateTable(tempName, this._table.fieldDescriptors, this._table.foreignKeys, { typeMapper: (field) => this.typeMapper(field) });
|
|
546
653
|
this._log(createSql);
|
|
547
654
|
this.driver.exec(createSql);
|
|
548
655
|
const oldCols = (await this.getExistingColumns()).map((c) => c.name);
|
|
@@ -576,6 +683,7 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
|
|
|
576
683
|
async dropTable() {
|
|
577
684
|
const tableName = this.resolveTableName();
|
|
578
685
|
this._dropAllFtsTables(tableName);
|
|
686
|
+
this._dropAllVecTables(tableName);
|
|
579
687
|
const ddl = `DROP TABLE IF EXISTS "${esc(tableName)}"`;
|
|
580
688
|
this._log(ddl);
|
|
581
689
|
this.driver.exec(ddl);
|
|
@@ -592,6 +700,7 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
|
|
|
592
700
|
}
|
|
593
701
|
async dropTableByName(tableName) {
|
|
594
702
|
this._dropAllFtsTables(tableName);
|
|
703
|
+
this._dropAllVecTables(tableName);
|
|
595
704
|
const ddl = `DROP TABLE IF EXISTS "${esc(tableName)}"`;
|
|
596
705
|
this._log(ddl);
|
|
597
706
|
this.driver.exec(ddl);
|
|
@@ -608,6 +717,7 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
|
|
|
608
717
|
this.driver.exec(ddl);
|
|
609
718
|
}
|
|
610
719
|
typeMapper(field) {
|
|
720
|
+
if (this._vectorFields.has(field.path)) return this._detectVectorSupport() ? "BLOB" : "TEXT";
|
|
611
721
|
if (field.isPrimaryKey && (field.designType === "number" || field.designType === "integer")) return "INTEGER";
|
|
612
722
|
return sqliteTypeFromDesignType(field.designType);
|
|
613
723
|
}
|
|
@@ -639,13 +749,21 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
|
|
|
639
749
|
shouldSkipType: (type) => type === "fulltext"
|
|
640
750
|
});
|
|
641
751
|
this._syncFtsIndexes(tableName);
|
|
752
|
+
this._syncVecIndexes(tableName);
|
|
642
753
|
}
|
|
643
754
|
getSearchIndexes() {
|
|
644
|
-
|
|
755
|
+
const indexes = [];
|
|
756
|
+
for (const idx of this._getFulltextIndexes()) indexes.push({
|
|
645
757
|
name: idx.name,
|
|
646
758
|
description: `FTS5 index (${idx.fields.map((f) => f.name).join(", ")})`,
|
|
647
759
|
type: "text"
|
|
648
|
-
})
|
|
760
|
+
});
|
|
761
|
+
for (const [field, vec] of this._vectorFields) indexes.push({
|
|
762
|
+
name: vec.indexName,
|
|
763
|
+
description: `vec0 index on ${field} (${vec.dimensions}, ${vec.similarity})`,
|
|
764
|
+
type: "vector"
|
|
765
|
+
});
|
|
766
|
+
return indexes;
|
|
649
767
|
}
|
|
650
768
|
async search(text, query, indexName) {
|
|
651
769
|
if (!text.trim()) return [];
|
|
@@ -781,6 +899,225 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
|
|
|
781
899
|
const ftsTables = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__fts__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE"));
|
|
782
900
|
for (const { name } of ftsTables) this._dropFtsTable(name);
|
|
783
901
|
}
|
|
902
|
+
static _RESIDUAL_OVERFETCH = 4;
|
|
903
|
+
async vectorSearch(vector, query, indexName) {
|
|
904
|
+
if (!this._detectVectorSupport()) throw new Error("Vector search requires the sqlite-vec extension. Construct BetterSqlite3Driver with { vector: true }.");
|
|
905
|
+
const base = this._buildVectorSearchBase(vector, query, indexName);
|
|
906
|
+
return this._runVectorSearch(base);
|
|
907
|
+
}
|
|
908
|
+
async vectorSearchWithCount(vector, query, indexName) {
|
|
909
|
+
if (!this._detectVectorSupport()) throw new Error("Vector search requires the sqlite-vec extension. Construct BetterSqlite3Driver with { vector: true }.");
|
|
910
|
+
const base = this._buildVectorSearchBase(vector, query, indexName);
|
|
911
|
+
const data = this._runVectorSearch(base);
|
|
912
|
+
const countSql = `SELECT COUNT(*) AS cnt ${base.fromWhere}`;
|
|
913
|
+
this._log(countSql, base.params);
|
|
914
|
+
return {
|
|
915
|
+
data,
|
|
916
|
+
count: this.driver.get(countSql, base.params)?.cnt ?? 0
|
|
917
|
+
};
|
|
918
|
+
}
|
|
919
|
+
_runVectorSearch(base) {
|
|
920
|
+
let sql = `SELECT * ${base.fromWhere} ORDER BY _distance ASC LIMIT ?`;
|
|
921
|
+
const params = [...base.params, base.limit];
|
|
922
|
+
if (base.skip > 0) {
|
|
923
|
+
sql += ` OFFSET ?`;
|
|
924
|
+
params.push(base.skip);
|
|
925
|
+
}
|
|
926
|
+
this._log(sql, params);
|
|
927
|
+
return this.driver.all(sql, params);
|
|
928
|
+
}
|
|
929
|
+
/** Resolves a vector index (by name or first available) and its partition fields. */
|
|
930
|
+
_resolveVectorIndex(indexName) {
|
|
931
|
+
let entry;
|
|
932
|
+
if (indexName) {
|
|
933
|
+
for (const [f, v] of this._vectorFields) if (v.indexName === indexName) {
|
|
934
|
+
entry = [f, v];
|
|
935
|
+
break;
|
|
936
|
+
}
|
|
937
|
+
if (!entry) throw new Error(`Vector index "${indexName}" not found`);
|
|
938
|
+
} else {
|
|
939
|
+
const first = this._vectorFields.entries().next();
|
|
940
|
+
if (first.done) throw new Error("No vector fields defined");
|
|
941
|
+
entry = first.value;
|
|
942
|
+
}
|
|
943
|
+
const [field, vec] = entry;
|
|
944
|
+
const partitionPhysicalNames = /* @__PURE__ */ new Set();
|
|
945
|
+
for (const logicalPath of this._vectorPartitionFields.get(vec.indexName) ?? []) {
|
|
946
|
+
const descriptor = this._table.fieldDescriptors.find((f) => f.path === logicalPath);
|
|
947
|
+
partitionPhysicalNames.add(descriptor?.physicalName ?? logicalPath);
|
|
948
|
+
}
|
|
949
|
+
return {
|
|
950
|
+
field,
|
|
951
|
+
vec,
|
|
952
|
+
partitionPhysicalNames
|
|
953
|
+
};
|
|
954
|
+
}
|
|
955
|
+
/** Query-time `$threshold` overrides the schema-level threshold (mirrors postgres precedence). */
|
|
956
|
+
_resolveVectorThreshold(controls, indexName) {
|
|
957
|
+
const queryThreshold = controls.$threshold;
|
|
958
|
+
if (queryThreshold !== void 0) return queryThreshold;
|
|
959
|
+
return this._vectorThresholds.get(indexName);
|
|
960
|
+
}
|
|
961
|
+
/**
|
|
962
|
+
* Splits the (already physical-name) filter into partition equality push-down
|
|
963
|
+
* vs. residual filter. Only top-level primitive equality is pushed down.
|
|
964
|
+
*/
|
|
965
|
+
_splitVectorFilter(filter, partitionPhysicalNames) {
|
|
966
|
+
const partition = [];
|
|
967
|
+
const residual = {};
|
|
968
|
+
if (!filter || typeof filter !== "object") return {
|
|
969
|
+
partition,
|
|
970
|
+
residual: filter
|
|
971
|
+
};
|
|
972
|
+
for (const [key, value] of Object.entries(filter)) if (partitionPhysicalNames.has(key) && value !== null && (typeof value === "string" || typeof value === "number" || typeof value === "boolean" || typeof value === "bigint")) partition.push({
|
|
973
|
+
name: key,
|
|
974
|
+
value
|
|
975
|
+
});
|
|
976
|
+
else residual[key] = value;
|
|
977
|
+
return {
|
|
978
|
+
partition,
|
|
979
|
+
residual
|
|
980
|
+
};
|
|
981
|
+
}
|
|
982
|
+
/**
|
|
983
|
+
* Builds the shared FROM+WHERE fragment for vec0 KNN queries (without ORDER/LIMIT).
|
|
984
|
+
* Both `vectorSearch` and `vectorSearchWithCount` reuse this — the former appends
|
|
985
|
+
* ORDER BY + LIMIT/OFFSET, the latter wraps it in a COUNT(*).
|
|
986
|
+
*/
|
|
987
|
+
_buildVectorSearchBase(vector, query, indexName) {
|
|
988
|
+
const { vec, partitionPhysicalNames } = this._resolveVectorIndex(indexName);
|
|
989
|
+
if (vector.length !== vec.dimensions) throw new Error(`Vector dimension mismatch: index "${vec.indexName}" expects ${vec.dimensions}, got ${vector.length}`);
|
|
990
|
+
const vecBuf = Buffer.from(new Float32Array(vector).buffer);
|
|
991
|
+
const controls = query.controls || {};
|
|
992
|
+
const limit = controls.$limit ?? 20;
|
|
993
|
+
const skip = controls.$skip ?? 0;
|
|
994
|
+
const threshold = this._resolveVectorThreshold(controls, vec.indexName);
|
|
995
|
+
const { partition, residual } = this._splitVectorFilter(query.filter, partitionPhysicalNames);
|
|
996
|
+
const residualWhere = buildPrefixedWhere("_vs", residual);
|
|
997
|
+
const hasResidual = residualWhere.sql !== "1=1";
|
|
998
|
+
const hasThreshold = threshold !== void 0;
|
|
999
|
+
const k = (limit + skip) * (hasResidual || hasThreshold ? SqliteAdapter._RESIDUAL_OVERFETCH : 1);
|
|
1000
|
+
const tableName = this.resolveTableName();
|
|
1001
|
+
const vecTable = this._vecTableName(vec.indexName);
|
|
1002
|
+
const innerWhereParts = ["v.embedding MATCH ?", "v.k = ?"];
|
|
1003
|
+
const params = [vecBuf, k];
|
|
1004
|
+
for (const p of partition) {
|
|
1005
|
+
innerWhereParts.push(`v."${esc(p.name)}" = ?`);
|
|
1006
|
+
params.push(p.value);
|
|
1007
|
+
}
|
|
1008
|
+
let fromWhere = `FROM (${`SELECT t.*, v.distance AS _distance FROM "${esc(vecTable)}" v JOIN "${esc(tableName)}" t ON t.rowid = v.rowid WHERE ${innerWhereParts.join(" AND ")}`}) _vs`;
|
|
1009
|
+
const outerWhereParts = [];
|
|
1010
|
+
if (hasThreshold) {
|
|
1011
|
+
outerWhereParts.push(`_distance <= ?`);
|
|
1012
|
+
params.push(thresholdToVecDistance(threshold, vec.similarity));
|
|
1013
|
+
}
|
|
1014
|
+
if (hasResidual) {
|
|
1015
|
+
outerWhereParts.push(`(${residualWhere.sql})`);
|
|
1016
|
+
params.push(...residualWhere.params);
|
|
1017
|
+
}
|
|
1018
|
+
if (outerWhereParts.length > 0) fromWhere += ` WHERE ${outerWhereParts.join(" AND ")}`;
|
|
1019
|
+
return {
|
|
1020
|
+
fromWhere,
|
|
1021
|
+
params,
|
|
1022
|
+
limit,
|
|
1023
|
+
skip
|
|
1024
|
+
};
|
|
1025
|
+
}
|
|
1026
|
+
/** Builds vec0 shadow table name from index name: `<table>__vec__<indexName>`. */
|
|
1027
|
+
_vecTableName(indexName) {
|
|
1028
|
+
return `${this.resolveTableName()}__vec__${indexName}`;
|
|
1029
|
+
}
|
|
1030
|
+
/**
|
|
1031
|
+
* Creates/drops vec0 virtual shadow tables and sync triggers to match desired vector fields.
|
|
1032
|
+
*/
|
|
1033
|
+
_syncVecIndexes(tableName) {
|
|
1034
|
+
if (this._vectorFields.size === 0) return;
|
|
1035
|
+
if (!this._detectVectorSupport()) {
|
|
1036
|
+
this._log("[atscript-db-sqlite] sqlite-vec not available, skipping vec0 shadow table sync (vector fields stored as JSON)");
|
|
1037
|
+
return;
|
|
1038
|
+
}
|
|
1039
|
+
const desiredVecTables = /* @__PURE__ */ new Map();
|
|
1040
|
+
for (const [fieldPath, meta] of this._vectorFields.entries()) desiredVecTables.set(this._vecTableName(meta.indexName), {
|
|
1041
|
+
field: fieldPath,
|
|
1042
|
+
meta
|
|
1043
|
+
});
|
|
1044
|
+
const existingVec = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__vec__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE")).map((r) => r.name);
|
|
1045
|
+
for (const name of existingVec) if (!desiredVecTables.has(name)) this._dropVecTable(name);
|
|
1046
|
+
const existingSet = new Set(existingVec);
|
|
1047
|
+
for (const [vecTable, { field, meta }] of desiredVecTables.entries()) if (!existingSet.has(vecTable)) this._createVecTable(tableName, vecTable, field, meta);
|
|
1048
|
+
}
|
|
1049
|
+
/** Creates a vec0 virtual shadow table with sync triggers and seeds it from existing rows. */
|
|
1050
|
+
_createVecTable(tableName, vecTable, field, vec) {
|
|
1051
|
+
const metric = similarityToVecMetric(vec.similarity);
|
|
1052
|
+
const embeddingCol = this._table.fieldDescriptors.find((f) => f.path === field)?.physicalName ?? field;
|
|
1053
|
+
const partitionFieldPaths = this._vectorPartitionFields.get(vec.indexName) ?? [];
|
|
1054
|
+
const partitionCols = [];
|
|
1055
|
+
for (const logicalPath of partitionFieldPaths) {
|
|
1056
|
+
const descriptor = this._table.fieldDescriptors.find((f) => f.path === logicalPath);
|
|
1057
|
+
if (!descriptor) {
|
|
1058
|
+
this._log(`[atscript-db-sqlite] vec0 partition field "${logicalPath}" not found for index "${vec.indexName}", defaulting to TEXT`);
|
|
1059
|
+
partitionCols.push({
|
|
1060
|
+
physicalName: logicalPath,
|
|
1061
|
+
sqlType: "TEXT"
|
|
1062
|
+
});
|
|
1063
|
+
} else partitionCols.push({
|
|
1064
|
+
physicalName: descriptor.physicalName,
|
|
1065
|
+
sqlType: this.typeMapper(descriptor)
|
|
1066
|
+
});
|
|
1067
|
+
}
|
|
1068
|
+
for (const c of partitionCols) if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(c.physicalName)) throw new Error(`vec0 partition column name "${c.physicalName}" is not a safe identifier (only [A-Za-z_][A-Za-z0-9_]* is allowed for @db.search.filter fields on SQLite).`);
|
|
1069
|
+
const ddlCols = [partitionCols.map((c) => `${c.physicalName} ${c.sqlType} partition key`).join(", "), `embedding float[${vec.dimensions}] distance_metric=${metric}`].filter(Boolean).join(", ");
|
|
1070
|
+
const createSql = `CREATE VIRTUAL TABLE IF NOT EXISTS "${esc(vecTable)}" USING vec0(${ddlCols})`;
|
|
1071
|
+
this._log(createSql);
|
|
1072
|
+
this.driver.exec(createSql);
|
|
1073
|
+
const partCols = partitionCols.map((c) => `"${esc(c.physicalName)}"`);
|
|
1074
|
+
const embCol = `"${esc(embeddingCol)}"`;
|
|
1075
|
+
const insertCols = [
|
|
1076
|
+
"rowid",
|
|
1077
|
+
...partCols,
|
|
1078
|
+
"embedding"
|
|
1079
|
+
].join(", ");
|
|
1080
|
+
const insertNewVals = [
|
|
1081
|
+
"new.rowid",
|
|
1082
|
+
...partitionCols.map((c) => `new."${esc(c.physicalName)}"`),
|
|
1083
|
+
`new.${embCol}`
|
|
1084
|
+
].join(", ");
|
|
1085
|
+
const seedColList = [
|
|
1086
|
+
"rowid",
|
|
1087
|
+
...partCols,
|
|
1088
|
+
embCol
|
|
1089
|
+
].join(", ");
|
|
1090
|
+
const ev = esc(vecTable);
|
|
1091
|
+
const et = esc(tableName);
|
|
1092
|
+
const aiSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__ai")}" AFTER INSERT ON "${et}" WHEN new.${embCol} IS NOT NULL BEGIN INSERT INTO "${ev}"(${insertCols}) VALUES(${insertNewVals}); END`;
|
|
1093
|
+
this._log(aiSql);
|
|
1094
|
+
this.driver.exec(aiSql);
|
|
1095
|
+
const adSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__ad")}" AFTER DELETE ON "${et}" BEGIN DELETE FROM "${ev}" WHERE rowid = old.rowid; END`;
|
|
1096
|
+
this._log(adSql);
|
|
1097
|
+
this.driver.exec(adSql);
|
|
1098
|
+
const auSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__au")}" AFTER UPDATE ON "${et}" BEGIN DELETE FROM "${ev}" WHERE rowid = old.rowid; INSERT INTO "${ev}"(${insertCols}) SELECT ${insertNewVals} WHERE new.${embCol} IS NOT NULL; END`;
|
|
1099
|
+
this._log(auSql);
|
|
1100
|
+
this.driver.exec(auSql);
|
|
1101
|
+
const seedSql = `INSERT INTO "${ev}"(${insertCols}) SELECT ${seedColList} FROM "${et}" WHERE ${embCol} IS NOT NULL`;
|
|
1102
|
+
this._log(seedSql);
|
|
1103
|
+
this.driver.exec(seedSql);
|
|
1104
|
+
}
|
|
1105
|
+
/** Drops a vec0 virtual table and its sync triggers. */
|
|
1106
|
+
_dropVecTable(vecTable) {
|
|
1107
|
+
for (const suffix of [
|
|
1108
|
+
"__ai",
|
|
1109
|
+
"__ad",
|
|
1110
|
+
"__au"
|
|
1111
|
+
]) this.driver.exec(`DROP TRIGGER IF EXISTS "${esc(vecTable + suffix)}"`);
|
|
1112
|
+
const sql = `DROP TABLE IF EXISTS "${esc(vecTable)}"`;
|
|
1113
|
+
this._log(sql);
|
|
1114
|
+
this.driver.exec(sql);
|
|
1115
|
+
}
|
|
1116
|
+
/** Drops all vec0 virtual shadow tables and triggers for a content table. */
|
|
1117
|
+
_dropAllVecTables(tableName) {
|
|
1118
|
+
const vecTables = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__vec__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE"));
|
|
1119
|
+
for (const { name } of vecTables) this._dropVecTable(name);
|
|
1120
|
+
}
|
|
784
1121
|
};
|
|
785
1122
|
/** Normalizes SQLite PRAGMA dflt_value to match serialized format.
|
|
786
1123
|
* PRAGMA returns `'active'` (SQL-quoted), we store `active` (raw). */
|
package/dist/index.d.cts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { BaseDbAdapter, DbQuery, DbSpace, FilterExpr, TColumnDiff, TDbDeleteResult, TDbInsertManyResult, TDbInsertResult, TDbUpdateResult, TExistingColumn, TFieldOps, TSearchIndexInfo, TSyncColumnResult } from "@atscript/db";
|
|
2
|
-
import
|
|
1
|
+
import { BaseDbAdapter, DbQuery, DbSpace, FilterExpr, TColumnDiff, TDbDeleteResult, TDbFieldMeta, TDbInsertManyResult, TDbInsertResult, TDbUpdateResult, TExistingColumn, TFieldOps, TSearchIndexInfo, TSyncColumnResult, TValueFormatterPair } from "@atscript/db";
|
|
2
|
+
import { TMetadataMap } from "@atscript/typescript/utils";
|
|
3
|
+
import * as _$better_sqlite30 from "better-sqlite3";
|
|
3
4
|
import { FilterExpr as FilterExpr$1 } from "@uniqu/core";
|
|
4
5
|
import { TSqlFragment, TSqlFragment as TSqlFragment$1 } from "@atscript/db-sql-tools";
|
|
5
6
|
|
|
@@ -47,6 +48,11 @@ interface TSqliteDriver {
|
|
|
47
48
|
* Close the database connection.
|
|
48
49
|
*/
|
|
49
50
|
close(): void;
|
|
51
|
+
/**
|
|
52
|
+
* Whether a vector search extension (e.g., `sqlite-vec`) is loaded.
|
|
53
|
+
* Consumers gate vec0 virtual tables and KNN queries on this flag.
|
|
54
|
+
*/
|
|
55
|
+
readonly hasVectorExt?: boolean;
|
|
50
56
|
}
|
|
51
57
|
//#endregion
|
|
52
58
|
//#region src/sqlite-adapter.d.ts
|
|
@@ -68,7 +74,19 @@ interface TSqliteDriver {
|
|
|
68
74
|
declare class SqliteAdapter extends BaseDbAdapter {
|
|
69
75
|
protected readonly driver: TSqliteDriver;
|
|
70
76
|
supportsNativeValueDefaults(): boolean;
|
|
77
|
+
/** Whether the SQLite connection has the sqlite-vec extension loaded. */
|
|
78
|
+
private _supportsVector;
|
|
79
|
+
/** Vector fields: field path → { dimensions, similarity, indexName }. */
|
|
80
|
+
private _vectorFields;
|
|
81
|
+
/** Default similarity thresholds per vector index (from @db.search.vector.threshold). */
|
|
82
|
+
private _vectorThresholds;
|
|
83
|
+
/** Partition filter fields per vector index (from @db.search.filter). Field paths. */
|
|
84
|
+
private _vectorPartitionFields;
|
|
71
85
|
constructor(driver: TSqliteDriver);
|
|
86
|
+
onFieldScanned(field: string, _type: unknown, metadata: TMetadataMap<AtscriptMetadata>): void;
|
|
87
|
+
formatValue(field: TDbFieldMeta): TValueFormatterPair | undefined;
|
|
88
|
+
isVectorSearchable(): boolean;
|
|
89
|
+
private _detectVectorSupport;
|
|
72
90
|
protected _beginTransaction(): Promise<unknown>;
|
|
73
91
|
protected _commitTransaction(): Promise<void>;
|
|
74
92
|
protected _rollbackTransaction(): Promise<void>;
|
|
@@ -110,10 +128,7 @@ declare class SqliteAdapter extends BaseDbAdapter {
|
|
|
110
128
|
dropTableByName(tableName: string): Promise<void>;
|
|
111
129
|
dropViewByName(viewName: string): Promise<void>;
|
|
112
130
|
renameTable(oldName: string): Promise<void>;
|
|
113
|
-
typeMapper(field:
|
|
114
|
-
designType: string;
|
|
115
|
-
isPrimaryKey: boolean;
|
|
116
|
-
}): string;
|
|
131
|
+
typeMapper(field: TDbFieldMeta): string;
|
|
117
132
|
getExistingColumnsForTable(tableName: string): Promise<TExistingColumn[]>;
|
|
118
133
|
syncIndexes(): Promise<void>;
|
|
119
134
|
getSearchIndexes(): TSearchIndexInfo[];
|
|
@@ -143,9 +158,49 @@ declare class SqliteAdapter extends BaseDbAdapter {
|
|
|
143
158
|
private _dropFtsTable;
|
|
144
159
|
/** Drops all FTS virtual tables and triggers for a content table. */
|
|
145
160
|
private _dropAllFtsTables;
|
|
161
|
+
private static readonly _RESIDUAL_OVERFETCH;
|
|
162
|
+
vectorSearch(vector: number[], query: DbQuery, indexName?: string): Promise<Array<Record<string, unknown>>>;
|
|
163
|
+
vectorSearchWithCount(vector: number[], query: DbQuery, indexName?: string): Promise<{
|
|
164
|
+
data: Array<Record<string, unknown>>;
|
|
165
|
+
count: number;
|
|
166
|
+
}>;
|
|
167
|
+
private _runVectorSearch;
|
|
168
|
+
/** Resolves a vector index (by name or first available) and its partition fields. */
|
|
169
|
+
private _resolveVectorIndex;
|
|
170
|
+
/** Query-time `$threshold` overrides the schema-level threshold (mirrors postgres precedence). */
|
|
171
|
+
private _resolveVectorThreshold;
|
|
172
|
+
/**
|
|
173
|
+
* Splits the (already physical-name) filter into partition equality push-down
|
|
174
|
+
* vs. residual filter. Only top-level primitive equality is pushed down.
|
|
175
|
+
*/
|
|
176
|
+
private _splitVectorFilter;
|
|
177
|
+
/**
|
|
178
|
+
* Builds the shared FROM+WHERE fragment for vec0 KNN queries (without ORDER/LIMIT).
|
|
179
|
+
* Both `vectorSearch` and `vectorSearchWithCount` reuse this — the former appends
|
|
180
|
+
* ORDER BY + LIMIT/OFFSET, the latter wraps it in a COUNT(*).
|
|
181
|
+
*/
|
|
182
|
+
private _buildVectorSearchBase;
|
|
183
|
+
/** Builds vec0 shadow table name from index name: `<table>__vec__<indexName>`. */
|
|
184
|
+
private _vecTableName;
|
|
185
|
+
/**
|
|
186
|
+
* Creates/drops vec0 virtual shadow tables and sync triggers to match desired vector fields.
|
|
187
|
+
*/
|
|
188
|
+
private _syncVecIndexes;
|
|
189
|
+
/** Creates a vec0 virtual shadow table with sync triggers and seeds it from existing rows. */
|
|
190
|
+
private _createVecTable;
|
|
191
|
+
/** Drops a vec0 virtual table and its sync triggers. */
|
|
192
|
+
private _dropVecTable;
|
|
193
|
+
/** Drops all vec0 virtual shadow tables and triggers for a content table. */
|
|
194
|
+
private _dropAllVecTables;
|
|
146
195
|
}
|
|
147
196
|
//#endregion
|
|
148
197
|
//#region src/better-sqlite3-driver.d.ts
|
|
198
|
+
interface TBetterSqlite3DriverOptions extends Record<string, unknown> {
|
|
199
|
+
/** Load the optional `sqlite-vec` extension. */
|
|
200
|
+
vector?: boolean;
|
|
201
|
+
/** Absolute paths to SQLite loadable extensions, passed to `Database.loadExtension`. */
|
|
202
|
+
loadExtensions?: string[];
|
|
203
|
+
}
|
|
149
204
|
/**
|
|
150
205
|
* {@link TSqliteDriver} implementation backed by `better-sqlite3`.
|
|
151
206
|
*
|
|
@@ -161,6 +216,9 @@ declare class SqliteAdapter extends BaseDbAdapter {
|
|
|
161
216
|
* // File-based database
|
|
162
217
|
* const driver = new BetterSqlite3Driver('./my-data.db')
|
|
163
218
|
*
|
|
219
|
+
* // With sqlite-vec extension loaded
|
|
220
|
+
* const driver = new BetterSqlite3Driver('./my-data.db', { vector: true })
|
|
221
|
+
*
|
|
164
222
|
* // Pre-created instance
|
|
165
223
|
* import Database from 'better-sqlite3'
|
|
166
224
|
* const db = new Database(':memory:', { verbose: console.log })
|
|
@@ -171,10 +229,16 @@ declare class SqliteAdapter extends BaseDbAdapter {
|
|
|
171
229
|
* ```bash
|
|
172
230
|
* pnpm add better-sqlite3
|
|
173
231
|
* ```
|
|
232
|
+
*
|
|
233
|
+
* Vector search support requires the optional `sqlite-vec` package:
|
|
234
|
+
* ```bash
|
|
235
|
+
* pnpm add sqlite-vec
|
|
236
|
+
* ```
|
|
174
237
|
*/
|
|
175
238
|
declare class BetterSqlite3Driver implements TSqliteDriver {
|
|
176
239
|
private db;
|
|
177
|
-
|
|
240
|
+
readonly hasVectorExt: boolean;
|
|
241
|
+
constructor(pathOrDb: string | _$better_sqlite30.Database, options?: TBetterSqlite3DriverOptions);
|
|
178
242
|
run(sql: string, params?: unknown[]): TSqliteRunResult;
|
|
179
243
|
all<T = Record<string, unknown>>(sql: string, params?: unknown[]): T[];
|
|
180
244
|
get<T = Record<string, unknown>>(sql: string, params?: unknown[]): T | null;
|
package/dist/index.d.mts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { BaseDbAdapter, DbQuery, DbSpace, FilterExpr, TColumnDiff, TDbDeleteResult, TDbInsertManyResult, TDbInsertResult, TDbUpdateResult, TExistingColumn, TFieldOps, TSearchIndexInfo, TSyncColumnResult } from "@atscript/db";
|
|
1
|
+
import { BaseDbAdapter, DbQuery, DbSpace, FilterExpr, TColumnDiff, TDbDeleteResult, TDbFieldMeta, TDbInsertManyResult, TDbInsertResult, TDbUpdateResult, TExistingColumn, TFieldOps, TSearchIndexInfo, TSyncColumnResult, TValueFormatterPair } from "@atscript/db";
|
|
2
2
|
import { TSqlFragment, TSqlFragment as TSqlFragment$1 } from "@atscript/db-sql-tools";
|
|
3
|
-
import
|
|
3
|
+
import { TMetadataMap } from "@atscript/typescript/utils";
|
|
4
|
+
import * as _$better_sqlite30 from "better-sqlite3";
|
|
4
5
|
import { FilterExpr as FilterExpr$1 } from "@uniqu/core";
|
|
5
6
|
|
|
6
7
|
//#region src/types.d.ts
|
|
@@ -47,6 +48,11 @@ interface TSqliteDriver {
|
|
|
47
48
|
* Close the database connection.
|
|
48
49
|
*/
|
|
49
50
|
close(): void;
|
|
51
|
+
/**
|
|
52
|
+
* Whether a vector search extension (e.g., `sqlite-vec`) is loaded.
|
|
53
|
+
* Consumers gate vec0 virtual tables and KNN queries on this flag.
|
|
54
|
+
*/
|
|
55
|
+
readonly hasVectorExt?: boolean;
|
|
50
56
|
}
|
|
51
57
|
//#endregion
|
|
52
58
|
//#region src/sqlite-adapter.d.ts
|
|
@@ -68,7 +74,19 @@ interface TSqliteDriver {
|
|
|
68
74
|
declare class SqliteAdapter extends BaseDbAdapter {
|
|
69
75
|
protected readonly driver: TSqliteDriver;
|
|
70
76
|
supportsNativeValueDefaults(): boolean;
|
|
77
|
+
/** Whether the SQLite connection has the sqlite-vec extension loaded. */
|
|
78
|
+
private _supportsVector;
|
|
79
|
+
/** Vector fields: field path → { dimensions, similarity, indexName }. */
|
|
80
|
+
private _vectorFields;
|
|
81
|
+
/** Default similarity thresholds per vector index (from @db.search.vector.threshold). */
|
|
82
|
+
private _vectorThresholds;
|
|
83
|
+
/** Partition filter fields per vector index (from @db.search.filter). Field paths. */
|
|
84
|
+
private _vectorPartitionFields;
|
|
71
85
|
constructor(driver: TSqliteDriver);
|
|
86
|
+
onFieldScanned(field: string, _type: unknown, metadata: TMetadataMap<AtscriptMetadata>): void;
|
|
87
|
+
formatValue(field: TDbFieldMeta): TValueFormatterPair | undefined;
|
|
88
|
+
isVectorSearchable(): boolean;
|
|
89
|
+
private _detectVectorSupport;
|
|
72
90
|
protected _beginTransaction(): Promise<unknown>;
|
|
73
91
|
protected _commitTransaction(): Promise<void>;
|
|
74
92
|
protected _rollbackTransaction(): Promise<void>;
|
|
@@ -110,10 +128,7 @@ declare class SqliteAdapter extends BaseDbAdapter {
|
|
|
110
128
|
dropTableByName(tableName: string): Promise<void>;
|
|
111
129
|
dropViewByName(viewName: string): Promise<void>;
|
|
112
130
|
renameTable(oldName: string): Promise<void>;
|
|
113
|
-
typeMapper(field:
|
|
114
|
-
designType: string;
|
|
115
|
-
isPrimaryKey: boolean;
|
|
116
|
-
}): string;
|
|
131
|
+
typeMapper(field: TDbFieldMeta): string;
|
|
117
132
|
getExistingColumnsForTable(tableName: string): Promise<TExistingColumn[]>;
|
|
118
133
|
syncIndexes(): Promise<void>;
|
|
119
134
|
getSearchIndexes(): TSearchIndexInfo[];
|
|
@@ -143,9 +158,49 @@ declare class SqliteAdapter extends BaseDbAdapter {
|
|
|
143
158
|
private _dropFtsTable;
|
|
144
159
|
/** Drops all FTS virtual tables and triggers for a content table. */
|
|
145
160
|
private _dropAllFtsTables;
|
|
161
|
+
private static readonly _RESIDUAL_OVERFETCH;
|
|
162
|
+
vectorSearch(vector: number[], query: DbQuery, indexName?: string): Promise<Array<Record<string, unknown>>>;
|
|
163
|
+
vectorSearchWithCount(vector: number[], query: DbQuery, indexName?: string): Promise<{
|
|
164
|
+
data: Array<Record<string, unknown>>;
|
|
165
|
+
count: number;
|
|
166
|
+
}>;
|
|
167
|
+
private _runVectorSearch;
|
|
168
|
+
/** Resolves a vector index (by name or first available) and its partition fields. */
|
|
169
|
+
private _resolveVectorIndex;
|
|
170
|
+
/** Query-time `$threshold` overrides the schema-level threshold (mirrors postgres precedence). */
|
|
171
|
+
private _resolveVectorThreshold;
|
|
172
|
+
/**
|
|
173
|
+
* Splits the (already physical-name) filter into partition equality push-down
|
|
174
|
+
* vs. residual filter. Only top-level primitive equality is pushed down.
|
|
175
|
+
*/
|
|
176
|
+
private _splitVectorFilter;
|
|
177
|
+
/**
|
|
178
|
+
* Builds the shared FROM+WHERE fragment for vec0 KNN queries (without ORDER/LIMIT).
|
|
179
|
+
* Both `vectorSearch` and `vectorSearchWithCount` reuse this — the former appends
|
|
180
|
+
* ORDER BY + LIMIT/OFFSET, the latter wraps it in a COUNT(*).
|
|
181
|
+
*/
|
|
182
|
+
private _buildVectorSearchBase;
|
|
183
|
+
/** Builds vec0 shadow table name from index name: `<table>__vec__<indexName>`. */
|
|
184
|
+
private _vecTableName;
|
|
185
|
+
/**
|
|
186
|
+
* Creates/drops vec0 virtual shadow tables and sync triggers to match desired vector fields.
|
|
187
|
+
*/
|
|
188
|
+
private _syncVecIndexes;
|
|
189
|
+
/** Creates a vec0 virtual shadow table with sync triggers and seeds it from existing rows. */
|
|
190
|
+
private _createVecTable;
|
|
191
|
+
/** Drops a vec0 virtual table and its sync triggers. */
|
|
192
|
+
private _dropVecTable;
|
|
193
|
+
/** Drops all vec0 virtual shadow tables and triggers for a content table. */
|
|
194
|
+
private _dropAllVecTables;
|
|
146
195
|
}
|
|
147
196
|
//#endregion
|
|
148
197
|
//#region src/better-sqlite3-driver.d.ts
|
|
198
|
+
interface TBetterSqlite3DriverOptions extends Record<string, unknown> {
|
|
199
|
+
/** Load the optional `sqlite-vec` extension. */
|
|
200
|
+
vector?: boolean;
|
|
201
|
+
/** Absolute paths to SQLite loadable extensions, passed to `Database.loadExtension`. */
|
|
202
|
+
loadExtensions?: string[];
|
|
203
|
+
}
|
|
149
204
|
/**
|
|
150
205
|
* {@link TSqliteDriver} implementation backed by `better-sqlite3`.
|
|
151
206
|
*
|
|
@@ -161,6 +216,9 @@ declare class SqliteAdapter extends BaseDbAdapter {
|
|
|
161
216
|
* // File-based database
|
|
162
217
|
* const driver = new BetterSqlite3Driver('./my-data.db')
|
|
163
218
|
*
|
|
219
|
+
* // With sqlite-vec extension loaded
|
|
220
|
+
* const driver = new BetterSqlite3Driver('./my-data.db', { vector: true })
|
|
221
|
+
*
|
|
164
222
|
* // Pre-created instance
|
|
165
223
|
* import Database from 'better-sqlite3'
|
|
166
224
|
* const db = new Database(':memory:', { verbose: console.log })
|
|
@@ -171,10 +229,16 @@ declare class SqliteAdapter extends BaseDbAdapter {
|
|
|
171
229
|
* ```bash
|
|
172
230
|
* pnpm add better-sqlite3
|
|
173
231
|
* ```
|
|
232
|
+
*
|
|
233
|
+
* Vector search support requires the optional `sqlite-vec` package:
|
|
234
|
+
* ```bash
|
|
235
|
+
* pnpm add sqlite-vec
|
|
236
|
+
* ```
|
|
174
237
|
*/
|
|
175
238
|
declare class BetterSqlite3Driver implements TSqliteDriver {
|
|
176
239
|
private db;
|
|
177
|
-
|
|
240
|
+
readonly hasVectorExt: boolean;
|
|
241
|
+
constructor(pathOrDb: string | _$better_sqlite30.Database, options?: TBetterSqlite3DriverOptions);
|
|
178
242
|
run(sql: string, params?: unknown[]): TSqliteRunResult;
|
|
179
243
|
all<T = Record<string, unknown>>(sql: string, params?: unknown[]): T[];
|
|
180
244
|
get<T = Record<string, unknown>>(sql: string, params?: unknown[]): T | null;
|
package/dist/index.mjs
CHANGED
|
@@ -17,6 +17,9 @@ import { buildAggregateCount, buildAggregateSelect, buildCreateView, buildDelete
|
|
|
17
17
|
* // File-based database
|
|
18
18
|
* const driver = new BetterSqlite3Driver('./my-data.db')
|
|
19
19
|
*
|
|
20
|
+
* // With sqlite-vec extension loaded
|
|
21
|
+
* const driver = new BetterSqlite3Driver('./my-data.db', { vector: true })
|
|
22
|
+
*
|
|
20
23
|
* // Pre-created instance
|
|
21
24
|
* import Database from 'better-sqlite3'
|
|
22
25
|
* const db = new Database(':memory:', { verbose: console.log })
|
|
@@ -27,12 +30,27 @@ import { buildAggregateCount, buildAggregateSelect, buildCreateView, buildDelete
|
|
|
27
30
|
* ```bash
|
|
28
31
|
* pnpm add better-sqlite3
|
|
29
32
|
* ```
|
|
33
|
+
*
|
|
34
|
+
* Vector search support requires the optional `sqlite-vec` package:
|
|
35
|
+
* ```bash
|
|
36
|
+
* pnpm add sqlite-vec
|
|
37
|
+
* ```
|
|
30
38
|
*/
|
|
31
39
|
var BetterSqlite3Driver = class {
|
|
32
40
|
db;
|
|
41
|
+
hasVectorExt = false;
|
|
33
42
|
constructor(pathOrDb, options) {
|
|
34
|
-
|
|
35
|
-
|
|
43
|
+
const { vector, loadExtensions, ...nativeOptions } = options ?? {};
|
|
44
|
+
const req = createRequire(import.meta.url);
|
|
45
|
+
if (typeof pathOrDb === "string") {
|
|
46
|
+
const Database = req("better-sqlite3");
|
|
47
|
+
this.db = new Database(pathOrDb, nativeOptions);
|
|
48
|
+
} else this.db = pathOrDb;
|
|
49
|
+
for (const ext of loadExtensions ?? []) this.db.loadExtension(ext);
|
|
50
|
+
if (vector) {
|
|
51
|
+
req("sqlite-vec").load(this.db);
|
|
52
|
+
this.hasVectorExt = true;
|
|
53
|
+
}
|
|
36
54
|
}
|
|
37
55
|
run(sql, params) {
|
|
38
56
|
const stmt = this.db.prepare(sql);
|
|
@@ -215,6 +233,27 @@ function buildAggregateCount$1(table, where, controls) {
|
|
|
215
233
|
return buildAggregateCount(sqliteDialect, table, where, controls);
|
|
216
234
|
}
|
|
217
235
|
/**
|
|
236
|
+
* Converts an Atlas-style similarity score (1 = exact, 0 = orthogonal) to a
|
|
237
|
+
* vec0 distance threshold.
|
|
238
|
+
*
|
|
239
|
+
* - cosine / dotProduct (mapped to cosine in DDL): vec0 distance = 1 - cos_sim,
|
|
240
|
+
* normalized score = (1 + cos_sim) / 2, so distance = 2 * (1 - score).
|
|
241
|
+
* - euclidean (l2): vec0 distance is unbounded; the value is treated as a max
|
|
242
|
+
* distance directly (same convention as pgvector).
|
|
243
|
+
*/
|
|
244
|
+
function thresholdToVecDistance(threshold, similarity) {
|
|
245
|
+
if (similarity === "euclidean") return threshold;
|
|
246
|
+
return 2 * (1 - threshold);
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Maps an Atscript `@db.search.vector` similarity value to a sqlite-vec `vec0` metric.
|
|
250
|
+
* `dotProduct` falls back to `cosine` because vec0 has no dedicated dot-product metric.
|
|
251
|
+
*/
|
|
252
|
+
function similarityToVecMetric(s) {
|
|
253
|
+
if (s === "euclidean") return "l2";
|
|
254
|
+
return "cosine";
|
|
255
|
+
}
|
|
256
|
+
/**
|
|
218
257
|
* Maps Atscript design types to SQLite storage types.
|
|
219
258
|
*/
|
|
220
259
|
function sqliteTypeFromDesignType(designType) {
|
|
@@ -231,12 +270,12 @@ function sqliteTypeFromDesignType(designType) {
|
|
|
231
270
|
* Builds a CREATE TABLE IF NOT EXISTS statement from field descriptors.
|
|
232
271
|
* Uses pre-computed {@link TDbFieldMeta} — no raw type introspection needed.
|
|
233
272
|
*/
|
|
234
|
-
function buildCreateTable(table, fields, foreignKeys) {
|
|
273
|
+
function buildCreateTable(table, fields, foreignKeys, options) {
|
|
235
274
|
const colDefs = [];
|
|
236
275
|
const primaryKeys = fields.filter((f) => f.isPrimaryKey);
|
|
237
276
|
for (const field of fields) {
|
|
238
277
|
if (field.ignored) continue;
|
|
239
|
-
const sqlType = field.isPrimaryKey && (field.designType === "number" || field.designType === "integer") ? "INTEGER" : sqliteTypeFromDesignType(field.designType);
|
|
278
|
+
const sqlType = options?.typeMapper?.(field) ?? (field.isPrimaryKey && (field.designType === "number" || field.designType === "integer") ? "INTEGER" : sqliteTypeFromDesignType(field.designType));
|
|
240
279
|
let def = `"${esc(field.physicalName)}" ${sqlType}`;
|
|
241
280
|
if (field.isPrimaryKey && primaryKeys.length === 1) {
|
|
242
281
|
def += " PRIMARY KEY";
|
|
@@ -301,15 +340,82 @@ function buildPrefixedWhere(alias, filter) {
|
|
|
301
340
|
* const users = new AtscriptDbTable(UsersType, adapter)
|
|
302
341
|
* ```
|
|
303
342
|
*/
|
|
304
|
-
var SqliteAdapter = class extends BaseDbAdapter {
|
|
343
|
+
var SqliteAdapter = class SqliteAdapter extends BaseDbAdapter {
|
|
305
344
|
supportsNativeValueDefaults() {
|
|
306
345
|
return true;
|
|
307
346
|
}
|
|
347
|
+
/** Whether the SQLite connection has the sqlite-vec extension loaded. */
|
|
348
|
+
_supportsVector;
|
|
349
|
+
/** Vector fields: field path → { dimensions, similarity, indexName }. */
|
|
350
|
+
_vectorFields = /* @__PURE__ */ new Map();
|
|
351
|
+
/** Default similarity thresholds per vector index (from @db.search.vector.threshold). */
|
|
352
|
+
_vectorThresholds = /* @__PURE__ */ new Map();
|
|
353
|
+
/** Partition filter fields per vector index (from @db.search.filter). Field paths. */
|
|
354
|
+
_vectorPartitionFields = /* @__PURE__ */ new Map();
|
|
308
355
|
constructor(driver) {
|
|
309
356
|
super();
|
|
310
357
|
this.driver = driver;
|
|
311
358
|
this.driver.exec("PRAGMA foreign_keys = ON");
|
|
312
359
|
}
|
|
360
|
+
onFieldScanned(field, _type, metadata) {
|
|
361
|
+
const vectorMeta = metadata.get("db.search.vector");
|
|
362
|
+
if (vectorMeta) {
|
|
363
|
+
const indexName = vectorMeta.indexName || field;
|
|
364
|
+
this._vectorFields.set(field, {
|
|
365
|
+
dimensions: vectorMeta.dimensions,
|
|
366
|
+
similarity: vectorMeta.similarity || "cosine",
|
|
367
|
+
indexName
|
|
368
|
+
});
|
|
369
|
+
const threshold = metadata.get("db.search.vector.threshold");
|
|
370
|
+
if (threshold !== void 0) this._vectorThresholds.set(indexName, threshold);
|
|
371
|
+
}
|
|
372
|
+
for (const indexName of metadata.get("db.search.filter") || []) {
|
|
373
|
+
const list = this._vectorPartitionFields.get(indexName);
|
|
374
|
+
if (list) list.push(field);
|
|
375
|
+
else this._vectorPartitionFields.set(indexName, [field]);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
formatValue(field) {
|
|
379
|
+
if (!this._vectorFields.has(field.path)) return;
|
|
380
|
+
if (this._detectVectorSupport()) return {
|
|
381
|
+
toStorage: (value) => Array.isArray(value) ? Buffer.from(new Float32Array(value).buffer) : value,
|
|
382
|
+
fromStorage: (value) => {
|
|
383
|
+
if (value instanceof Buffer || value instanceof Uint8Array) {
|
|
384
|
+
const buf = Buffer.isBuffer(value) ? value : Buffer.from(value);
|
|
385
|
+
const arr = new Float32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4));
|
|
386
|
+
return Array.from(arr);
|
|
387
|
+
}
|
|
388
|
+
return value;
|
|
389
|
+
}
|
|
390
|
+
};
|
|
391
|
+
return {
|
|
392
|
+
toStorage: (value) => Array.isArray(value) ? JSON.stringify(value) : value,
|
|
393
|
+
fromStorage: (value) => {
|
|
394
|
+
if (typeof value === "string") try {
|
|
395
|
+
return JSON.parse(value);
|
|
396
|
+
} catch {
|
|
397
|
+
return value;
|
|
398
|
+
}
|
|
399
|
+
return value;
|
|
400
|
+
}
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
isVectorSearchable() {
|
|
404
|
+
if (this._vectorFields.size === 0) return false;
|
|
405
|
+
return this._detectVectorSupport();
|
|
406
|
+
}
|
|
407
|
+
_detectVectorSupport() {
|
|
408
|
+
if (this._supportsVector !== void 0) return this._supportsVector;
|
|
409
|
+
if (this.driver.hasVectorExt !== void 0) this._supportsVector = this.driver.hasVectorExt;
|
|
410
|
+
else try {
|
|
411
|
+
this.driver.get("SELECT vec_version()");
|
|
412
|
+
this._supportsVector = true;
|
|
413
|
+
} catch {
|
|
414
|
+
this._supportsVector = false;
|
|
415
|
+
}
|
|
416
|
+
if (!this._supportsVector && this._vectorFields.size > 0) this._log("[atscript-db-sqlite] sqlite-vec extension not available — vector fields will be stored as JSON TEXT (no similarity search).");
|
|
417
|
+
return this._supportsVector;
|
|
418
|
+
}
|
|
313
419
|
async _beginTransaction() {
|
|
314
420
|
this._log("BEGIN");
|
|
315
421
|
this.driver.exec("BEGIN");
|
|
@@ -477,7 +583,7 @@ var SqliteAdapter = class extends BaseDbAdapter {
|
|
|
477
583
|
}
|
|
478
584
|
async ensureTable() {
|
|
479
585
|
if (this._table instanceof AtscriptDbView) return this.ensureView();
|
|
480
|
-
const sql = buildCreateTable(this.resolveTableName(), this._table.fieldDescriptors, this._table.foreignKeys);
|
|
586
|
+
const sql = buildCreateTable(this.resolveTableName(), this._table.fieldDescriptors, this._table.foreignKeys, { typeMapper: (field) => this.typeMapper(field) });
|
|
481
587
|
this._log(sql);
|
|
482
588
|
this.driver.exec(sql);
|
|
483
589
|
this._seedIncrementStart();
|
|
@@ -538,10 +644,11 @@ var SqliteAdapter = class extends BaseDbAdapter {
|
|
|
538
644
|
const tableName = this.resolveTableName();
|
|
539
645
|
const tempName = `${tableName}__tmp_${Date.now()}`;
|
|
540
646
|
this._dropAllFtsTables(tableName);
|
|
647
|
+
this._dropAllVecTables(tableName);
|
|
541
648
|
this.driver.exec("PRAGMA foreign_keys = OFF");
|
|
542
649
|
this.driver.exec("PRAGMA legacy_alter_table = ON");
|
|
543
650
|
try {
|
|
544
|
-
const createSql = buildCreateTable(tempName, this._table.fieldDescriptors, this._table.foreignKeys);
|
|
651
|
+
const createSql = buildCreateTable(tempName, this._table.fieldDescriptors, this._table.foreignKeys, { typeMapper: (field) => this.typeMapper(field) });
|
|
545
652
|
this._log(createSql);
|
|
546
653
|
this.driver.exec(createSql);
|
|
547
654
|
const oldCols = (await this.getExistingColumns()).map((c) => c.name);
|
|
@@ -575,6 +682,7 @@ var SqliteAdapter = class extends BaseDbAdapter {
|
|
|
575
682
|
async dropTable() {
|
|
576
683
|
const tableName = this.resolveTableName();
|
|
577
684
|
this._dropAllFtsTables(tableName);
|
|
685
|
+
this._dropAllVecTables(tableName);
|
|
578
686
|
const ddl = `DROP TABLE IF EXISTS "${esc(tableName)}"`;
|
|
579
687
|
this._log(ddl);
|
|
580
688
|
this.driver.exec(ddl);
|
|
@@ -591,6 +699,7 @@ var SqliteAdapter = class extends BaseDbAdapter {
|
|
|
591
699
|
}
|
|
592
700
|
async dropTableByName(tableName) {
|
|
593
701
|
this._dropAllFtsTables(tableName);
|
|
702
|
+
this._dropAllVecTables(tableName);
|
|
594
703
|
const ddl = `DROP TABLE IF EXISTS "${esc(tableName)}"`;
|
|
595
704
|
this._log(ddl);
|
|
596
705
|
this.driver.exec(ddl);
|
|
@@ -607,6 +716,7 @@ var SqliteAdapter = class extends BaseDbAdapter {
|
|
|
607
716
|
this.driver.exec(ddl);
|
|
608
717
|
}
|
|
609
718
|
typeMapper(field) {
|
|
719
|
+
if (this._vectorFields.has(field.path)) return this._detectVectorSupport() ? "BLOB" : "TEXT";
|
|
610
720
|
if (field.isPrimaryKey && (field.designType === "number" || field.designType === "integer")) return "INTEGER";
|
|
611
721
|
return sqliteTypeFromDesignType(field.designType);
|
|
612
722
|
}
|
|
@@ -638,13 +748,21 @@ var SqliteAdapter = class extends BaseDbAdapter {
|
|
|
638
748
|
shouldSkipType: (type) => type === "fulltext"
|
|
639
749
|
});
|
|
640
750
|
this._syncFtsIndexes(tableName);
|
|
751
|
+
this._syncVecIndexes(tableName);
|
|
641
752
|
}
|
|
642
753
|
getSearchIndexes() {
|
|
643
|
-
|
|
754
|
+
const indexes = [];
|
|
755
|
+
for (const idx of this._getFulltextIndexes()) indexes.push({
|
|
644
756
|
name: idx.name,
|
|
645
757
|
description: `FTS5 index (${idx.fields.map((f) => f.name).join(", ")})`,
|
|
646
758
|
type: "text"
|
|
647
|
-
})
|
|
759
|
+
});
|
|
760
|
+
for (const [field, vec] of this._vectorFields) indexes.push({
|
|
761
|
+
name: vec.indexName,
|
|
762
|
+
description: `vec0 index on ${field} (${vec.dimensions}, ${vec.similarity})`,
|
|
763
|
+
type: "vector"
|
|
764
|
+
});
|
|
765
|
+
return indexes;
|
|
648
766
|
}
|
|
649
767
|
async search(text, query, indexName) {
|
|
650
768
|
if (!text.trim()) return [];
|
|
@@ -780,6 +898,225 @@ var SqliteAdapter = class extends BaseDbAdapter {
|
|
|
780
898
|
const ftsTables = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__fts__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE"));
|
|
781
899
|
for (const { name } of ftsTables) this._dropFtsTable(name);
|
|
782
900
|
}
|
|
901
|
+
static _RESIDUAL_OVERFETCH = 4;
|
|
902
|
+
async vectorSearch(vector, query, indexName) {
|
|
903
|
+
if (!this._detectVectorSupport()) throw new Error("Vector search requires the sqlite-vec extension. Construct BetterSqlite3Driver with { vector: true }.");
|
|
904
|
+
const base = this._buildVectorSearchBase(vector, query, indexName);
|
|
905
|
+
return this._runVectorSearch(base);
|
|
906
|
+
}
|
|
907
|
+
async vectorSearchWithCount(vector, query, indexName) {
|
|
908
|
+
if (!this._detectVectorSupport()) throw new Error("Vector search requires the sqlite-vec extension. Construct BetterSqlite3Driver with { vector: true }.");
|
|
909
|
+
const base = this._buildVectorSearchBase(vector, query, indexName);
|
|
910
|
+
const data = this._runVectorSearch(base);
|
|
911
|
+
const countSql = `SELECT COUNT(*) AS cnt ${base.fromWhere}`;
|
|
912
|
+
this._log(countSql, base.params);
|
|
913
|
+
return {
|
|
914
|
+
data,
|
|
915
|
+
count: this.driver.get(countSql, base.params)?.cnt ?? 0
|
|
916
|
+
};
|
|
917
|
+
}
|
|
918
|
+
_runVectorSearch(base) {
|
|
919
|
+
let sql = `SELECT * ${base.fromWhere} ORDER BY _distance ASC LIMIT ?`;
|
|
920
|
+
const params = [...base.params, base.limit];
|
|
921
|
+
if (base.skip > 0) {
|
|
922
|
+
sql += ` OFFSET ?`;
|
|
923
|
+
params.push(base.skip);
|
|
924
|
+
}
|
|
925
|
+
this._log(sql, params);
|
|
926
|
+
return this.driver.all(sql, params);
|
|
927
|
+
}
|
|
928
|
+
/** Resolves a vector index (by name or first available) and its partition fields. */
|
|
929
|
+
_resolveVectorIndex(indexName) {
|
|
930
|
+
let entry;
|
|
931
|
+
if (indexName) {
|
|
932
|
+
for (const [f, v] of this._vectorFields) if (v.indexName === indexName) {
|
|
933
|
+
entry = [f, v];
|
|
934
|
+
break;
|
|
935
|
+
}
|
|
936
|
+
if (!entry) throw new Error(`Vector index "${indexName}" not found`);
|
|
937
|
+
} else {
|
|
938
|
+
const first = this._vectorFields.entries().next();
|
|
939
|
+
if (first.done) throw new Error("No vector fields defined");
|
|
940
|
+
entry = first.value;
|
|
941
|
+
}
|
|
942
|
+
const [field, vec] = entry;
|
|
943
|
+
const partitionPhysicalNames = /* @__PURE__ */ new Set();
|
|
944
|
+
for (const logicalPath of this._vectorPartitionFields.get(vec.indexName) ?? []) {
|
|
945
|
+
const descriptor = this._table.fieldDescriptors.find((f) => f.path === logicalPath);
|
|
946
|
+
partitionPhysicalNames.add(descriptor?.physicalName ?? logicalPath);
|
|
947
|
+
}
|
|
948
|
+
return {
|
|
949
|
+
field,
|
|
950
|
+
vec,
|
|
951
|
+
partitionPhysicalNames
|
|
952
|
+
};
|
|
953
|
+
}
|
|
954
|
+
/** Query-time `$threshold` overrides the schema-level threshold (mirrors postgres precedence). */
|
|
955
|
+
_resolveVectorThreshold(controls, indexName) {
|
|
956
|
+
const queryThreshold = controls.$threshold;
|
|
957
|
+
if (queryThreshold !== void 0) return queryThreshold;
|
|
958
|
+
return this._vectorThresholds.get(indexName);
|
|
959
|
+
}
|
|
960
|
+
/**
|
|
961
|
+
* Splits the (already physical-name) filter into partition equality push-down
|
|
962
|
+
* vs. residual filter. Only top-level primitive equality is pushed down.
|
|
963
|
+
*/
|
|
964
|
+
_splitVectorFilter(filter, partitionPhysicalNames) {
|
|
965
|
+
const partition = [];
|
|
966
|
+
const residual = {};
|
|
967
|
+
if (!filter || typeof filter !== "object") return {
|
|
968
|
+
partition,
|
|
969
|
+
residual: filter
|
|
970
|
+
};
|
|
971
|
+
for (const [key, value] of Object.entries(filter)) if (partitionPhysicalNames.has(key) && value !== null && (typeof value === "string" || typeof value === "number" || typeof value === "boolean" || typeof value === "bigint")) partition.push({
|
|
972
|
+
name: key,
|
|
973
|
+
value
|
|
974
|
+
});
|
|
975
|
+
else residual[key] = value;
|
|
976
|
+
return {
|
|
977
|
+
partition,
|
|
978
|
+
residual
|
|
979
|
+
};
|
|
980
|
+
}
|
|
981
|
+
/**
|
|
982
|
+
* Builds the shared FROM+WHERE fragment for vec0 KNN queries (without ORDER/LIMIT).
|
|
983
|
+
* Both `vectorSearch` and `vectorSearchWithCount` reuse this — the former appends
|
|
984
|
+
* ORDER BY + LIMIT/OFFSET, the latter wraps it in a COUNT(*).
|
|
985
|
+
*/
|
|
986
|
+
_buildVectorSearchBase(vector, query, indexName) {
|
|
987
|
+
const { vec, partitionPhysicalNames } = this._resolveVectorIndex(indexName);
|
|
988
|
+
if (vector.length !== vec.dimensions) throw new Error(`Vector dimension mismatch: index "${vec.indexName}" expects ${vec.dimensions}, got ${vector.length}`);
|
|
989
|
+
const vecBuf = Buffer.from(new Float32Array(vector).buffer);
|
|
990
|
+
const controls = query.controls || {};
|
|
991
|
+
const limit = controls.$limit ?? 20;
|
|
992
|
+
const skip = controls.$skip ?? 0;
|
|
993
|
+
const threshold = this._resolveVectorThreshold(controls, vec.indexName);
|
|
994
|
+
const { partition, residual } = this._splitVectorFilter(query.filter, partitionPhysicalNames);
|
|
995
|
+
const residualWhere = buildPrefixedWhere("_vs", residual);
|
|
996
|
+
const hasResidual = residualWhere.sql !== "1=1";
|
|
997
|
+
const hasThreshold = threshold !== void 0;
|
|
998
|
+
const k = (limit + skip) * (hasResidual || hasThreshold ? SqliteAdapter._RESIDUAL_OVERFETCH : 1);
|
|
999
|
+
const tableName = this.resolveTableName();
|
|
1000
|
+
const vecTable = this._vecTableName(vec.indexName);
|
|
1001
|
+
const innerWhereParts = ["v.embedding MATCH ?", "v.k = ?"];
|
|
1002
|
+
const params = [vecBuf, k];
|
|
1003
|
+
for (const p of partition) {
|
|
1004
|
+
innerWhereParts.push(`v."${esc(p.name)}" = ?`);
|
|
1005
|
+
params.push(p.value);
|
|
1006
|
+
}
|
|
1007
|
+
let fromWhere = `FROM (${`SELECT t.*, v.distance AS _distance FROM "${esc(vecTable)}" v JOIN "${esc(tableName)}" t ON t.rowid = v.rowid WHERE ${innerWhereParts.join(" AND ")}`}) _vs`;
|
|
1008
|
+
const outerWhereParts = [];
|
|
1009
|
+
if (hasThreshold) {
|
|
1010
|
+
outerWhereParts.push(`_distance <= ?`);
|
|
1011
|
+
params.push(thresholdToVecDistance(threshold, vec.similarity));
|
|
1012
|
+
}
|
|
1013
|
+
if (hasResidual) {
|
|
1014
|
+
outerWhereParts.push(`(${residualWhere.sql})`);
|
|
1015
|
+
params.push(...residualWhere.params);
|
|
1016
|
+
}
|
|
1017
|
+
if (outerWhereParts.length > 0) fromWhere += ` WHERE ${outerWhereParts.join(" AND ")}`;
|
|
1018
|
+
return {
|
|
1019
|
+
fromWhere,
|
|
1020
|
+
params,
|
|
1021
|
+
limit,
|
|
1022
|
+
skip
|
|
1023
|
+
};
|
|
1024
|
+
}
|
|
1025
|
+
/** Builds vec0 shadow table name from index name: `<table>__vec__<indexName>`. */
|
|
1026
|
+
_vecTableName(indexName) {
|
|
1027
|
+
return `${this.resolveTableName()}__vec__${indexName}`;
|
|
1028
|
+
}
|
|
1029
|
+
/**
|
|
1030
|
+
* Creates/drops vec0 virtual shadow tables and sync triggers to match desired vector fields.
|
|
1031
|
+
*/
|
|
1032
|
+
_syncVecIndexes(tableName) {
|
|
1033
|
+
if (this._vectorFields.size === 0) return;
|
|
1034
|
+
if (!this._detectVectorSupport()) {
|
|
1035
|
+
this._log("[atscript-db-sqlite] sqlite-vec not available, skipping vec0 shadow table sync (vector fields stored as JSON)");
|
|
1036
|
+
return;
|
|
1037
|
+
}
|
|
1038
|
+
const desiredVecTables = /* @__PURE__ */ new Map();
|
|
1039
|
+
for (const [fieldPath, meta] of this._vectorFields.entries()) desiredVecTables.set(this._vecTableName(meta.indexName), {
|
|
1040
|
+
field: fieldPath,
|
|
1041
|
+
meta
|
|
1042
|
+
});
|
|
1043
|
+
const existingVec = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__vec__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE")).map((r) => r.name);
|
|
1044
|
+
for (const name of existingVec) if (!desiredVecTables.has(name)) this._dropVecTable(name);
|
|
1045
|
+
const existingSet = new Set(existingVec);
|
|
1046
|
+
for (const [vecTable, { field, meta }] of desiredVecTables.entries()) if (!existingSet.has(vecTable)) this._createVecTable(tableName, vecTable, field, meta);
|
|
1047
|
+
}
|
|
1048
|
+
/** Creates a vec0 virtual shadow table with sync triggers and seeds it from existing rows. */
|
|
1049
|
+
_createVecTable(tableName, vecTable, field, vec) {
|
|
1050
|
+
const metric = similarityToVecMetric(vec.similarity);
|
|
1051
|
+
const embeddingCol = this._table.fieldDescriptors.find((f) => f.path === field)?.physicalName ?? field;
|
|
1052
|
+
const partitionFieldPaths = this._vectorPartitionFields.get(vec.indexName) ?? [];
|
|
1053
|
+
const partitionCols = [];
|
|
1054
|
+
for (const logicalPath of partitionFieldPaths) {
|
|
1055
|
+
const descriptor = this._table.fieldDescriptors.find((f) => f.path === logicalPath);
|
|
1056
|
+
if (!descriptor) {
|
|
1057
|
+
this._log(`[atscript-db-sqlite] vec0 partition field "${logicalPath}" not found for index "${vec.indexName}", defaulting to TEXT`);
|
|
1058
|
+
partitionCols.push({
|
|
1059
|
+
physicalName: logicalPath,
|
|
1060
|
+
sqlType: "TEXT"
|
|
1061
|
+
});
|
|
1062
|
+
} else partitionCols.push({
|
|
1063
|
+
physicalName: descriptor.physicalName,
|
|
1064
|
+
sqlType: this.typeMapper(descriptor)
|
|
1065
|
+
});
|
|
1066
|
+
}
|
|
1067
|
+
for (const c of partitionCols) if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(c.physicalName)) throw new Error(`vec0 partition column name "${c.physicalName}" is not a safe identifier (only [A-Za-z_][A-Za-z0-9_]* is allowed for @db.search.filter fields on SQLite).`);
|
|
1068
|
+
const ddlCols = [partitionCols.map((c) => `${c.physicalName} ${c.sqlType} partition key`).join(", "), `embedding float[${vec.dimensions}] distance_metric=${metric}`].filter(Boolean).join(", ");
|
|
1069
|
+
const createSql = `CREATE VIRTUAL TABLE IF NOT EXISTS "${esc(vecTable)}" USING vec0(${ddlCols})`;
|
|
1070
|
+
this._log(createSql);
|
|
1071
|
+
this.driver.exec(createSql);
|
|
1072
|
+
const partCols = partitionCols.map((c) => `"${esc(c.physicalName)}"`);
|
|
1073
|
+
const embCol = `"${esc(embeddingCol)}"`;
|
|
1074
|
+
const insertCols = [
|
|
1075
|
+
"rowid",
|
|
1076
|
+
...partCols,
|
|
1077
|
+
"embedding"
|
|
1078
|
+
].join(", ");
|
|
1079
|
+
const insertNewVals = [
|
|
1080
|
+
"new.rowid",
|
|
1081
|
+
...partitionCols.map((c) => `new."${esc(c.physicalName)}"`),
|
|
1082
|
+
`new.${embCol}`
|
|
1083
|
+
].join(", ");
|
|
1084
|
+
const seedColList = [
|
|
1085
|
+
"rowid",
|
|
1086
|
+
...partCols,
|
|
1087
|
+
embCol
|
|
1088
|
+
].join(", ");
|
|
1089
|
+
const ev = esc(vecTable);
|
|
1090
|
+
const et = esc(tableName);
|
|
1091
|
+
const aiSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__ai")}" AFTER INSERT ON "${et}" WHEN new.${embCol} IS NOT NULL BEGIN INSERT INTO "${ev}"(${insertCols}) VALUES(${insertNewVals}); END`;
|
|
1092
|
+
this._log(aiSql);
|
|
1093
|
+
this.driver.exec(aiSql);
|
|
1094
|
+
const adSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__ad")}" AFTER DELETE ON "${et}" BEGIN DELETE FROM "${ev}" WHERE rowid = old.rowid; END`;
|
|
1095
|
+
this._log(adSql);
|
|
1096
|
+
this.driver.exec(adSql);
|
|
1097
|
+
const auSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__au")}" AFTER UPDATE ON "${et}" BEGIN DELETE FROM "${ev}" WHERE rowid = old.rowid; INSERT INTO "${ev}"(${insertCols}) SELECT ${insertNewVals} WHERE new.${embCol} IS NOT NULL; END`;
|
|
1098
|
+
this._log(auSql);
|
|
1099
|
+
this.driver.exec(auSql);
|
|
1100
|
+
const seedSql = `INSERT INTO "${ev}"(${insertCols}) SELECT ${seedColList} FROM "${et}" WHERE ${embCol} IS NOT NULL`;
|
|
1101
|
+
this._log(seedSql);
|
|
1102
|
+
this.driver.exec(seedSql);
|
|
1103
|
+
}
|
|
1104
|
+
/** Drops a vec0 virtual table and its sync triggers. */
|
|
1105
|
+
_dropVecTable(vecTable) {
|
|
1106
|
+
for (const suffix of [
|
|
1107
|
+
"__ai",
|
|
1108
|
+
"__ad",
|
|
1109
|
+
"__au"
|
|
1110
|
+
]) this.driver.exec(`DROP TRIGGER IF EXISTS "${esc(vecTable + suffix)}"`);
|
|
1111
|
+
const sql = `DROP TABLE IF EXISTS "${esc(vecTable)}"`;
|
|
1112
|
+
this._log(sql);
|
|
1113
|
+
this.driver.exec(sql);
|
|
1114
|
+
}
|
|
1115
|
+
/** Drops all vec0 virtual shadow tables and triggers for a content table. */
|
|
1116
|
+
_dropAllVecTables(tableName) {
|
|
1117
|
+
const vecTables = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__vec__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE"));
|
|
1118
|
+
for (const { name } of vecTables) this._dropVecTable(name);
|
|
1119
|
+
}
|
|
783
1120
|
};
|
|
784
1121
|
/** Normalizes SQLite PRAGMA dflt_value to match serialized format.
|
|
785
1122
|
* PRAGMA returns `'active'` (SQL-quoted), we store `active` (raw). */
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@atscript/db-sqlite",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.74",
|
|
4
4
|
"description": "SQLite adapter for @atscript/db with swappable driver support.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"atscript",
|
|
@@ -41,6 +41,7 @@
|
|
|
41
41
|
"@types/better-sqlite3": "^7.6.13",
|
|
42
42
|
"@uniqu/core": "^0.1.6",
|
|
43
43
|
"better-sqlite3": "^12.6.2",
|
|
44
|
+
"sqlite-vec": "^0.1.9",
|
|
44
45
|
"unplugin-atscript": "^0.1.53"
|
|
45
46
|
},
|
|
46
47
|
"peerDependencies": {
|
|
@@ -48,14 +49,17 @@
|
|
|
48
49
|
"@atscript/typescript": "^0.1.53",
|
|
49
50
|
"@uniqu/core": "^0.1.6",
|
|
50
51
|
"better-sqlite3": ">=11.0.0",
|
|
51
|
-
"@atscript/db": "^0.1.
|
|
52
|
-
"@atscript/db
|
|
52
|
+
"@atscript/db-sql-tools": "^0.1.74",
|
|
53
|
+
"@atscript/db": "^0.1.74"
|
|
53
54
|
},
|
|
54
55
|
"peerDependenciesMeta": {
|
|
55
56
|
"better-sqlite3": {
|
|
56
57
|
"optional": true
|
|
57
58
|
}
|
|
58
59
|
},
|
|
60
|
+
"optionalDependencies": {
|
|
61
|
+
"sqlite-vec": "^0.1.9"
|
|
62
|
+
},
|
|
59
63
|
"scripts": {
|
|
60
64
|
"postinstall": "asc -f dts",
|
|
61
65
|
"build": "vp pack",
|