@atscript/db-sqlite 0.1.72 → 0.1.74

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -20,6 +20,9 @@ SQLite adapter for `@atscript/db` with a swappable driver architecture. Ships wi
20
20
 
21
21
  ```bash
22
22
  pnpm add @atscript/db-sqlite better-sqlite3
23
+
24
+ # Optional — enables @db.search.vector via the sqlite-vec extension:
25
+ pnpm add sqlite-vec
23
26
  ```
24
27
 
25
28
  ## Quick Start
@@ -37,10 +40,11 @@ await users.insertOne({ name: "John", email: "john@example.com" });
37
40
  ## Features
38
41
 
39
42
  - Swappable driver via `TSqliteDriver` interface (5 methods)
40
- - Built-in `BetterSqlite3Driver` for immediate use
43
+ - Built-in `BetterSqlite3Driver` for immediate use (with optional `{ vector: true }` and `loadExtensions` hooks)
41
44
  - MongoDB-style filter translation to parameterized SQL (no injection risk)
42
45
  - Automatic schema management from `@db.*` annotations
43
46
  - FTS5 full-text search support
47
+ - Vector similarity search via [`sqlite-vec`](https://github.com/asg017/sqlite-vec) — `vec0` shadow tables with AI/AU/AD sync triggers, KNN with partition push-down, threshold control
44
48
  - Embedded object flattening and `@db.json` storage
45
49
  - Schema sync via `@atscript/db/sync`
46
50
 
package/dist/index.cjs CHANGED
@@ -18,6 +18,9 @@ let _atscript_db_sql_tools = require("@atscript/db-sql-tools");
18
18
  * // File-based database
19
19
  * const driver = new BetterSqlite3Driver('./my-data.db')
20
20
  *
21
+ * // With sqlite-vec extension loaded
22
+ * const driver = new BetterSqlite3Driver('./my-data.db', { vector: true })
23
+ *
21
24
  * // Pre-created instance
22
25
  * import Database from 'better-sqlite3'
23
26
  * const db = new Database(':memory:', { verbose: console.log })
@@ -28,12 +31,27 @@ let _atscript_db_sql_tools = require("@atscript/db-sql-tools");
28
31
  * ```bash
29
32
  * pnpm add better-sqlite3
30
33
  * ```
34
+ *
35
+ * Vector search support requires the optional `sqlite-vec` package:
36
+ * ```bash
37
+ * pnpm add sqlite-vec
38
+ * ```
31
39
  */
32
40
  var BetterSqlite3Driver = class {
33
41
  db;
42
+ hasVectorExt = false;
34
43
  constructor(pathOrDb, options) {
35
- if (typeof pathOrDb === "string") this.db = new ((0, node_module.createRequire)(require("url").pathToFileURL(__filename).href)("better-sqlite3"))(pathOrDb, options);
36
- else this.db = pathOrDb;
44
+ const { vector, loadExtensions, ...nativeOptions } = options ?? {};
45
+ const req = (0, node_module.createRequire)(require("url").pathToFileURL(__filename).href);
46
+ if (typeof pathOrDb === "string") {
47
+ const Database = req("better-sqlite3");
48
+ this.db = new Database(pathOrDb, nativeOptions);
49
+ } else this.db = pathOrDb;
50
+ for (const ext of loadExtensions ?? []) this.db.loadExtension(ext);
51
+ if (vector) {
52
+ req("sqlite-vec").load(this.db);
53
+ this.hasVectorExt = true;
54
+ }
37
55
  }
38
56
  run(sql, params) {
39
57
  const stmt = this.db.prepare(sql);
@@ -216,6 +234,27 @@ function buildAggregateCount(table, where, controls) {
216
234
  return (0, _atscript_db_sql_tools.buildAggregateCount)(sqliteDialect, table, where, controls);
217
235
  }
218
236
  /**
237
+ * Converts an Atlas-style similarity score (1 = exact, 0 = orthogonal) to a
238
+ * vec0 distance threshold.
239
+ *
240
+ * - cosine / dotProduct (mapped to cosine in DDL): vec0 distance = 1 - cos_sim,
241
+ * normalized score = (1 + cos_sim) / 2, so distance = 2 * (1 - score).
242
+ * - euclidean (l2): vec0 distance is unbounded; the value is treated as a max
243
+ * distance directly (same convention as pgvector).
244
+ */
245
+ function thresholdToVecDistance(threshold, similarity) {
246
+ if (similarity === "euclidean") return threshold;
247
+ return 2 * (1 - threshold);
248
+ }
249
+ /**
250
+ * Maps an Atscript `@db.search.vector` similarity value to a sqlite-vec `vec0` metric.
251
+ * `dotProduct` falls back to `cosine` because vec0 has no dedicated dot-product metric.
252
+ */
253
+ function similarityToVecMetric(s) {
254
+ if (s === "euclidean") return "l2";
255
+ return "cosine";
256
+ }
257
+ /**
219
258
  * Maps Atscript design types to SQLite storage types.
220
259
  */
221
260
  function sqliteTypeFromDesignType(designType) {
@@ -232,12 +271,12 @@ function sqliteTypeFromDesignType(designType) {
232
271
  * Builds a CREATE TABLE IF NOT EXISTS statement from field descriptors.
233
272
  * Uses pre-computed {@link TDbFieldMeta} — no raw type introspection needed.
234
273
  */
235
- function buildCreateTable(table, fields, foreignKeys) {
274
+ function buildCreateTable(table, fields, foreignKeys, options) {
236
275
  const colDefs = [];
237
276
  const primaryKeys = fields.filter((f) => f.isPrimaryKey);
238
277
  for (const field of fields) {
239
278
  if (field.ignored) continue;
240
- const sqlType = field.isPrimaryKey && (field.designType === "number" || field.designType === "integer") ? "INTEGER" : sqliteTypeFromDesignType(field.designType);
279
+ const sqlType = options?.typeMapper?.(field) ?? (field.isPrimaryKey && (field.designType === "number" || field.designType === "integer") ? "INTEGER" : sqliteTypeFromDesignType(field.designType));
241
280
  let def = `"${esc(field.physicalName)}" ${sqlType}`;
242
281
  if (field.isPrimaryKey && primaryKeys.length === 1) {
243
282
  def += " PRIMARY KEY";
@@ -302,15 +341,82 @@ function buildPrefixedWhere(alias, filter) {
302
341
  * const users = new AtscriptDbTable(UsersType, adapter)
303
342
  * ```
304
343
  */
305
- var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
344
+ var SqliteAdapter = class SqliteAdapter extends _atscript_db.BaseDbAdapter {
306
345
  supportsNativeValueDefaults() {
307
346
  return true;
308
347
  }
348
+ /** Whether the SQLite connection has the sqlite-vec extension loaded. */
349
+ _supportsVector;
350
+ /** Vector fields: field path → { dimensions, similarity, indexName }. */
351
+ _vectorFields = /* @__PURE__ */ new Map();
352
+ /** Default similarity thresholds per vector index (from @db.search.vector.threshold). */
353
+ _vectorThresholds = /* @__PURE__ */ new Map();
354
+ /** Partition filter fields per vector index (from @db.search.filter). Field paths. */
355
+ _vectorPartitionFields = /* @__PURE__ */ new Map();
309
356
  constructor(driver) {
310
357
  super();
311
358
  this.driver = driver;
312
359
  this.driver.exec("PRAGMA foreign_keys = ON");
313
360
  }
361
+ onFieldScanned(field, _type, metadata) {
362
+ const vectorMeta = metadata.get("db.search.vector");
363
+ if (vectorMeta) {
364
+ const indexName = vectorMeta.indexName || field;
365
+ this._vectorFields.set(field, {
366
+ dimensions: vectorMeta.dimensions,
367
+ similarity: vectorMeta.similarity || "cosine",
368
+ indexName
369
+ });
370
+ const threshold = metadata.get("db.search.vector.threshold");
371
+ if (threshold !== void 0) this._vectorThresholds.set(indexName, threshold);
372
+ }
373
+ for (const indexName of metadata.get("db.search.filter") || []) {
374
+ const list = this._vectorPartitionFields.get(indexName);
375
+ if (list) list.push(field);
376
+ else this._vectorPartitionFields.set(indexName, [field]);
377
+ }
378
+ }
379
+ formatValue(field) {
380
+ if (!this._vectorFields.has(field.path)) return;
381
+ if (this._detectVectorSupport()) return {
382
+ toStorage: (value) => Array.isArray(value) ? Buffer.from(new Float32Array(value).buffer) : value,
383
+ fromStorage: (value) => {
384
+ if (value instanceof Buffer || value instanceof Uint8Array) {
385
+ const buf = Buffer.isBuffer(value) ? value : Buffer.from(value);
386
+ const arr = new Float32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4));
387
+ return Array.from(arr);
388
+ }
389
+ return value;
390
+ }
391
+ };
392
+ return {
393
+ toStorage: (value) => Array.isArray(value) ? JSON.stringify(value) : value,
394
+ fromStorage: (value) => {
395
+ if (typeof value === "string") try {
396
+ return JSON.parse(value);
397
+ } catch {
398
+ return value;
399
+ }
400
+ return value;
401
+ }
402
+ };
403
+ }
404
+ isVectorSearchable() {
405
+ if (this._vectorFields.size === 0) return false;
406
+ return this._detectVectorSupport();
407
+ }
408
+ _detectVectorSupport() {
409
+ if (this._supportsVector !== void 0) return this._supportsVector;
410
+ if (this.driver.hasVectorExt !== void 0) this._supportsVector = this.driver.hasVectorExt;
411
+ else try {
412
+ this.driver.get("SELECT vec_version()");
413
+ this._supportsVector = true;
414
+ } catch {
415
+ this._supportsVector = false;
416
+ }
417
+ if (!this._supportsVector && this._vectorFields.size > 0) this._log("[atscript-db-sqlite] sqlite-vec extension not available — vector fields will be stored as JSON TEXT (no similarity search).");
418
+ return this._supportsVector;
419
+ }
314
420
  async _beginTransaction() {
315
421
  this._log("BEGIN");
316
422
  this.driver.exec("BEGIN");
@@ -478,7 +584,7 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
478
584
  }
479
585
  async ensureTable() {
480
586
  if (this._table instanceof _atscript_db.AtscriptDbView) return this.ensureView();
481
- const sql = buildCreateTable(this.resolveTableName(), this._table.fieldDescriptors, this._table.foreignKeys);
587
+ const sql = buildCreateTable(this.resolveTableName(), this._table.fieldDescriptors, this._table.foreignKeys, { typeMapper: (field) => this.typeMapper(field) });
482
588
  this._log(sql);
483
589
  this.driver.exec(sql);
484
590
  this._seedIncrementStart();
@@ -539,10 +645,11 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
539
645
  const tableName = this.resolveTableName();
540
646
  const tempName = `${tableName}__tmp_${Date.now()}`;
541
647
  this._dropAllFtsTables(tableName);
648
+ this._dropAllVecTables(tableName);
542
649
  this.driver.exec("PRAGMA foreign_keys = OFF");
543
650
  this.driver.exec("PRAGMA legacy_alter_table = ON");
544
651
  try {
545
- const createSql = buildCreateTable(tempName, this._table.fieldDescriptors, this._table.foreignKeys);
652
+ const createSql = buildCreateTable(tempName, this._table.fieldDescriptors, this._table.foreignKeys, { typeMapper: (field) => this.typeMapper(field) });
546
653
  this._log(createSql);
547
654
  this.driver.exec(createSql);
548
655
  const oldCols = (await this.getExistingColumns()).map((c) => c.name);
@@ -576,6 +683,7 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
576
683
  async dropTable() {
577
684
  const tableName = this.resolveTableName();
578
685
  this._dropAllFtsTables(tableName);
686
+ this._dropAllVecTables(tableName);
579
687
  const ddl = `DROP TABLE IF EXISTS "${esc(tableName)}"`;
580
688
  this._log(ddl);
581
689
  this.driver.exec(ddl);
@@ -592,6 +700,7 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
592
700
  }
593
701
  async dropTableByName(tableName) {
594
702
  this._dropAllFtsTables(tableName);
703
+ this._dropAllVecTables(tableName);
595
704
  const ddl = `DROP TABLE IF EXISTS "${esc(tableName)}"`;
596
705
  this._log(ddl);
597
706
  this.driver.exec(ddl);
@@ -608,6 +717,7 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
608
717
  this.driver.exec(ddl);
609
718
  }
610
719
  typeMapper(field) {
720
+ if (this._vectorFields.has(field.path)) return this._detectVectorSupport() ? "BLOB" : "TEXT";
611
721
  if (field.isPrimaryKey && (field.designType === "number" || field.designType === "integer")) return "INTEGER";
612
722
  return sqliteTypeFromDesignType(field.designType);
613
723
  }
@@ -639,13 +749,21 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
639
749
  shouldSkipType: (type) => type === "fulltext"
640
750
  });
641
751
  this._syncFtsIndexes(tableName);
752
+ this._syncVecIndexes(tableName);
642
753
  }
643
754
  getSearchIndexes() {
644
- return this._getFulltextIndexes().map((idx) => ({
755
+ const indexes = [];
756
+ for (const idx of this._getFulltextIndexes()) indexes.push({
645
757
  name: idx.name,
646
758
  description: `FTS5 index (${idx.fields.map((f) => f.name).join(", ")})`,
647
759
  type: "text"
648
- }));
760
+ });
761
+ for (const [field, vec] of this._vectorFields) indexes.push({
762
+ name: vec.indexName,
763
+ description: `vec0 index on ${field} (${vec.dimensions}, ${vec.similarity})`,
764
+ type: "vector"
765
+ });
766
+ return indexes;
649
767
  }
650
768
  async search(text, query, indexName) {
651
769
  if (!text.trim()) return [];
@@ -781,6 +899,225 @@ var SqliteAdapter = class extends _atscript_db.BaseDbAdapter {
781
899
  const ftsTables = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__fts__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE"));
782
900
  for (const { name } of ftsTables) this._dropFtsTable(name);
783
901
  }
902
+ static _RESIDUAL_OVERFETCH = 4;
903
+ async vectorSearch(vector, query, indexName) {
904
+ if (!this._detectVectorSupport()) throw new Error("Vector search requires the sqlite-vec extension. Construct BetterSqlite3Driver with { vector: true }.");
905
+ const base = this._buildVectorSearchBase(vector, query, indexName);
906
+ return this._runVectorSearch(base);
907
+ }
908
+ async vectorSearchWithCount(vector, query, indexName) {
909
+ if (!this._detectVectorSupport()) throw new Error("Vector search requires the sqlite-vec extension. Construct BetterSqlite3Driver with { vector: true }.");
910
+ const base = this._buildVectorSearchBase(vector, query, indexName);
911
+ const data = this._runVectorSearch(base);
912
+ const countSql = `SELECT COUNT(*) AS cnt ${base.fromWhere}`;
913
+ this._log(countSql, base.params);
914
+ return {
915
+ data,
916
+ count: this.driver.get(countSql, base.params)?.cnt ?? 0
917
+ };
918
+ }
919
+ _runVectorSearch(base) {
920
+ let sql = `SELECT * ${base.fromWhere} ORDER BY _distance ASC LIMIT ?`;
921
+ const params = [...base.params, base.limit];
922
+ if (base.skip > 0) {
923
+ sql += ` OFFSET ?`;
924
+ params.push(base.skip);
925
+ }
926
+ this._log(sql, params);
927
+ return this.driver.all(sql, params);
928
+ }
929
+ /** Resolves a vector index (by name or first available) and its partition fields. */
930
+ _resolveVectorIndex(indexName) {
931
+ let entry;
932
+ if (indexName) {
933
+ for (const [f, v] of this._vectorFields) if (v.indexName === indexName) {
934
+ entry = [f, v];
935
+ break;
936
+ }
937
+ if (!entry) throw new Error(`Vector index "${indexName}" not found`);
938
+ } else {
939
+ const first = this._vectorFields.entries().next();
940
+ if (first.done) throw new Error("No vector fields defined");
941
+ entry = first.value;
942
+ }
943
+ const [field, vec] = entry;
944
+ const partitionPhysicalNames = /* @__PURE__ */ new Set();
945
+ for (const logicalPath of this._vectorPartitionFields.get(vec.indexName) ?? []) {
946
+ const descriptor = this._table.fieldDescriptors.find((f) => f.path === logicalPath);
947
+ partitionPhysicalNames.add(descriptor?.physicalName ?? logicalPath);
948
+ }
949
+ return {
950
+ field,
951
+ vec,
952
+ partitionPhysicalNames
953
+ };
954
+ }
955
+ /** Query-time `$threshold` overrides the schema-level threshold (mirrors postgres precedence). */
956
+ _resolveVectorThreshold(controls, indexName) {
957
+ const queryThreshold = controls.$threshold;
958
+ if (queryThreshold !== void 0) return queryThreshold;
959
+ return this._vectorThresholds.get(indexName);
960
+ }
961
+ /**
962
+ * Splits the (already physical-name) filter into partition equality push-down
963
+ * vs. residual filter. Only top-level primitive equality is pushed down.
964
+ */
965
+ _splitVectorFilter(filter, partitionPhysicalNames) {
966
+ const partition = [];
967
+ const residual = {};
968
+ if (!filter || typeof filter !== "object") return {
969
+ partition,
970
+ residual: filter
971
+ };
972
+ for (const [key, value] of Object.entries(filter)) if (partitionPhysicalNames.has(key) && value !== null && (typeof value === "string" || typeof value === "number" || typeof value === "boolean" || typeof value === "bigint")) partition.push({
973
+ name: key,
974
+ value
975
+ });
976
+ else residual[key] = value;
977
+ return {
978
+ partition,
979
+ residual
980
+ };
981
+ }
982
+ /**
983
+ * Builds the shared FROM+WHERE fragment for vec0 KNN queries (without ORDER/LIMIT).
984
+ * Both `vectorSearch` and `vectorSearchWithCount` reuse this — the former appends
985
+ * ORDER BY + LIMIT/OFFSET, the latter wraps it in a COUNT(*).
986
+ */
987
+ _buildVectorSearchBase(vector, query, indexName) {
988
+ const { vec, partitionPhysicalNames } = this._resolveVectorIndex(indexName);
989
+ if (vector.length !== vec.dimensions) throw new Error(`Vector dimension mismatch: index "${vec.indexName}" expects ${vec.dimensions}, got ${vector.length}`);
990
+ const vecBuf = Buffer.from(new Float32Array(vector).buffer);
991
+ const controls = query.controls || {};
992
+ const limit = controls.$limit ?? 20;
993
+ const skip = controls.$skip ?? 0;
994
+ const threshold = this._resolveVectorThreshold(controls, vec.indexName);
995
+ const { partition, residual } = this._splitVectorFilter(query.filter, partitionPhysicalNames);
996
+ const residualWhere = buildPrefixedWhere("_vs", residual);
997
+ const hasResidual = residualWhere.sql !== "1=1";
998
+ const hasThreshold = threshold !== void 0;
999
+ const k = (limit + skip) * (hasResidual || hasThreshold ? SqliteAdapter._RESIDUAL_OVERFETCH : 1);
1000
+ const tableName = this.resolveTableName();
1001
+ const vecTable = this._vecTableName(vec.indexName);
1002
+ const innerWhereParts = ["v.embedding MATCH ?", "v.k = ?"];
1003
+ const params = [vecBuf, k];
1004
+ for (const p of partition) {
1005
+ innerWhereParts.push(`v."${esc(p.name)}" = ?`);
1006
+ params.push(p.value);
1007
+ }
1008
+ let fromWhere = `FROM (${`SELECT t.*, v.distance AS _distance FROM "${esc(vecTable)}" v JOIN "${esc(tableName)}" t ON t.rowid = v.rowid WHERE ${innerWhereParts.join(" AND ")}`}) _vs`;
1009
+ const outerWhereParts = [];
1010
+ if (hasThreshold) {
1011
+ outerWhereParts.push(`_distance <= ?`);
1012
+ params.push(thresholdToVecDistance(threshold, vec.similarity));
1013
+ }
1014
+ if (hasResidual) {
1015
+ outerWhereParts.push(`(${residualWhere.sql})`);
1016
+ params.push(...residualWhere.params);
1017
+ }
1018
+ if (outerWhereParts.length > 0) fromWhere += ` WHERE ${outerWhereParts.join(" AND ")}`;
1019
+ return {
1020
+ fromWhere,
1021
+ params,
1022
+ limit,
1023
+ skip
1024
+ };
1025
+ }
1026
+ /** Builds vec0 shadow table name from index name: `<table>__vec__<indexName>`. */
1027
+ _vecTableName(indexName) {
1028
+ return `${this.resolveTableName()}__vec__${indexName}`;
1029
+ }
1030
+ /**
1031
+ * Creates/drops vec0 virtual shadow tables and sync triggers to match desired vector fields.
1032
+ */
1033
+ _syncVecIndexes(tableName) {
1034
+ if (this._vectorFields.size === 0) return;
1035
+ if (!this._detectVectorSupport()) {
1036
+ this._log("[atscript-db-sqlite] sqlite-vec not available, skipping vec0 shadow table sync (vector fields stored as JSON)");
1037
+ return;
1038
+ }
1039
+ const desiredVecTables = /* @__PURE__ */ new Map();
1040
+ for (const [fieldPath, meta] of this._vectorFields.entries()) desiredVecTables.set(this._vecTableName(meta.indexName), {
1041
+ field: fieldPath,
1042
+ meta
1043
+ });
1044
+ const existingVec = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__vec__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE")).map((r) => r.name);
1045
+ for (const name of existingVec) if (!desiredVecTables.has(name)) this._dropVecTable(name);
1046
+ const existingSet = new Set(existingVec);
1047
+ for (const [vecTable, { field, meta }] of desiredVecTables.entries()) if (!existingSet.has(vecTable)) this._createVecTable(tableName, vecTable, field, meta);
1048
+ }
1049
+ /** Creates a vec0 virtual shadow table with sync triggers and seeds it from existing rows. */
1050
+ _createVecTable(tableName, vecTable, field, vec) {
1051
+ const metric = similarityToVecMetric(vec.similarity);
1052
+ const embeddingCol = this._table.fieldDescriptors.find((f) => f.path === field)?.physicalName ?? field;
1053
+ const partitionFieldPaths = this._vectorPartitionFields.get(vec.indexName) ?? [];
1054
+ const partitionCols = [];
1055
+ for (const logicalPath of partitionFieldPaths) {
1056
+ const descriptor = this._table.fieldDescriptors.find((f) => f.path === logicalPath);
1057
+ if (!descriptor) {
1058
+ this._log(`[atscript-db-sqlite] vec0 partition field "${logicalPath}" not found for index "${vec.indexName}", defaulting to TEXT`);
1059
+ partitionCols.push({
1060
+ physicalName: logicalPath,
1061
+ sqlType: "TEXT"
1062
+ });
1063
+ } else partitionCols.push({
1064
+ physicalName: descriptor.physicalName,
1065
+ sqlType: this.typeMapper(descriptor)
1066
+ });
1067
+ }
1068
+ for (const c of partitionCols) if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(c.physicalName)) throw new Error(`vec0 partition column name "${c.physicalName}" is not a safe identifier (only [A-Za-z_][A-Za-z0-9_]* is allowed for @db.search.filter fields on SQLite).`);
1069
+ const ddlCols = [partitionCols.map((c) => `${c.physicalName} ${c.sqlType} partition key`).join(", "), `embedding float[${vec.dimensions}] distance_metric=${metric}`].filter(Boolean).join(", ");
1070
+ const createSql = `CREATE VIRTUAL TABLE IF NOT EXISTS "${esc(vecTable)}" USING vec0(${ddlCols})`;
1071
+ this._log(createSql);
1072
+ this.driver.exec(createSql);
1073
+ const partCols = partitionCols.map((c) => `"${esc(c.physicalName)}"`);
1074
+ const embCol = `"${esc(embeddingCol)}"`;
1075
+ const insertCols = [
1076
+ "rowid",
1077
+ ...partCols,
1078
+ "embedding"
1079
+ ].join(", ");
1080
+ const insertNewVals = [
1081
+ "new.rowid",
1082
+ ...partitionCols.map((c) => `new."${esc(c.physicalName)}"`),
1083
+ `new.${embCol}`
1084
+ ].join(", ");
1085
+ const seedColList = [
1086
+ "rowid",
1087
+ ...partCols,
1088
+ embCol
1089
+ ].join(", ");
1090
+ const ev = esc(vecTable);
1091
+ const et = esc(tableName);
1092
+ const aiSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__ai")}" AFTER INSERT ON "${et}" WHEN new.${embCol} IS NOT NULL BEGIN INSERT INTO "${ev}"(${insertCols}) VALUES(${insertNewVals}); END`;
1093
+ this._log(aiSql);
1094
+ this.driver.exec(aiSql);
1095
+ const adSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__ad")}" AFTER DELETE ON "${et}" BEGIN DELETE FROM "${ev}" WHERE rowid = old.rowid; END`;
1096
+ this._log(adSql);
1097
+ this.driver.exec(adSql);
1098
+ const auSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__au")}" AFTER UPDATE ON "${et}" BEGIN DELETE FROM "${ev}" WHERE rowid = old.rowid; INSERT INTO "${ev}"(${insertCols}) SELECT ${insertNewVals} WHERE new.${embCol} IS NOT NULL; END`;
1099
+ this._log(auSql);
1100
+ this.driver.exec(auSql);
1101
+ const seedSql = `INSERT INTO "${ev}"(${insertCols}) SELECT ${seedColList} FROM "${et}" WHERE ${embCol} IS NOT NULL`;
1102
+ this._log(seedSql);
1103
+ this.driver.exec(seedSql);
1104
+ }
1105
+ /** Drops a vec0 virtual table and its sync triggers. */
1106
+ _dropVecTable(vecTable) {
1107
+ for (const suffix of [
1108
+ "__ai",
1109
+ "__ad",
1110
+ "__au"
1111
+ ]) this.driver.exec(`DROP TRIGGER IF EXISTS "${esc(vecTable + suffix)}"`);
1112
+ const sql = `DROP TABLE IF EXISTS "${esc(vecTable)}"`;
1113
+ this._log(sql);
1114
+ this.driver.exec(sql);
1115
+ }
1116
+ /** Drops all vec0 virtual shadow tables and triggers for a content table. */
1117
+ _dropAllVecTables(tableName) {
1118
+ const vecTables = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__vec__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE"));
1119
+ for (const { name } of vecTables) this._dropVecTable(name);
1120
+ }
784
1121
  };
785
1122
  /** Normalizes SQLite PRAGMA dflt_value to match serialized format.
786
1123
  * PRAGMA returns `'active'` (SQL-quoted), we store `active` (raw). */
package/dist/index.d.cts CHANGED
@@ -1,5 +1,6 @@
1
- import { BaseDbAdapter, DbQuery, DbSpace, FilterExpr, TColumnDiff, TDbDeleteResult, TDbInsertManyResult, TDbInsertResult, TDbUpdateResult, TExistingColumn, TFieldOps, TSearchIndexInfo, TSyncColumnResult } from "@atscript/db";
2
- import * as better_sqlite30 from "better-sqlite3";
1
+ import { BaseDbAdapter, DbQuery, DbSpace, FilterExpr, TColumnDiff, TDbDeleteResult, TDbFieldMeta, TDbInsertManyResult, TDbInsertResult, TDbUpdateResult, TExistingColumn, TFieldOps, TSearchIndexInfo, TSyncColumnResult, TValueFormatterPair } from "@atscript/db";
2
+ import { TMetadataMap } from "@atscript/typescript/utils";
3
+ import * as _$better_sqlite30 from "better-sqlite3";
3
4
  import { FilterExpr as FilterExpr$1 } from "@uniqu/core";
4
5
  import { TSqlFragment, TSqlFragment as TSqlFragment$1 } from "@atscript/db-sql-tools";
5
6
 
@@ -47,6 +48,11 @@ interface TSqliteDriver {
47
48
  * Close the database connection.
48
49
  */
49
50
  close(): void;
51
+ /**
52
+ * Whether a vector search extension (e.g., `sqlite-vec`) is loaded.
53
+ * Consumers gate vec0 virtual tables and KNN queries on this flag.
54
+ */
55
+ readonly hasVectorExt?: boolean;
50
56
  }
51
57
  //#endregion
52
58
  //#region src/sqlite-adapter.d.ts
@@ -68,7 +74,19 @@ interface TSqliteDriver {
68
74
  declare class SqliteAdapter extends BaseDbAdapter {
69
75
  protected readonly driver: TSqliteDriver;
70
76
  supportsNativeValueDefaults(): boolean;
77
+ /** Whether the SQLite connection has the sqlite-vec extension loaded. */
78
+ private _supportsVector;
79
+ /** Vector fields: field path → { dimensions, similarity, indexName }. */
80
+ private _vectorFields;
81
+ /** Default similarity thresholds per vector index (from @db.search.vector.threshold). */
82
+ private _vectorThresholds;
83
+ /** Partition filter fields per vector index (from @db.search.filter). Field paths. */
84
+ private _vectorPartitionFields;
71
85
  constructor(driver: TSqliteDriver);
86
+ onFieldScanned(field: string, _type: unknown, metadata: TMetadataMap<AtscriptMetadata>): void;
87
+ formatValue(field: TDbFieldMeta): TValueFormatterPair | undefined;
88
+ isVectorSearchable(): boolean;
89
+ private _detectVectorSupport;
72
90
  protected _beginTransaction(): Promise<unknown>;
73
91
  protected _commitTransaction(): Promise<void>;
74
92
  protected _rollbackTransaction(): Promise<void>;
@@ -110,10 +128,7 @@ declare class SqliteAdapter extends BaseDbAdapter {
110
128
  dropTableByName(tableName: string): Promise<void>;
111
129
  dropViewByName(viewName: string): Promise<void>;
112
130
  renameTable(oldName: string): Promise<void>;
113
- typeMapper(field: {
114
- designType: string;
115
- isPrimaryKey: boolean;
116
- }): string;
131
+ typeMapper(field: TDbFieldMeta): string;
117
132
  getExistingColumnsForTable(tableName: string): Promise<TExistingColumn[]>;
118
133
  syncIndexes(): Promise<void>;
119
134
  getSearchIndexes(): TSearchIndexInfo[];
@@ -143,9 +158,49 @@ declare class SqliteAdapter extends BaseDbAdapter {
143
158
  private _dropFtsTable;
144
159
  /** Drops all FTS virtual tables and triggers for a content table. */
145
160
  private _dropAllFtsTables;
161
+ private static readonly _RESIDUAL_OVERFETCH;
162
+ vectorSearch(vector: number[], query: DbQuery, indexName?: string): Promise<Array<Record<string, unknown>>>;
163
+ vectorSearchWithCount(vector: number[], query: DbQuery, indexName?: string): Promise<{
164
+ data: Array<Record<string, unknown>>;
165
+ count: number;
166
+ }>;
167
+ private _runVectorSearch;
168
+ /** Resolves a vector index (by name or first available) and its partition fields. */
169
+ private _resolveVectorIndex;
170
+ /** Query-time `$threshold` overrides the schema-level threshold (mirrors postgres precedence). */
171
+ private _resolveVectorThreshold;
172
+ /**
173
+ * Splits the (already physical-name) filter into partition equality push-down
174
+ * vs. residual filter. Only top-level primitive equality is pushed down.
175
+ */
176
+ private _splitVectorFilter;
177
+ /**
178
+ * Builds the shared FROM+WHERE fragment for vec0 KNN queries (without ORDER/LIMIT).
179
+ * Both `vectorSearch` and `vectorSearchWithCount` reuse this — the former appends
180
+ * ORDER BY + LIMIT/OFFSET, the latter wraps it in a COUNT(*).
181
+ */
182
+ private _buildVectorSearchBase;
183
+ /** Builds vec0 shadow table name from index name: `<table>__vec__<indexName>`. */
184
+ private _vecTableName;
185
+ /**
186
+ * Creates/drops vec0 virtual shadow tables and sync triggers to match desired vector fields.
187
+ */
188
+ private _syncVecIndexes;
189
+ /** Creates a vec0 virtual shadow table with sync triggers and seeds it from existing rows. */
190
+ private _createVecTable;
191
+ /** Drops a vec0 virtual table and its sync triggers. */
192
+ private _dropVecTable;
193
+ /** Drops all vec0 virtual shadow tables and triggers for a content table. */
194
+ private _dropAllVecTables;
146
195
  }
147
196
  //#endregion
148
197
  //#region src/better-sqlite3-driver.d.ts
198
+ interface TBetterSqlite3DriverOptions extends Record<string, unknown> {
199
+ /** Load the optional `sqlite-vec` extension. */
200
+ vector?: boolean;
201
+ /** Absolute paths to SQLite loadable extensions, passed to `Database.loadExtension`. */
202
+ loadExtensions?: string[];
203
+ }
149
204
  /**
150
205
  * {@link TSqliteDriver} implementation backed by `better-sqlite3`.
151
206
  *
@@ -161,6 +216,9 @@ declare class SqliteAdapter extends BaseDbAdapter {
161
216
  * // File-based database
162
217
  * const driver = new BetterSqlite3Driver('./my-data.db')
163
218
  *
219
+ * // With sqlite-vec extension loaded
220
+ * const driver = new BetterSqlite3Driver('./my-data.db', { vector: true })
221
+ *
164
222
  * // Pre-created instance
165
223
  * import Database from 'better-sqlite3'
166
224
  * const db = new Database(':memory:', { verbose: console.log })
@@ -171,10 +229,16 @@ declare class SqliteAdapter extends BaseDbAdapter {
171
229
  * ```bash
172
230
  * pnpm add better-sqlite3
173
231
  * ```
232
+ *
233
+ * Vector search support requires the optional `sqlite-vec` package:
234
+ * ```bash
235
+ * pnpm add sqlite-vec
236
+ * ```
174
237
  */
175
238
  declare class BetterSqlite3Driver implements TSqliteDriver {
176
239
  private db;
177
- constructor(pathOrDb: string | better_sqlite30.Database, options?: Record<string, unknown>);
240
+ readonly hasVectorExt: boolean;
241
+ constructor(pathOrDb: string | _$better_sqlite30.Database, options?: TBetterSqlite3DriverOptions);
178
242
  run(sql: string, params?: unknown[]): TSqliteRunResult;
179
243
  all<T = Record<string, unknown>>(sql: string, params?: unknown[]): T[];
180
244
  get<T = Record<string, unknown>>(sql: string, params?: unknown[]): T | null;
package/dist/index.d.mts CHANGED
@@ -1,6 +1,7 @@
1
- import { BaseDbAdapter, DbQuery, DbSpace, FilterExpr, TColumnDiff, TDbDeleteResult, TDbInsertManyResult, TDbInsertResult, TDbUpdateResult, TExistingColumn, TFieldOps, TSearchIndexInfo, TSyncColumnResult } from "@atscript/db";
1
+ import { BaseDbAdapter, DbQuery, DbSpace, FilterExpr, TColumnDiff, TDbDeleteResult, TDbFieldMeta, TDbInsertManyResult, TDbInsertResult, TDbUpdateResult, TExistingColumn, TFieldOps, TSearchIndexInfo, TSyncColumnResult, TValueFormatterPair } from "@atscript/db";
2
2
  import { TSqlFragment, TSqlFragment as TSqlFragment$1 } from "@atscript/db-sql-tools";
3
- import * as better_sqlite30 from "better-sqlite3";
3
+ import { TMetadataMap } from "@atscript/typescript/utils";
4
+ import * as _$better_sqlite30 from "better-sqlite3";
4
5
  import { FilterExpr as FilterExpr$1 } from "@uniqu/core";
5
6
 
6
7
  //#region src/types.d.ts
@@ -47,6 +48,11 @@ interface TSqliteDriver {
47
48
  * Close the database connection.
48
49
  */
49
50
  close(): void;
51
+ /**
52
+ * Whether a vector search extension (e.g., `sqlite-vec`) is loaded.
53
+ * Consumers gate vec0 virtual tables and KNN queries on this flag.
54
+ */
55
+ readonly hasVectorExt?: boolean;
50
56
  }
51
57
  //#endregion
52
58
  //#region src/sqlite-adapter.d.ts
@@ -68,7 +74,19 @@ interface TSqliteDriver {
68
74
  declare class SqliteAdapter extends BaseDbAdapter {
69
75
  protected readonly driver: TSqliteDriver;
70
76
  supportsNativeValueDefaults(): boolean;
77
+ /** Whether the SQLite connection has the sqlite-vec extension loaded. */
78
+ private _supportsVector;
79
+ /** Vector fields: field path → { dimensions, similarity, indexName }. */
80
+ private _vectorFields;
81
+ /** Default similarity thresholds per vector index (from @db.search.vector.threshold). */
82
+ private _vectorThresholds;
83
+ /** Partition filter fields per vector index (from @db.search.filter). Field paths. */
84
+ private _vectorPartitionFields;
71
85
  constructor(driver: TSqliteDriver);
86
+ onFieldScanned(field: string, _type: unknown, metadata: TMetadataMap<AtscriptMetadata>): void;
87
+ formatValue(field: TDbFieldMeta): TValueFormatterPair | undefined;
88
+ isVectorSearchable(): boolean;
89
+ private _detectVectorSupport;
72
90
  protected _beginTransaction(): Promise<unknown>;
73
91
  protected _commitTransaction(): Promise<void>;
74
92
  protected _rollbackTransaction(): Promise<void>;
@@ -110,10 +128,7 @@ declare class SqliteAdapter extends BaseDbAdapter {
110
128
  dropTableByName(tableName: string): Promise<void>;
111
129
  dropViewByName(viewName: string): Promise<void>;
112
130
  renameTable(oldName: string): Promise<void>;
113
- typeMapper(field: {
114
- designType: string;
115
- isPrimaryKey: boolean;
116
- }): string;
131
+ typeMapper(field: TDbFieldMeta): string;
117
132
  getExistingColumnsForTable(tableName: string): Promise<TExistingColumn[]>;
118
133
  syncIndexes(): Promise<void>;
119
134
  getSearchIndexes(): TSearchIndexInfo[];
@@ -143,9 +158,49 @@ declare class SqliteAdapter extends BaseDbAdapter {
143
158
  private _dropFtsTable;
144
159
  /** Drops all FTS virtual tables and triggers for a content table. */
145
160
  private _dropAllFtsTables;
161
+ private static readonly _RESIDUAL_OVERFETCH;
162
+ vectorSearch(vector: number[], query: DbQuery, indexName?: string): Promise<Array<Record<string, unknown>>>;
163
+ vectorSearchWithCount(vector: number[], query: DbQuery, indexName?: string): Promise<{
164
+ data: Array<Record<string, unknown>>;
165
+ count: number;
166
+ }>;
167
+ private _runVectorSearch;
168
+ /** Resolves a vector index (by name or first available) and its partition fields. */
169
+ private _resolveVectorIndex;
170
+ /** Query-time `$threshold` overrides the schema-level threshold (mirrors postgres precedence). */
171
+ private _resolveVectorThreshold;
172
+ /**
173
+ * Splits the (already physical-name) filter into partition equality push-down
174
+ * vs. residual filter. Only top-level primitive equality is pushed down.
175
+ */
176
+ private _splitVectorFilter;
177
+ /**
178
+ * Builds the shared FROM+WHERE fragment for vec0 KNN queries (without ORDER/LIMIT).
179
+ * Both `vectorSearch` and `vectorSearchWithCount` reuse this — the former appends
180
+ * ORDER BY + LIMIT/OFFSET, the latter wraps it in a COUNT(*).
181
+ */
182
+ private _buildVectorSearchBase;
183
+ /** Builds vec0 shadow table name from index name: `<table>__vec__<indexName>`. */
184
+ private _vecTableName;
185
+ /**
186
+ * Creates/drops vec0 virtual shadow tables and sync triggers to match desired vector fields.
187
+ */
188
+ private _syncVecIndexes;
189
+ /** Creates a vec0 virtual shadow table with sync triggers and seeds it from existing rows. */
190
+ private _createVecTable;
191
+ /** Drops a vec0 virtual table and its sync triggers. */
192
+ private _dropVecTable;
193
+ /** Drops all vec0 virtual shadow tables and triggers for a content table. */
194
+ private _dropAllVecTables;
146
195
  }
147
196
  //#endregion
148
197
  //#region src/better-sqlite3-driver.d.ts
198
+ interface TBetterSqlite3DriverOptions extends Record<string, unknown> {
199
+ /** Load the optional `sqlite-vec` extension. */
200
+ vector?: boolean;
201
+ /** Absolute paths to SQLite loadable extensions, passed to `Database.loadExtension`. */
202
+ loadExtensions?: string[];
203
+ }
149
204
  /**
150
205
  * {@link TSqliteDriver} implementation backed by `better-sqlite3`.
151
206
  *
@@ -161,6 +216,9 @@ declare class SqliteAdapter extends BaseDbAdapter {
161
216
  * // File-based database
162
217
  * const driver = new BetterSqlite3Driver('./my-data.db')
163
218
  *
219
+ * // With sqlite-vec extension loaded
220
+ * const driver = new BetterSqlite3Driver('./my-data.db', { vector: true })
221
+ *
164
222
  * // Pre-created instance
165
223
  * import Database from 'better-sqlite3'
166
224
  * const db = new Database(':memory:', { verbose: console.log })
@@ -171,10 +229,16 @@ declare class SqliteAdapter extends BaseDbAdapter {
171
229
  * ```bash
172
230
  * pnpm add better-sqlite3
173
231
  * ```
232
+ *
233
+ * Vector search support requires the optional `sqlite-vec` package:
234
+ * ```bash
235
+ * pnpm add sqlite-vec
236
+ * ```
174
237
  */
175
238
  declare class BetterSqlite3Driver implements TSqliteDriver {
176
239
  private db;
177
- constructor(pathOrDb: string | better_sqlite30.Database, options?: Record<string, unknown>);
240
+ readonly hasVectorExt: boolean;
241
+ constructor(pathOrDb: string | _$better_sqlite30.Database, options?: TBetterSqlite3DriverOptions);
178
242
  run(sql: string, params?: unknown[]): TSqliteRunResult;
179
243
  all<T = Record<string, unknown>>(sql: string, params?: unknown[]): T[];
180
244
  get<T = Record<string, unknown>>(sql: string, params?: unknown[]): T | null;
package/dist/index.mjs CHANGED
@@ -17,6 +17,9 @@ import { buildAggregateCount, buildAggregateSelect, buildCreateView, buildDelete
17
17
  * // File-based database
18
18
  * const driver = new BetterSqlite3Driver('./my-data.db')
19
19
  *
20
+ * // With sqlite-vec extension loaded
21
+ * const driver = new BetterSqlite3Driver('./my-data.db', { vector: true })
22
+ *
20
23
  * // Pre-created instance
21
24
  * import Database from 'better-sqlite3'
22
25
  * const db = new Database(':memory:', { verbose: console.log })
@@ -27,12 +30,27 @@ import { buildAggregateCount, buildAggregateSelect, buildCreateView, buildDelete
27
30
  * ```bash
28
31
  * pnpm add better-sqlite3
29
32
  * ```
33
+ *
34
+ * Vector search support requires the optional `sqlite-vec` package:
35
+ * ```bash
36
+ * pnpm add sqlite-vec
37
+ * ```
30
38
  */
31
39
  var BetterSqlite3Driver = class {
32
40
  db;
41
+ hasVectorExt = false;
33
42
  constructor(pathOrDb, options) {
34
- if (typeof pathOrDb === "string") this.db = new (createRequire(import.meta.url)("better-sqlite3"))(pathOrDb, options);
35
- else this.db = pathOrDb;
43
+ const { vector, loadExtensions, ...nativeOptions } = options ?? {};
44
+ const req = createRequire(import.meta.url);
45
+ if (typeof pathOrDb === "string") {
46
+ const Database = req("better-sqlite3");
47
+ this.db = new Database(pathOrDb, nativeOptions);
48
+ } else this.db = pathOrDb;
49
+ for (const ext of loadExtensions ?? []) this.db.loadExtension(ext);
50
+ if (vector) {
51
+ req("sqlite-vec").load(this.db);
52
+ this.hasVectorExt = true;
53
+ }
36
54
  }
37
55
  run(sql, params) {
38
56
  const stmt = this.db.prepare(sql);
@@ -215,6 +233,27 @@ function buildAggregateCount$1(table, where, controls) {
215
233
  return buildAggregateCount(sqliteDialect, table, where, controls);
216
234
  }
217
235
  /**
236
+ * Converts an Atlas-style similarity score (1 = exact, 0 = orthogonal) to a
237
+ * vec0 distance threshold.
238
+ *
239
+ * - cosine / dotProduct (mapped to cosine in DDL): vec0 distance = 1 - cos_sim,
240
+ * normalized score = (1 + cos_sim) / 2, so distance = 2 * (1 - score).
241
+ * - euclidean (l2): vec0 distance is unbounded; the value is treated as a max
242
+ * distance directly (same convention as pgvector).
243
+ */
244
+ function thresholdToVecDistance(threshold, similarity) {
245
+ if (similarity === "euclidean") return threshold;
246
+ return 2 * (1 - threshold);
247
+ }
248
+ /**
249
+ * Maps an Atscript `@db.search.vector` similarity value to a sqlite-vec `vec0` metric.
250
+ * `dotProduct` falls back to `cosine` because vec0 has no dedicated dot-product metric.
251
+ */
252
+ function similarityToVecMetric(s) {
253
+ if (s === "euclidean") return "l2";
254
+ return "cosine";
255
+ }
256
+ /**
218
257
  * Maps Atscript design types to SQLite storage types.
219
258
  */
220
259
  function sqliteTypeFromDesignType(designType) {
@@ -231,12 +270,12 @@ function sqliteTypeFromDesignType(designType) {
231
270
  * Builds a CREATE TABLE IF NOT EXISTS statement from field descriptors.
232
271
  * Uses pre-computed {@link TDbFieldMeta} — no raw type introspection needed.
233
272
  */
234
- function buildCreateTable(table, fields, foreignKeys) {
273
+ function buildCreateTable(table, fields, foreignKeys, options) {
235
274
  const colDefs = [];
236
275
  const primaryKeys = fields.filter((f) => f.isPrimaryKey);
237
276
  for (const field of fields) {
238
277
  if (field.ignored) continue;
239
- const sqlType = field.isPrimaryKey && (field.designType === "number" || field.designType === "integer") ? "INTEGER" : sqliteTypeFromDesignType(field.designType);
278
+ const sqlType = options?.typeMapper?.(field) ?? (field.isPrimaryKey && (field.designType === "number" || field.designType === "integer") ? "INTEGER" : sqliteTypeFromDesignType(field.designType));
240
279
  let def = `"${esc(field.physicalName)}" ${sqlType}`;
241
280
  if (field.isPrimaryKey && primaryKeys.length === 1) {
242
281
  def += " PRIMARY KEY";
@@ -301,15 +340,82 @@ function buildPrefixedWhere(alias, filter) {
301
340
  * const users = new AtscriptDbTable(UsersType, adapter)
302
341
  * ```
303
342
  */
304
- var SqliteAdapter = class extends BaseDbAdapter {
343
+ var SqliteAdapter = class SqliteAdapter extends BaseDbAdapter {
305
344
  supportsNativeValueDefaults() {
306
345
  return true;
307
346
  }
347
+ /** Whether the SQLite connection has the sqlite-vec extension loaded. */
348
+ _supportsVector;
349
+ /** Vector fields: field path → { dimensions, similarity, indexName }. */
350
+ _vectorFields = /* @__PURE__ */ new Map();
351
+ /** Default similarity thresholds per vector index (from @db.search.vector.threshold). */
352
+ _vectorThresholds = /* @__PURE__ */ new Map();
353
+ /** Partition filter fields per vector index (from @db.search.filter). Field paths. */
354
+ _vectorPartitionFields = /* @__PURE__ */ new Map();
308
355
  constructor(driver) {
309
356
  super();
310
357
  this.driver = driver;
311
358
  this.driver.exec("PRAGMA foreign_keys = ON");
312
359
  }
360
+ onFieldScanned(field, _type, metadata) {
361
+ const vectorMeta = metadata.get("db.search.vector");
362
+ if (vectorMeta) {
363
+ const indexName = vectorMeta.indexName || field;
364
+ this._vectorFields.set(field, {
365
+ dimensions: vectorMeta.dimensions,
366
+ similarity: vectorMeta.similarity || "cosine",
367
+ indexName
368
+ });
369
+ const threshold = metadata.get("db.search.vector.threshold");
370
+ if (threshold !== void 0) this._vectorThresholds.set(indexName, threshold);
371
+ }
372
+ for (const indexName of metadata.get("db.search.filter") || []) {
373
+ const list = this._vectorPartitionFields.get(indexName);
374
+ if (list) list.push(field);
375
+ else this._vectorPartitionFields.set(indexName, [field]);
376
+ }
377
+ }
378
+ formatValue(field) {
379
+ if (!this._vectorFields.has(field.path)) return;
380
+ if (this._detectVectorSupport()) return {
381
+ toStorage: (value) => Array.isArray(value) ? Buffer.from(new Float32Array(value).buffer) : value,
382
+ fromStorage: (value) => {
383
+ if (value instanceof Buffer || value instanceof Uint8Array) {
384
+ const buf = Buffer.isBuffer(value) ? value : Buffer.from(value);
385
+ const arr = new Float32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4));
386
+ return Array.from(arr);
387
+ }
388
+ return value;
389
+ }
390
+ };
391
+ return {
392
+ toStorage: (value) => Array.isArray(value) ? JSON.stringify(value) : value,
393
+ fromStorage: (value) => {
394
+ if (typeof value === "string") try {
395
+ return JSON.parse(value);
396
+ } catch {
397
+ return value;
398
+ }
399
+ return value;
400
+ }
401
+ };
402
+ }
403
+ isVectorSearchable() {
404
+ if (this._vectorFields.size === 0) return false;
405
+ return this._detectVectorSupport();
406
+ }
407
+ _detectVectorSupport() {
408
+ if (this._supportsVector !== void 0) return this._supportsVector;
409
+ if (this.driver.hasVectorExt !== void 0) this._supportsVector = this.driver.hasVectorExt;
410
+ else try {
411
+ this.driver.get("SELECT vec_version()");
412
+ this._supportsVector = true;
413
+ } catch {
414
+ this._supportsVector = false;
415
+ }
416
+ if (!this._supportsVector && this._vectorFields.size > 0) this._log("[atscript-db-sqlite] sqlite-vec extension not available — vector fields will be stored as JSON TEXT (no similarity search).");
417
+ return this._supportsVector;
418
+ }
313
419
  async _beginTransaction() {
314
420
  this._log("BEGIN");
315
421
  this.driver.exec("BEGIN");
@@ -477,7 +583,7 @@ var SqliteAdapter = class extends BaseDbAdapter {
477
583
  }
478
584
  async ensureTable() {
479
585
  if (this._table instanceof AtscriptDbView) return this.ensureView();
480
- const sql = buildCreateTable(this.resolveTableName(), this._table.fieldDescriptors, this._table.foreignKeys);
586
+ const sql = buildCreateTable(this.resolveTableName(), this._table.fieldDescriptors, this._table.foreignKeys, { typeMapper: (field) => this.typeMapper(field) });
481
587
  this._log(sql);
482
588
  this.driver.exec(sql);
483
589
  this._seedIncrementStart();
@@ -538,10 +644,11 @@ var SqliteAdapter = class extends BaseDbAdapter {
538
644
  const tableName = this.resolveTableName();
539
645
  const tempName = `${tableName}__tmp_${Date.now()}`;
540
646
  this._dropAllFtsTables(tableName);
647
+ this._dropAllVecTables(tableName);
541
648
  this.driver.exec("PRAGMA foreign_keys = OFF");
542
649
  this.driver.exec("PRAGMA legacy_alter_table = ON");
543
650
  try {
544
- const createSql = buildCreateTable(tempName, this._table.fieldDescriptors, this._table.foreignKeys);
651
+ const createSql = buildCreateTable(tempName, this._table.fieldDescriptors, this._table.foreignKeys, { typeMapper: (field) => this.typeMapper(field) });
545
652
  this._log(createSql);
546
653
  this.driver.exec(createSql);
547
654
  const oldCols = (await this.getExistingColumns()).map((c) => c.name);
@@ -575,6 +682,7 @@ var SqliteAdapter = class extends BaseDbAdapter {
575
682
  async dropTable() {
576
683
  const tableName = this.resolveTableName();
577
684
  this._dropAllFtsTables(tableName);
685
+ this._dropAllVecTables(tableName);
578
686
  const ddl = `DROP TABLE IF EXISTS "${esc(tableName)}"`;
579
687
  this._log(ddl);
580
688
  this.driver.exec(ddl);
@@ -591,6 +699,7 @@ var SqliteAdapter = class extends BaseDbAdapter {
591
699
  }
592
700
  async dropTableByName(tableName) {
593
701
  this._dropAllFtsTables(tableName);
702
+ this._dropAllVecTables(tableName);
594
703
  const ddl = `DROP TABLE IF EXISTS "${esc(tableName)}"`;
595
704
  this._log(ddl);
596
705
  this.driver.exec(ddl);
@@ -607,6 +716,7 @@ var SqliteAdapter = class extends BaseDbAdapter {
607
716
  this.driver.exec(ddl);
608
717
  }
609
718
  typeMapper(field) {
719
+ if (this._vectorFields.has(field.path)) return this._detectVectorSupport() ? "BLOB" : "TEXT";
610
720
  if (field.isPrimaryKey && (field.designType === "number" || field.designType === "integer")) return "INTEGER";
611
721
  return sqliteTypeFromDesignType(field.designType);
612
722
  }
@@ -638,13 +748,21 @@ var SqliteAdapter = class extends BaseDbAdapter {
638
748
  shouldSkipType: (type) => type === "fulltext"
639
749
  });
640
750
  this._syncFtsIndexes(tableName);
751
+ this._syncVecIndexes(tableName);
641
752
  }
642
753
  getSearchIndexes() {
643
- return this._getFulltextIndexes().map((idx) => ({
754
+ const indexes = [];
755
+ for (const idx of this._getFulltextIndexes()) indexes.push({
644
756
  name: idx.name,
645
757
  description: `FTS5 index (${idx.fields.map((f) => f.name).join(", ")})`,
646
758
  type: "text"
647
- }));
759
+ });
760
+ for (const [field, vec] of this._vectorFields) indexes.push({
761
+ name: vec.indexName,
762
+ description: `vec0 index on ${field} (${vec.dimensions}, ${vec.similarity})`,
763
+ type: "vector"
764
+ });
765
+ return indexes;
648
766
  }
649
767
  async search(text, query, indexName) {
650
768
  if (!text.trim()) return [];
@@ -780,6 +898,225 @@ var SqliteAdapter = class extends BaseDbAdapter {
780
898
  const ftsTables = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__fts__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE"));
781
899
  for (const { name } of ftsTables) this._dropFtsTable(name);
782
900
  }
901
+ static _RESIDUAL_OVERFETCH = 4;
902
+ async vectorSearch(vector, query, indexName) {
903
+ if (!this._detectVectorSupport()) throw new Error("Vector search requires the sqlite-vec extension. Construct BetterSqlite3Driver with { vector: true }.");
904
+ const base = this._buildVectorSearchBase(vector, query, indexName);
905
+ return this._runVectorSearch(base);
906
+ }
907
+ async vectorSearchWithCount(vector, query, indexName) {
908
+ if (!this._detectVectorSupport()) throw new Error("Vector search requires the sqlite-vec extension. Construct BetterSqlite3Driver with { vector: true }.");
909
+ const base = this._buildVectorSearchBase(vector, query, indexName);
910
+ const data = this._runVectorSearch(base);
911
+ const countSql = `SELECT COUNT(*) AS cnt ${base.fromWhere}`;
912
+ this._log(countSql, base.params);
913
+ return {
914
+ data,
915
+ count: this.driver.get(countSql, base.params)?.cnt ?? 0
916
+ };
917
+ }
918
+ _runVectorSearch(base) {
919
+ let sql = `SELECT * ${base.fromWhere} ORDER BY _distance ASC LIMIT ?`;
920
+ const params = [...base.params, base.limit];
921
+ if (base.skip > 0) {
922
+ sql += ` OFFSET ?`;
923
+ params.push(base.skip);
924
+ }
925
+ this._log(sql, params);
926
+ return this.driver.all(sql, params);
927
+ }
928
+ /** Resolves a vector index (by name or first available) and its partition fields. */
929
+ _resolveVectorIndex(indexName) {
930
+ let entry;
931
+ if (indexName) {
932
+ for (const [f, v] of this._vectorFields) if (v.indexName === indexName) {
933
+ entry = [f, v];
934
+ break;
935
+ }
936
+ if (!entry) throw new Error(`Vector index "${indexName}" not found`);
937
+ } else {
938
+ const first = this._vectorFields.entries().next();
939
+ if (first.done) throw new Error("No vector fields defined");
940
+ entry = first.value;
941
+ }
942
+ const [field, vec] = entry;
943
+ const partitionPhysicalNames = /* @__PURE__ */ new Set();
944
+ for (const logicalPath of this._vectorPartitionFields.get(vec.indexName) ?? []) {
945
+ const descriptor = this._table.fieldDescriptors.find((f) => f.path === logicalPath);
946
+ partitionPhysicalNames.add(descriptor?.physicalName ?? logicalPath);
947
+ }
948
+ return {
949
+ field,
950
+ vec,
951
+ partitionPhysicalNames
952
+ };
953
+ }
954
+ /** Query-time `$threshold` overrides the schema-level threshold (mirrors postgres precedence). */
955
+ _resolveVectorThreshold(controls, indexName) {
956
+ const queryThreshold = controls.$threshold;
957
+ if (queryThreshold !== void 0) return queryThreshold;
958
+ return this._vectorThresholds.get(indexName);
959
+ }
960
+ /**
961
+ * Splits the (already physical-name) filter into partition equality push-down
962
+ * vs. residual filter. Only top-level primitive equality is pushed down.
963
+ */
964
+ _splitVectorFilter(filter, partitionPhysicalNames) {
965
+ const partition = [];
966
+ const residual = {};
967
+ if (!filter || typeof filter !== "object") return {
968
+ partition,
969
+ residual: filter
970
+ };
971
+ for (const [key, value] of Object.entries(filter)) if (partitionPhysicalNames.has(key) && value !== null && (typeof value === "string" || typeof value === "number" || typeof value === "boolean" || typeof value === "bigint")) partition.push({
972
+ name: key,
973
+ value
974
+ });
975
+ else residual[key] = value;
976
+ return {
977
+ partition,
978
+ residual
979
+ };
980
+ }
981
+ /**
982
+ * Builds the shared FROM+WHERE fragment for vec0 KNN queries (without ORDER/LIMIT).
983
+ * Both `vectorSearch` and `vectorSearchWithCount` reuse this — the former appends
984
+ * ORDER BY + LIMIT/OFFSET, the latter wraps it in a COUNT(*).
985
+ */
986
+ _buildVectorSearchBase(vector, query, indexName) {
987
+ const { vec, partitionPhysicalNames } = this._resolveVectorIndex(indexName);
988
+ if (vector.length !== vec.dimensions) throw new Error(`Vector dimension mismatch: index "${vec.indexName}" expects ${vec.dimensions}, got ${vector.length}`);
989
+ const vecBuf = Buffer.from(new Float32Array(vector).buffer);
990
+ const controls = query.controls || {};
991
+ const limit = controls.$limit ?? 20;
992
+ const skip = controls.$skip ?? 0;
993
+ const threshold = this._resolveVectorThreshold(controls, vec.indexName);
994
+ const { partition, residual } = this._splitVectorFilter(query.filter, partitionPhysicalNames);
995
+ const residualWhere = buildPrefixedWhere("_vs", residual);
996
+ const hasResidual = residualWhere.sql !== "1=1";
997
+ const hasThreshold = threshold !== void 0;
998
+ const k = (limit + skip) * (hasResidual || hasThreshold ? SqliteAdapter._RESIDUAL_OVERFETCH : 1);
999
+ const tableName = this.resolveTableName();
1000
+ const vecTable = this._vecTableName(vec.indexName);
1001
+ const innerWhereParts = ["v.embedding MATCH ?", "v.k = ?"];
1002
+ const params = [vecBuf, k];
1003
+ for (const p of partition) {
1004
+ innerWhereParts.push(`v."${esc(p.name)}" = ?`);
1005
+ params.push(p.value);
1006
+ }
1007
+ let fromWhere = `FROM (${`SELECT t.*, v.distance AS _distance FROM "${esc(vecTable)}" v JOIN "${esc(tableName)}" t ON t.rowid = v.rowid WHERE ${innerWhereParts.join(" AND ")}`}) _vs`;
1008
+ const outerWhereParts = [];
1009
+ if (hasThreshold) {
1010
+ outerWhereParts.push(`_distance <= ?`);
1011
+ params.push(thresholdToVecDistance(threshold, vec.similarity));
1012
+ }
1013
+ if (hasResidual) {
1014
+ outerWhereParts.push(`(${residualWhere.sql})`);
1015
+ params.push(...residualWhere.params);
1016
+ }
1017
+ if (outerWhereParts.length > 0) fromWhere += ` WHERE ${outerWhereParts.join(" AND ")}`;
1018
+ return {
1019
+ fromWhere,
1020
+ params,
1021
+ limit,
1022
+ skip
1023
+ };
1024
+ }
1025
+ /** Builds vec0 shadow table name from index name: `<table>__vec__<indexName>`. */
1026
+ _vecTableName(indexName) {
1027
+ return `${this.resolveTableName()}__vec__${indexName}`;
1028
+ }
1029
+ /**
1030
+ * Creates/drops vec0 virtual shadow tables and sync triggers to match desired vector fields.
1031
+ */
1032
+ _syncVecIndexes(tableName) {
1033
+ if (this._vectorFields.size === 0) return;
1034
+ if (!this._detectVectorSupport()) {
1035
+ this._log("[atscript-db-sqlite] sqlite-vec not available, skipping vec0 shadow table sync (vector fields stored as JSON)");
1036
+ return;
1037
+ }
1038
+ const desiredVecTables = /* @__PURE__ */ new Map();
1039
+ for (const [fieldPath, meta] of this._vectorFields.entries()) desiredVecTables.set(this._vecTableName(meta.indexName), {
1040
+ field: fieldPath,
1041
+ meta
1042
+ });
1043
+ const existingVec = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__vec__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE")).map((r) => r.name);
1044
+ for (const name of existingVec) if (!desiredVecTables.has(name)) this._dropVecTable(name);
1045
+ const existingSet = new Set(existingVec);
1046
+ for (const [vecTable, { field, meta }] of desiredVecTables.entries()) if (!existingSet.has(vecTable)) this._createVecTable(tableName, vecTable, field, meta);
1047
+ }
1048
+ /** Creates a vec0 virtual shadow table with sync triggers and seeds it from existing rows. */
1049
+ _createVecTable(tableName, vecTable, field, vec) {
1050
+ const metric = similarityToVecMetric(vec.similarity);
1051
+ const embeddingCol = this._table.fieldDescriptors.find((f) => f.path === field)?.physicalName ?? field;
1052
+ const partitionFieldPaths = this._vectorPartitionFields.get(vec.indexName) ?? [];
1053
+ const partitionCols = [];
1054
+ for (const logicalPath of partitionFieldPaths) {
1055
+ const descriptor = this._table.fieldDescriptors.find((f) => f.path === logicalPath);
1056
+ if (!descriptor) {
1057
+ this._log(`[atscript-db-sqlite] vec0 partition field "${logicalPath}" not found for index "${vec.indexName}", defaulting to TEXT`);
1058
+ partitionCols.push({
1059
+ physicalName: logicalPath,
1060
+ sqlType: "TEXT"
1061
+ });
1062
+ } else partitionCols.push({
1063
+ physicalName: descriptor.physicalName,
1064
+ sqlType: this.typeMapper(descriptor)
1065
+ });
1066
+ }
1067
+ for (const c of partitionCols) if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(c.physicalName)) throw new Error(`vec0 partition column name "${c.physicalName}" is not a safe identifier (only [A-Za-z_][A-Za-z0-9_]* is allowed for @db.search.filter fields on SQLite).`);
1068
+ const ddlCols = [partitionCols.map((c) => `${c.physicalName} ${c.sqlType} partition key`).join(", "), `embedding float[${vec.dimensions}] distance_metric=${metric}`].filter(Boolean).join(", ");
1069
+ const createSql = `CREATE VIRTUAL TABLE IF NOT EXISTS "${esc(vecTable)}" USING vec0(${ddlCols})`;
1070
+ this._log(createSql);
1071
+ this.driver.exec(createSql);
1072
+ const partCols = partitionCols.map((c) => `"${esc(c.physicalName)}"`);
1073
+ const embCol = `"${esc(embeddingCol)}"`;
1074
+ const insertCols = [
1075
+ "rowid",
1076
+ ...partCols,
1077
+ "embedding"
1078
+ ].join(", ");
1079
+ const insertNewVals = [
1080
+ "new.rowid",
1081
+ ...partitionCols.map((c) => `new."${esc(c.physicalName)}"`),
1082
+ `new.${embCol}`
1083
+ ].join(", ");
1084
+ const seedColList = [
1085
+ "rowid",
1086
+ ...partCols,
1087
+ embCol
1088
+ ].join(", ");
1089
+ const ev = esc(vecTable);
1090
+ const et = esc(tableName);
1091
+ const aiSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__ai")}" AFTER INSERT ON "${et}" WHEN new.${embCol} IS NOT NULL BEGIN INSERT INTO "${ev}"(${insertCols}) VALUES(${insertNewVals}); END`;
1092
+ this._log(aiSql);
1093
+ this.driver.exec(aiSql);
1094
+ const adSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__ad")}" AFTER DELETE ON "${et}" BEGIN DELETE FROM "${ev}" WHERE rowid = old.rowid; END`;
1095
+ this._log(adSql);
1096
+ this.driver.exec(adSql);
1097
+ const auSql = `CREATE TRIGGER IF NOT EXISTS "${esc(vecTable + "__au")}" AFTER UPDATE ON "${et}" BEGIN DELETE FROM "${ev}" WHERE rowid = old.rowid; INSERT INTO "${ev}"(${insertCols}) SELECT ${insertNewVals} WHERE new.${embCol} IS NOT NULL; END`;
1098
+ this._log(auSql);
1099
+ this.driver.exec(auSql);
1100
+ const seedSql = `INSERT INTO "${ev}"(${insertCols}) SELECT ${seedColList} FROM "${et}" WHERE ${embCol} IS NOT NULL`;
1101
+ this._log(seedSql);
1102
+ this.driver.exec(seedSql);
1103
+ }
1104
+ /** Drops a vec0 virtual table and its sync triggers. */
1105
+ _dropVecTable(vecTable) {
1106
+ for (const suffix of [
1107
+ "__ai",
1108
+ "__ad",
1109
+ "__au"
1110
+ ]) this.driver.exec(`DROP TRIGGER IF EXISTS "${esc(vecTable + suffix)}"`);
1111
+ const sql = `DROP TABLE IF EXISTS "${esc(vecTable)}"`;
1112
+ this._log(sql);
1113
+ this.driver.exec(sql);
1114
+ }
1115
+ /** Drops all vec0 virtual shadow tables and triggers for a content table. */
1116
+ _dropAllVecTables(tableName) {
1117
+ const vecTables = this.driver.all(`SELECT name, sql FROM sqlite_master WHERE type='table' AND name LIKE ?`, [`${tableName}__vec__%`]).filter((r) => r.sql.startsWith("CREATE VIRTUAL TABLE"));
1118
+ for (const { name } of vecTables) this._dropVecTable(name);
1119
+ }
783
1120
  };
784
1121
  /** Normalizes SQLite PRAGMA dflt_value to match serialized format.
785
1122
  * PRAGMA returns `'active'` (SQL-quoted), we store `active` (raw). */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@atscript/db-sqlite",
3
- "version": "0.1.72",
3
+ "version": "0.1.74",
4
4
  "description": "SQLite adapter for @atscript/db with swappable driver support.",
5
5
  "keywords": [
6
6
  "atscript",
@@ -41,6 +41,7 @@
41
41
  "@types/better-sqlite3": "^7.6.13",
42
42
  "@uniqu/core": "^0.1.6",
43
43
  "better-sqlite3": "^12.6.2",
44
+ "sqlite-vec": "^0.1.9",
44
45
  "unplugin-atscript": "^0.1.53"
45
46
  },
46
47
  "peerDependencies": {
@@ -48,14 +49,17 @@
48
49
  "@atscript/typescript": "^0.1.53",
49
50
  "@uniqu/core": "^0.1.6",
50
51
  "better-sqlite3": ">=11.0.0",
51
- "@atscript/db": "^0.1.72",
52
- "@atscript/db-sql-tools": "^0.1.72"
52
+ "@atscript/db-sql-tools": "^0.1.74",
53
+ "@atscript/db": "^0.1.74"
53
54
  },
54
55
  "peerDependenciesMeta": {
55
56
  "better-sqlite3": {
56
57
  "optional": true
57
58
  }
58
59
  },
60
+ "optionalDependencies": {
61
+ "sqlite-vec": "^0.1.9"
62
+ },
59
63
  "scripts": {
60
64
  "postinstall": "asc -f dts",
61
65
  "build": "vp pack",