@an5/adapters 0.2.11 → 0.2.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +133 -2
  2. package/dist/an5Adapter.d.ts +37 -0
  3. package/dist/an5Adapter.js +138 -35
  4. package/dist/base/types.d.ts +14 -0
  5. package/dist/index.d.ts +2 -0
  6. package/dist/index.js +8 -1
  7. package/dist/sqlite/browserEngine.d.ts +13 -0
  8. package/dist/sqlite/browserEngine.js +13 -0
  9. package/dist/sqlite/engine.d.ts +5 -0
  10. package/dist/sqlite/engine.js +10 -0
  11. package/dist/sqlite/index.d.ts +1 -0
  12. package/dist/sqlite/index.js +1 -0
  13. package/dist/sqlite/vector.d.ts +155 -0
  14. package/dist/sqlite/vector.js +436 -0
  15. package/dotnet/An5Adapter.cs +85 -27
  16. package/dotnet/Base/Core.cs +14 -0
  17. package/dotnet/Mssql/MssqlEngine.cs +44 -1
  18. package/dotnet/Postgres/PostgresEngine.cs +2 -0
  19. package/dotnet/Sqlite/SqliteEngine.cs +82 -2
  20. package/dotnet/Sqlite/SqliteVectors.cs +401 -0
  21. package/golang/an5_adapter.go +84 -32
  22. package/golang/base/vectors.go +263 -0
  23. package/golang/sqlite/vector.go +248 -0
  24. package/golang/sqlite_vector_test.go +288 -0
  25. package/golang/table_client.go +67 -3
  26. package/java/pom.xml +177 -0
  27. package/java/src/main/java/an5/adapters/An5Adapter.java +374 -0
  28. package/java/src/main/java/an5/adapters/An5Aggregate.java +116 -0
  29. package/java/src/main/java/an5/adapters/An5GroupBy.java +97 -0
  30. package/java/src/main/java/an5/adapters/An5Query.java +100 -0
  31. package/java/src/main/java/an5/adapters/An5TableClient.java +1031 -0
  32. package/java/src/main/java/an5/adapters/An5ViewClient.java +107 -0
  33. package/java/src/main/java/an5/adapters/base/Dialect.java +103 -0
  34. package/java/src/main/java/an5/adapters/base/FieldMeta.java +42 -0
  35. package/java/src/main/java/an5/adapters/base/JdbcUrls.java +201 -0
  36. package/java/src/main/java/an5/adapters/base/Metadata.java +195 -0
  37. package/java/src/main/java/an5/adapters/base/RelationDef.java +26 -0
  38. package/java/src/main/java/an5/adapters/base/SqlBuilder.java +422 -0
  39. package/java/src/main/java/an5/adapters/base/SqliteVectors.java +375 -0
  40. package/java/src/main/java/an5/adapters/base/Vectors.java +123 -0
  41. package/java/src/test/java/An5AdaptersSmoke.java +733 -0
  42. package/java/src/test/java/Json.java +174 -0
  43. package/java/src/test/java/QuerySemantics.java +111 -0
  44. package/kotlin/build.gradle.kts +145 -0
  45. package/kotlin/settings.gradle.kts +10 -0
  46. package/kotlin/src/main/kotlin/an5/adapters/Aggregate.kt +89 -0
  47. package/kotlin/src/main/kotlin/an5/adapters/An5.kt +91 -0
  48. package/kotlin/src/main/kotlin/an5/adapters/Query.kt +130 -0
  49. package/kotlin/src/main/kotlin/an5/adapters/Rows.kt +166 -0
  50. package/kotlin/src/main/kotlin/an5/adapters/TableClient.kt +169 -0
  51. package/kotlin/src/main/kotlin/an5/adapters/Types.kt +77 -0
  52. package/kotlin/src/test/kotlin/an5/adapters/An5Smoke.kt +329 -0
  53. package/native/sqlite/CMakeLists.txt +12 -0
  54. package/native/sqlite/README.md +117 -0
  55. package/native/sqlite/an5_vector.c +209 -0
  56. package/package.json +33 -7
  57. package/python/an5_adapter.py +37 -6
  58. package/python/base/__init__.py +15 -0
  59. package/python/base/vectors.py +434 -0
  60. package/python/sqlite/__init__.py +2 -2
  61. package/python/sqlite/provider.py +21 -1
  62. package/python/table_client.py +129 -59
  63. package/rust/src/adapter.rs +342 -24
  64. package/rust/src/base/mod.rs +7 -0
  65. package/rust/src/base/vectors.rs +293 -0
  66. package/rust/src/lib.rs +3 -3
  67. package/scripts/sqlite-native-build.js +32 -0
  68. package/swift/Package.swift +22 -0
  69. package/swift/Sources/An5Adapters/An5Adapter.swift +160 -0
  70. package/swift/Sources/An5Adapters/Metadata.swift +183 -0
  71. package/swift/Sources/An5Adapters/Query.swift +105 -0
  72. package/swift/Sources/An5Adapters/Row+Accessors.swift +129 -0
  73. package/swift/Sources/An5Adapters/SQLBuilder.swift +359 -0
  74. package/swift/Sources/An5Adapters/SQLiteDriver.swift +301 -0
  75. package/swift/Sources/An5Adapters/SqliteVectors.swift +243 -0
  76. package/swift/Sources/An5Adapters/TableClient.swift +671 -0
  77. package/swift/Sources/An5Adapters/Types.swift +127 -0
  78. package/swift/Sources/An5Adapters/Vectors.swift +86 -0
  79. package/swift/Sources/An5Adapters/ViewClient.swift +50 -0
  80. package/swift/Sources/CSQLite/module.modulemap +8 -0
  81. package/swift/Sources/CSQLite/shim.h +9 -0
  82. package/swift/Tests/An5AdaptersTests/QuerySemanticsTests.swift +51 -0
  83. package/swift/Tests/An5AdaptersTests/RuntimeSmokeTests.swift +488 -0
  84. package/typescript/src/an5Adapter.ts +146 -29
  85. package/typescript/src/base/types.ts +15 -0
  86. package/typescript/src/index.ts +13 -0
  87. package/typescript/src/sqlite/browserEngine.ts +24 -0
  88. package/typescript/src/sqlite/engine.ts +13 -0
  89. package/typescript/src/sqlite/index.ts +1 -1
  90. package/typescript/src/sqlite/vector.ts +541 -0
  91. package/golang/sqlite/sqlite.go +0 -14
package/README.md CHANGED
@@ -13,7 +13,7 @@ Standalone runtime database adapter and query engine for AN5 ORM. Provides conne
13
13
  - **Field Math Operators** — Atomic updates with `increment`, `decrement`, `multiply`, `divide`, `set`
14
14
  - **Vector Search** — Similarity search using NBase, pgvector (PostgreSQL), `VECTOR_DISTANCE` (MSSQL), or in-memory cosine/euclidean/dot similarity
15
15
  - **Real Transactions** — Interactive and callback transactions (`$transaction(async tx => ...)` and `$begin()/$commit()/$rollback()`)
16
- - **Cross-Language** — Unified API in TypeScript, Python, .NET (C#), Golang, and Rust
16
+ - **Cross-Language** — Unified API in TypeScript, Python, .NET (C#), Golang, Rust, Java, Kotlin, and Swift
17
17
  - **Google Sheets Database** — Use spreadsheets as a live database with full CRUD and SQL syntax support
18
18
 
19
19
  ---
@@ -52,6 +52,78 @@ cargo add an5-adapters
52
52
 
53
53
  The crate is driver-agnostic — it depends on `sqlx` with `any`, so the app enables the driver it needs. See [crates.io](https://crates.io/crates/an5-adapters) and [docs.rs](https://docs.rs/an5-adapters).
54
54
 
55
+ ### Java
56
+
57
+ ```bash
58
+ # Included in the npm package under java/, which ships its own pom.xml
59
+ ```
60
+
61
+ `java/` is a Maven module with **no dependencies**: the adapter speaks JDBC through
62
+ `java.sql`, so the app picks the driver it already has — `org.xerial:sqlite-jdbc`,
63
+ `org.postgresql:postgresql` or `com.microsoft.sqlserver:mssql-jdbc`. That is what lets one
64
+ runtime serve all three engines and what keeps it usable on Android, where an adapter that
65
+ dragged in three drivers would not fit.
66
+
67
+ ```xml
68
+ <dependency>
69
+ <groupId>io.github.an5orm</groupId>
70
+ <artifactId>an5-adapters-java</artifactId>
71
+ <version>0.2.13</version>
72
+ </dependency>
73
+ ```
74
+
75
+ From a checkout the same module is built and installed into `~/.m2`:
76
+
77
+ ```bash
78
+ mvn -f java/pom.xml install
79
+ mvn -f java/pom.xml test
80
+ ```
81
+
82
+ ### Kotlin
83
+
84
+ ```bash
85
+ # Included in the npm package under kotlin/, which ships build.gradle.kts
86
+ ```
87
+
88
+ Gradle rather than Maven, because Kotlin on Android is built with Gradle and a runtime a
89
+ phone app cannot resolve is not a mobile runtime. The Kotlin runtime depends on the Java one
90
+ rather than reimplementing it, so the dialect rules and the where builder exist once:
91
+
92
+ ```kotlin
93
+ dependencies {
94
+ implementation("io.github.an5orm:an5-adapters-kotlin:0.2.13")
95
+ }
96
+ ```
97
+
98
+ From a checkout the Kotlin module resolves its Java sibling from `~/.m2` — declared as
99
+ `mavenLocal()` in `build.gradle.kts`, so the checkout builds without publishing anything
100
+ first:
101
+
102
+ ```bash
103
+ mvn -f java/pom.xml install
104
+ gradle -p kotlin build
105
+ ```
106
+
107
+ Both JVM modules publish to Maven Central, and each release ships both
108
+ `io.github.an5orm:an5-adapters-java:<version>` and
109
+ `io.github.an5orm:an5-adapters-kotlin:<version>`: a `v*` tag runs the
110
+ `publish-maven` job in `.github/workflows/publish.yml` (`mvn -Prelease deploy` and
111
+ `gradle publish`), and a manual run is available through the workflow's `publish-maven`
112
+ input. It needs the repository secrets `MAVEN_CENTRAL_USERNAME` and `MAVEN_CENTRAL_TOKEN`
113
+ (a Central Portal token), `MAVEN_GPG_PRIVATE_KEY` and `MAVEN_GPG_PASSPHRASE`.
114
+
115
+ ### Swift
116
+
117
+ ```swift
118
+ .package(url: "https://github.com/an5ORM/an5Adapters.git", from: "0.2.13")
119
+ ```
120
+
121
+ A SwiftPM package that links the **system** SQLite — the database every Apple platform
122
+ already ships, so there is no bundled engine and the same file works on device and in the
123
+ simulator. Building it needs SQLite's headers (`apt-get install libsqlite3-dev` on Linux).
124
+ The runtime takes a `SQLDriver`, so an app that already has a database layer can put GRDB or
125
+ SQLite.swift underneath instead.
126
+
55
127
  ---
56
128
 
57
129
  ## NBase — Neural Vector Database
@@ -139,6 +211,65 @@ If NBase is unreachable, `vectorSearch` logs a warning and falls back to the
139
211
  database's native vector support, then to in-memory similarity, so an outage
140
212
  degrades performance rather than breaking queries.
141
213
 
214
+ ---
215
+
216
+ ## SQLite — vector search without a vector database
217
+
218
+ For host-language-free distance calculation, build the shared
219
+ [AN5 C extension](native/sqlite/README.md) with
220
+ `npm run build:sqlite:native` and pass the printed library path as `sqliteVec`.
221
+ It runs cosine, Euclidean and dot-product distances natively, with SSE2 on
222
+ x86-64 and query-vector caching. The existing fallback remains available when
223
+ an extension is not loaded.
224
+
225
+ SQLite has no vector type, so a `VECTOR(n)` column stores a **BLOB of
226
+ little-endian float32** — 4 bytes per dimension. Write a plain array of numbers
227
+ and the adapter encodes it; read it back and you get the same array. A column
228
+ that already holds JSON text (`'[0.1, 0.2]'`) keeps working, so nothing has to
229
+ be migrated.
230
+
231
+ ```ts
232
+ await db.table('Article').create({ data: { title: '…', embedding: queryEmbedding } });
233
+
234
+ const row = await db.table('Article').findFirst({ where: { title: '…' } });
235
+ row.embedding; // number[]
236
+ ```
237
+
238
+ The search is ranked **inside the database**, trying four strategies in order:
239
+
240
+ | Order | Strategy | Needs | Reaches |
241
+ |-------|----------|-------|---------|
242
+ | 1 | `sqlite-vec` | The extension loaded | Scalar distances over ordinary tables |
243
+ | 2 | `udf` | A driver that can register a function | `BLOB` and legacy text columns |
244
+ | 3 | `sql` | JSON1 (built in since SQLite 3.38) | Legacy text columns |
245
+ | 4 | `memory` | Nothing | Any column, whole table loaded |
246
+
247
+ Strategies 1-3 transfer only the rows that match, which is the reason to prefer
248
+ them. A row whose stored vector cannot be scored is left out rather than returned
249
+ with a null distance.
250
+
251
+ [sqlite-vec](https://github.com/asg017/sqlite-vec) supplies native vector distance
252
+ functions for exact search. The adapter scans an ordinary table with those
253
+ functions; it does not create or synchronize a `vec0` virtual table:
254
+
255
+ ```ts
256
+ const db = createAn5Adapter({
257
+ connectionString: 'sqlite:///app.db',
258
+ sqliteVec: require('sqlite-vec').getLoadablePath(),
259
+ });
260
+ ```
261
+
262
+ `vectorStrategy` pins one strategy instead of probing — `'auto'` (the default),
263
+ `'sqlite-vec'`, `'udf'`, `'sql'` or `'memory'`. Every other runtime implements
264
+ the same four strategies with the same order; `sqlx` (Rust) and JDBC (Java,
265
+ Kotlin) use `sql` for JSON text and `memory` for BLOB columns without
266
+ driver-specific vector functions.
267
+
268
+ ```python
269
+ from an5_adapter import create_an5_adapter
270
+ db = create_an5_adapter("sqlite:///app.db", sqlite_vec="vec0", vector_strategy="auto")
271
+ ```
272
+
142
273
 
143
274
  ## Usage
144
275
 
@@ -274,7 +405,7 @@ db.transaction(perform_transfer)
274
405
  | `count(args)` | Count matching records |
275
406
  | `aggregate(args)` | Compute `_count`, `_sum`, `_avg`, `_min`, `_max` |
276
407
  | `groupBy(args)` | Group by fields with aggregations and pagination |
277
- | `vectorSearch(args)` | Semantic vector similarity search |
408
+ | `vectorSearch(args)` | Semantic vector similarity search; SQLite ranks it in the database (sqlite-vec, `an5_vec_*`, `json_each`), everything else uses the engine's native support or scores in memory |
278
409
 
279
410
  Generated TypeScript field metadata marks schema primary keys with `isId: true`.
280
411
  SQL and Google Sheets `create` use this marker to identify custom primary-key
@@ -1,6 +1,7 @@
1
1
  import { An5SheetsAdapter, An5SheetsAdapterConfig, SheetsTableClient } from './googlesheets';
2
2
  import type { An5AdapterConfig, Dialect, TransactionHandle } from './base/types';
3
3
  import { type NBaseAdapterConfig, type NBaseVectorClient } from './nbase';
4
+ import { type SqliteVectorSupport, type VectorStrategyPreference } from './sqlite/vector';
4
5
  export type { An5AdapterConfig, Dialect, QueryEngine, TransactionHandle } from './base/types';
5
6
  export { setAdapterMetadata, createAdapterProxy, type AdapterMetadata } from './base/metadata';
6
7
  export type { TypedAn5Adapter, AdapterAPI } from './typed';
@@ -36,6 +37,14 @@ export declare class An5Adapter {
36
37
  private requireEngine;
37
38
  private requireSheetsAdapter;
38
39
  exec<T = any>(query: string, params?: Record<string, any>): Promise<T[]>;
40
+ /**
41
+ * The SQLite engine's vector hooks, or null when this is not a SQLite adapter.
42
+ * `vectorSearch` needs them to load sqlite-vec and register the distance
43
+ * functions; a connection with no hooks still searches in memory.
44
+ */
45
+ sqliteVectorSupport(): Promise<SqliteVectorSupport | null>;
46
+ /** `vectorStrategy` from the config, so a project can pin one strategy. */
47
+ get vectorStrategy(): VectorStrategyPreference | undefined;
39
48
  /** INTERNAL: used by AdapterTableClient for DML statements needing row count */
40
49
  _executeRaw(query: string, params?: Record<string, any>): Promise<number>;
41
50
  /** Execute a raw query with positional values → @p_0, @p_1, ... per dialect */
@@ -141,11 +150,30 @@ export declare class AdapterTableClient<T = any> {
141
150
  constructor(adapter: An5Adapter | An5AdapterTx, modelName: string);
142
151
  get dialect(): Dialect;
143
152
  get tableName(): string;
153
+ /** The table name this model maps to, before any quoting is applied. */
154
+ private get mappedTableName();
155
+ /**
156
+ * The bare table name, for helpers that take a string literal. `@@map` values
157
+ * may be pre-quoted or schema-qualified, neither of which `pragma_table_info`
158
+ * accepts, so the last segment is unwrapped here.
159
+ */
160
+ get rawTableName(): string;
144
161
  private get nolock();
145
162
  /** Build the shared context used by parseWhere to resolve relations and self references. */
146
163
  private whereContext;
147
164
  private doExec;
148
165
  private doExecuteRaw;
166
+ /**
167
+ * Encode every `VECTOR(n)` value on its way into SQLite.
168
+ *
169
+ * The column holds a float32 BLOB, which is a third of the size of the JSON
170
+ * text it replaces and is what sqlite-vec reads. Values that already arrived
171
+ * as text or bytes are left alone, so rows written by an older version stay
172
+ * readable.
173
+ */
174
+ private encodeVectorFields;
175
+ /** Decode `VECTOR(n)` columns back into `number[]` after a read. */
176
+ private decodeVectorFields;
149
177
  findMany(args?: {
150
178
  where?: any;
151
179
  orderBy?: any;
@@ -246,6 +274,15 @@ export declare class AdapterTableClient<T = any> {
246
274
  }): Promise<(T & {
247
275
  distance: number;
248
276
  })[]>;
277
+ /**
278
+ * Rank the vector column in the client. This is the last resort for every
279
+ * provider, and the only option on SQLite connections that offer neither
280
+ * sqlite-vec, a driver function nor JSON1.
281
+ *
282
+ * The whole table is loaded, so it is only reasonable for a table small
283
+ * enough to fit in memory.
284
+ */
285
+ private scoreVectorsInMemory;
249
286
  }
250
287
  export declare function createAn5Adapter(config: AnyAdapterConfig): AnyAdapter;
251
288
  export declare function createAn5Adapter<TModels extends object>(config: AnyAdapterConfig): import('./typed').TypedAn5Adapter<TModels>;
@@ -17,6 +17,7 @@ const parseConnectionString_1 = require("./googlesheets/parseConnectionString");
17
17
  const SQLITE_FILE_SUFFIXES = ['.sqlite', '.sqlite3', '.db'];
18
18
  const nbase_1 = require("./nbase");
19
19
  const sql_1 = require("./base/sql");
20
+ const vector_1 = require("./sqlite/vector");
20
21
  const metadata_1 = require("./base/metadata");
21
22
  var metadata_2 = require("./base/metadata");
22
23
  Object.defineProperty(exports, "setAdapterMetadata", { enumerable: true, get: function () { return metadata_2.setAdapterMetadata; } });
@@ -50,7 +51,10 @@ function isDuplicateKeyError(error, dialect) {
50
51
  function appendUpdateSet(sets, params, col, val, dialect) {
51
52
  const quoted = (0, sql_1.quote)(col, dialect);
52
53
  const safeCol = sanitizeParamName(col);
53
- if (val && typeof val === 'object' && !(val instanceof Date)) {
54
+ // `TypedArray.prototype.set` is the copy method, not the update operator, so a
55
+ // vector written as a float32 BLOB must not be read as `{ set: ... }`.
56
+ const isLiteral = Array.isArray(val) || ArrayBuffer.isView(val) || val instanceof ArrayBuffer;
57
+ if (val && typeof val === 'object' && !(val instanceof Date) && !isLiteral) {
54
58
  if (val.increment !== undefined) {
55
59
  const p = (0, sql_1.bindParam)(params, `s_${safeCol}_inc`, val.increment);
56
60
  sets.push(`${quoted} = ${quoted} + @${p}`);
@@ -343,6 +347,21 @@ class An5Adapter {
343
347
  return this.sheetsAdapter.exec(query, params);
344
348
  return (await this.requireEngine()).exec(query, params);
345
349
  }
350
+ /**
351
+ * The SQLite engine's vector hooks, or null when this is not a SQLite adapter.
352
+ * `vectorSearch` needs them to load sqlite-vec and register the distance
353
+ * functions; a connection with no hooks still searches in memory.
354
+ */
355
+ async sqliteVectorSupport() {
356
+ if (this.sheetsAdapter)
357
+ return null;
358
+ const engine = await this.requireEngine();
359
+ return engine.dialect === 'sqlite' ? (engine.vectorSupport ?? null) : null;
360
+ }
361
+ /** `vectorStrategy` from the config, so a project can pin one strategy. */
362
+ get vectorStrategy() {
363
+ return this._engineConfig?.vectorStrategy;
364
+ }
346
365
  /** INTERNAL: used by AdapterTableClient for DML statements needing row count */
347
366
  async _executeRaw(query, params) {
348
367
  return (await this.requireEngine()).executeRaw(query, params);
@@ -655,6 +674,15 @@ class AdapterTableClient {
655
674
  }
656
675
  get dialect() { return this.adapter.dialect; }
657
676
  get tableName() {
677
+ const t = this.mappedTableName;
678
+ if (t.startsWith('[') || t.startsWith('"') || t.startsWith('`'))
679
+ return t;
680
+ if (t.includes('.'))
681
+ return t.split('.').map(p => (0, sql_1.quote)(p, this.dialect)).join('.');
682
+ return (0, sql_1.quote)(t, this.dialect);
683
+ }
684
+ /** The table name this model maps to, before any quoting is applied. */
685
+ get mappedTableName() {
658
686
  const name = this.modelName;
659
687
  let t = name;
660
688
  const modelToTable = (0, metadata_1.getModelToTable)();
@@ -670,11 +698,16 @@ class AdapterTableClient {
670
698
  t = modelToTable[lower];
671
699
  }
672
700
  }
673
- if (t.startsWith('[') || t.startsWith('"') || t.startsWith('`'))
674
- return t;
675
- if (t.includes('.'))
676
- return t.split('.').map(p => (0, sql_1.quote)(p, this.dialect)).join('.');
677
- return (0, sql_1.quote)(t, this.dialect);
701
+ return t;
702
+ }
703
+ /**
704
+ * The bare table name, for helpers that take a string literal. `@@map` values
705
+ * may be pre-quoted or schema-qualified, neither of which `pragma_table_info`
706
+ * accepts, so the last segment is unwrapped here.
707
+ */
708
+ get rawTableName() {
709
+ const last = this.mappedTableName.split('.').pop() ?? this.mappedTableName;
710
+ return last.replace(/^[\s]*[\[\]"`]+|[\[\]"`]+[\s]*$/g, '');
678
711
  }
679
712
  get nolock() {
680
713
  return this.dialect === 'mssql' ? ' WITH (NOLOCK)' : '';
@@ -693,6 +726,58 @@ class AdapterTableClient {
693
726
  async doExecuteRaw(query, params) {
694
727
  return this.adapter._executeRaw(query, params);
695
728
  }
729
+ /**
730
+ * Encode every `VECTOR(n)` value on its way into SQLite.
731
+ *
732
+ * The column holds a float32 BLOB, which is a third of the size of the JSON
733
+ * text it replaces and is what sqlite-vec reads. Values that already arrived
734
+ * as text or bytes are left alone, so rows written by an older version stay
735
+ * readable.
736
+ */
737
+ encodeVectorFields(data) {
738
+ if (this.dialect !== 'sqlite')
739
+ return data;
740
+ const fields = (0, metadata_1.getFieldsForModel)(this.modelName);
741
+ const vectorFields = Object.keys(fields).filter((name) => (0, vector_1.isVectorField)(fields[name]));
742
+ if (vectorFields.length === 0)
743
+ return data;
744
+ const { encodeVector } = require('./sqlite/vector');
745
+ const out = { ...data };
746
+ for (const name of vectorFields) {
747
+ const value = out[name];
748
+ if ((0, vector_1.needsVectorEncoding)(value)) {
749
+ const parsed = Array.from(value);
750
+ if (parsed.length > 0)
751
+ out[name] = encodeVector(parsed);
752
+ }
753
+ }
754
+ return out;
755
+ }
756
+ /** Decode `VECTOR(n)` columns back into `number[]` after a read. */
757
+ decodeVectorFields(rows) {
758
+ if (this.dialect !== 'sqlite' || !Array.isArray(rows) || rows.length === 0)
759
+ return rows;
760
+ const fields = (0, metadata_1.getFieldsForModel)(this.modelName);
761
+ const vectorFields = Object.keys(fields).filter((name) => (0, vector_1.isVectorField)(fields[name]));
762
+ if (vectorFields.length === 0)
763
+ return rows;
764
+ return rows.map((row) => {
765
+ if (!row || typeof row !== 'object')
766
+ return row;
767
+ let changed = false;
768
+ const out = { ...row };
769
+ for (const name of vectorFields) {
770
+ if (!(name in out))
771
+ continue;
772
+ const decoded = (0, vector_1.decodeVector)(out[name]);
773
+ if (decoded) {
774
+ out[name] = decoded;
775
+ changed = true;
776
+ }
777
+ }
778
+ return changed ? out : row;
779
+ });
780
+ }
696
781
  async findMany(args) {
697
782
  const params = {};
698
783
  const whereCtx = this.whereContext();
@@ -758,7 +843,7 @@ class AdapterTableClient {
758
843
  if (orderSql)
759
844
  query += ` ${orderSql}`;
760
845
  }
761
- const rows = await this.doExec(query, params);
846
+ const rows = this.decodeVectorFields(await this.doExec(query, params));
762
847
  if (args?.include) {
763
848
  await resolveIncludes(this.modelName, rows, args.include, this.adapter);
764
849
  }
@@ -800,7 +885,7 @@ class AdapterTableClient {
800
885
  }
801
886
  const fields = (0, metadata_1.getFieldsForModel)(this.modelName);
802
887
  const idFieldName = (0, metadata_1.resolveIdField)(fields);
803
- const data = { ...rawData };
888
+ const data = this.encodeVectorFields(rawData);
804
889
  if (idFieldName) {
805
890
  const fieldDef = fields[idFieldName];
806
891
  const rawType = typeof fieldDef === 'string' ? fieldDef : (fieldDef?.ts || fieldDef?.sql || fieldDef?.type || '');
@@ -870,9 +955,10 @@ class AdapterTableClient {
870
955
  const rowPlaceholders = [];
871
956
  for (let r = 0; r < args.data.length; r++) {
872
957
  const row = args.data[r];
958
+ const encoded = this.encodeVectorFields(row);
873
959
  const vals = firstCols.map(col => {
874
960
  const p = `r${r}_${col}`;
875
- params[p] = row[col] ?? null;
961
+ params[p] = encoded[col] ?? null;
876
962
  return `@${p}`;
877
963
  });
878
964
  rowPlaceholders.push(`(${vals.join(', ')})`);
@@ -905,13 +991,14 @@ class AdapterTableClient {
905
991
  delete rawData[key];
906
992
  }
907
993
  }
994
+ const data = this.encodeVectorFields(rawData);
908
995
  const setCols = Object.keys(rawData).filter(k => rawData[k] !== undefined);
909
996
  if (setCols.length > 0) {
910
997
  const params = {};
911
998
  const whereSql = (0, sql_1.parseWhere)(this.modelName, args.where, params, this.dialect, 'w_', this.whereContext());
912
999
  const sets = [];
913
1000
  for (const col of setCols) {
914
- appendUpdateSet(sets, params, col, rawData[col], this.dialect);
1001
+ appendUpdateSet(sets, params, col, data[col], this.dialect);
915
1002
  }
916
1003
  if (sets.length > 0) {
917
1004
  const query = `UPDATE ${this.tableName} SET ${sets.join(', ')}${whereSql ? ` WHERE ${whereSql}` : ''}`;
@@ -963,10 +1050,11 @@ class AdapterTableClient {
963
1050
  async updateMany(args) {
964
1051
  const params = {};
965
1052
  const whereSql = (0, sql_1.parseWhere)(this.modelName, args.where, params, this.dialect, 'w_', this.whereContext());
1053
+ const data = this.encodeVectorFields(args.data);
966
1054
  const setCols = Object.keys(args.data).filter(k => args.data[k] !== undefined);
967
1055
  const sets = [];
968
1056
  for (const col of setCols) {
969
- appendUpdateSet(sets, params, col, args.data[col], this.dialect);
1057
+ appendUpdateSet(sets, params, col, data[col], this.dialect);
970
1058
  }
971
1059
  if (sets.length === 0)
972
1060
  return { count: 0 };
@@ -1307,6 +1395,30 @@ class AdapterTableClient {
1307
1395
  throw err;
1308
1396
  }
1309
1397
  }
1398
+ else if (this.dialect === 'sqlite') {
1399
+ // SQLite has no vector type, so the ranking runs through sqlite-vec, a
1400
+ // driver function, or json_each — whichever this connection supports.
1401
+ const whereSql = (0, sql_1.parseWhere)(this.modelName, args.where, params, this.dialect, '', this.whereContext());
1402
+ const support = await this.adapter.sqliteVectorSupport?.() ?? null;
1403
+ const ranked = await (0, vector_1.runSqliteVectorSearch)({
1404
+ exec: (sql, p) => this.doExec(sql, p),
1405
+ vectorSupport: support,
1406
+ table: this.tableName,
1407
+ rawTable: this.rawTableName,
1408
+ column: col,
1409
+ rawColumn: vectorField,
1410
+ vector: args.vector,
1411
+ metric,
1412
+ take,
1413
+ tail: whereSql ? ` WHERE ${whereSql}` : '',
1414
+ params,
1415
+ preference: this.adapter.vectorStrategy,
1416
+ });
1417
+ if (ranked)
1418
+ return this.decodeVectorFields(ranked.rows);
1419
+ // No in-database strategy: score the column in the client instead.
1420
+ return this.scoreVectorsInMemory(args, metric, take, vectorField);
1421
+ }
1310
1422
  else if (this.dialect === 'mssql') {
1311
1423
  const distFn = metric === 'cosine' ? 'cosine' : metric === 'euclidean' ? 'euclidean' : 'dot';
1312
1424
  const whereSql = (0, sql_1.parseWhere)(this.modelName, args.where, params, this.dialect, '', this.whereContext());
@@ -1333,35 +1445,26 @@ class AdapterTableClient {
1333
1445
  }
1334
1446
  }
1335
1447
  // 2. Fallback: in-memory similarity computation when native vector support is unavailable
1448
+ return this.scoreVectorsInMemory(args, metric, take, vectorField);
1449
+ }
1450
+ /**
1451
+ * Rank the vector column in the client. This is the last resort for every
1452
+ * provider, and the only option on SQLite connections that offer neither
1453
+ * sqlite-vec, a driver function nor JSON1.
1454
+ *
1455
+ * The whole table is loaded, so it is only reasonable for a table small
1456
+ * enough to fit in memory.
1457
+ */
1458
+ async scoreVectorsInMemory(args, metric, take, vectorField = 'embedding') {
1336
1459
  const rows = await this.findMany({ where: args.where });
1337
1460
  const scored = [];
1338
1461
  for (const row of rows) {
1339
- const raw = row[vectorField];
1340
- if (!raw)
1462
+ const vec = (0, vector_1.decodeVector)(row[vectorField]);
1463
+ if (!vec)
1341
1464
  continue;
1342
- let vec = [];
1343
- try {
1344
- vec = typeof raw === 'string' ? JSON.parse(raw) : raw;
1345
- }
1346
- catch {
1465
+ const dist = (0, vector_1.vectorDistance)(args.vector, vec, metric);
1466
+ if (dist === null)
1347
1467
  continue;
1348
- }
1349
- if (!Array.isArray(vec) || vec.length !== args.vector.length)
1350
- continue;
1351
- let dot = 0, m1 = 0, m2 = 0;
1352
- for (let i = 0; i < args.vector.length; i++) {
1353
- const av = args.vector[i] ?? 0;
1354
- const bv = vec[i] ?? 0;
1355
- dot += av * bv;
1356
- m1 += av ** 2;
1357
- m2 += bv ** 2;
1358
- }
1359
- const cosine = m1 && m2 ? dot / (Math.sqrt(m1) * Math.sqrt(m2)) : 0;
1360
- const dist = metric === 'cosine'
1361
- ? 1 - cosine
1362
- : metric === 'dot'
1363
- ? -dot
1364
- : Math.sqrt(args.vector.reduce((s, v, i) => s + (v - (vec[i] ?? 0)) ** 2, 0));
1365
1468
  scored.push({ row, dist });
1366
1469
  }
1367
1470
  scored.sort((a, b) => a.dist - b.dist);
@@ -1,3 +1,4 @@
1
+ import type { SqliteVectorSupport, VectorStrategyPreference } from '../sqlite/vector';
1
2
  export type Dialect = 'mssql' | 'postgres' | 'mysql' | 'sqlite' | 'googlesheets';
2
3
  export interface An5AdapterConfig {
3
4
  connectionString?: string;
@@ -7,6 +8,17 @@ export interface An5AdapterConfig {
7
8
  * run in NBase while the rows stay in `connectionString`'s database.
8
9
  */
9
10
  nbase?: string;
11
+ /**
12
+ * Path to sqlite-vec or the AN5 native vector extension. On SQLite it is loaded before the
13
+ * vector capabilities are probed, so `vectorSearch` ranks inside the database
14
+ * instead of loading the table into memory. Other providers ignore it.
15
+ */
16
+ sqliteVec?: string;
17
+ /**
18
+ * Pin the SQLite vector search strategy. `auto` (the default) probes for
19
+ * sqlite-vec, a driver function and JSON1, and uses the first that works.
20
+ */
21
+ vectorStrategy?: VectorStrategyPreference;
10
22
  engine?: QueryEngine;
11
23
  db?: any;
12
24
  driver?: any;
@@ -27,4 +39,6 @@ export interface QueryEngine {
27
39
  connect(): Promise<void>;
28
40
  disconnect(): Promise<void>;
29
41
  beginTransaction(): Promise<TransactionHandle>;
42
+ /** SQLite only: extension loading and distance-function registration. */
43
+ vectorSupport?: SqliteVectorSupport;
30
44
  }
package/dist/index.d.ts CHANGED
@@ -3,6 +3,8 @@ export { parseWhere, buildOrderBy, quote } from './base/sql';
3
3
  export { NBaseVectorClient, NBaseError, createNBaseVectorClient, createNBaseClient, parseNBaseConnectionString, buildVectorId, parseVectorId, } from './nbase';
4
4
  export type { NBaseAdapterConfig, NBaseClientConfig, NBaseVector, NBaseSearchOptions, NBaseSearchResult, NBaseStats, } from './nbase';
5
5
  export type { An5AdapterConfig, AdapterMetadata, Dialect, } from './an5Adapter';
6
+ export { encodeVector, decodeVector, vectorDistance, isVectorField, AN5_VECTOR_FUNCTIONS, SQLITE_VEC_FUNCTIONS, } from './sqlite/vector';
7
+ export type { SqliteVectorStrategy, VectorMetric, VectorStrategyPreference, } from './sqlite/vector';
6
8
  export type { TypedAn5Adapter, AdapterAPI } from './typed';
7
9
  export { getLlmConfig, setLlmConfig, getEmbeddingConfig, setEmbeddingConfig, resetAdapter, } from './config';
8
10
  export type { LlmConfigData, EmbeddingConfigData, } from './config';
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.parseSheetsConnectionString = exports.createAn5SheetsAdapter = exports.SheetsTableClient = exports.An5SheetsAdapter = exports.resetAdapter = exports.setEmbeddingConfig = exports.getEmbeddingConfig = exports.setLlmConfig = exports.getLlmConfig = exports.parseVectorId = exports.buildVectorId = exports.parseNBaseConnectionString = exports.createNBaseClient = exports.createNBaseVectorClient = exports.NBaseError = exports.NBaseVectorClient = exports.quote = exports.buildOrderBy = exports.parseWhere = exports.setAdapterMetadata = exports.executorFromAdapter = exports.createAdapterProxy = exports.createAn5Adapter = exports.AdapterTableClient = exports.An5Adapter = void 0;
3
+ exports.parseSheetsConnectionString = exports.createAn5SheetsAdapter = exports.SheetsTableClient = exports.An5SheetsAdapter = exports.resetAdapter = exports.setEmbeddingConfig = exports.getEmbeddingConfig = exports.setLlmConfig = exports.getLlmConfig = exports.SQLITE_VEC_FUNCTIONS = exports.AN5_VECTOR_FUNCTIONS = exports.isVectorField = exports.vectorDistance = exports.decodeVector = exports.encodeVector = exports.parseVectorId = exports.buildVectorId = exports.parseNBaseConnectionString = exports.createNBaseClient = exports.createNBaseVectorClient = exports.NBaseError = exports.NBaseVectorClient = exports.quote = exports.buildOrderBy = exports.parseWhere = exports.setAdapterMetadata = exports.executorFromAdapter = exports.createAdapterProxy = exports.createAn5Adapter = exports.AdapterTableClient = exports.An5Adapter = void 0;
4
4
  var an5Adapter_1 = require("./an5Adapter");
5
5
  Object.defineProperty(exports, "An5Adapter", { enumerable: true, get: function () { return an5Adapter_1.An5Adapter; } });
6
6
  Object.defineProperty(exports, "AdapterTableClient", { enumerable: true, get: function () { return an5Adapter_1.AdapterTableClient; } });
@@ -20,6 +20,13 @@ Object.defineProperty(exports, "createNBaseClient", { enumerable: true, get: fun
20
20
  Object.defineProperty(exports, "parseNBaseConnectionString", { enumerable: true, get: function () { return nbase_1.parseNBaseConnectionString; } });
21
21
  Object.defineProperty(exports, "buildVectorId", { enumerable: true, get: function () { return nbase_1.buildVectorId; } });
22
22
  Object.defineProperty(exports, "parseVectorId", { enumerable: true, get: function () { return nbase_1.parseVectorId; } });
23
+ var vector_1 = require("./sqlite/vector");
24
+ Object.defineProperty(exports, "encodeVector", { enumerable: true, get: function () { return vector_1.encodeVector; } });
25
+ Object.defineProperty(exports, "decodeVector", { enumerable: true, get: function () { return vector_1.decodeVector; } });
26
+ Object.defineProperty(exports, "vectorDistance", { enumerable: true, get: function () { return vector_1.vectorDistance; } });
27
+ Object.defineProperty(exports, "isVectorField", { enumerable: true, get: function () { return vector_1.isVectorField; } });
28
+ Object.defineProperty(exports, "AN5_VECTOR_FUNCTIONS", { enumerable: true, get: function () { return vector_1.AN5_VECTOR_FUNCTIONS; } });
29
+ Object.defineProperty(exports, "SQLITE_VEC_FUNCTIONS", { enumerable: true, get: function () { return vector_1.SQLITE_VEC_FUNCTIONS; } });
23
30
  var config_1 = require("./config");
24
31
  Object.defineProperty(exports, "getLlmConfig", { enumerable: true, get: function () { return config_1.getLlmConfig; } });
25
32
  Object.defineProperty(exports, "setLlmConfig", { enumerable: true, get: function () { return config_1.setLlmConfig; } });
@@ -1,4 +1,5 @@
1
1
  import type { Dialect, QueryEngine, TransactionHandle } from '../base/types';
2
+ import { SqliteVectorSupport } from './vector';
2
3
  export interface SqliteDriver {
3
4
  exec<T = any>(query: string, params?: Record<string, any>): Promise<T[]> | T[];
4
5
  executeRaw?(query: string, params?: Record<string, any>): Promise<number> | number;
@@ -9,9 +10,21 @@ export interface SqliteBrowserConfig {
9
10
  driver?: SqliteDriver;
10
11
  exec?: <T = any>(query: string, params?: Record<string, any>) => Promise<T[]> | T[];
11
12
  executeRaw?: (query: string, params?: Record<string, any>) => Promise<number> | number;
13
+ /** Path to the sqlite-vec extension, for WASM builds that allow loading one. */
14
+ sqliteVec?: string;
15
+ /** Pin the vector search strategy instead of probing for one. */
16
+ vectorStrategy?: 'auto' | 'sqlite-vec' | 'udf' | 'sql' | 'memory';
17
+ /** Register a scalar function with the driver, when its API allows it. */
18
+ registerFunction?: (name: string, fn: (...args: any[]) => any) => void;
12
19
  }
13
20
  export declare class SqliteBrowserEngine implements QueryEngine {
14
21
  dialect: Dialect;
22
+ /**
23
+ * A WASM build cannot register a JS callback as an SQL function, so this only
24
+ * reaches the `sql` strategy (json_each in plain SQL) and the in-memory one.
25
+ * A custom `registerFunction` hook unlocks `udf` as well.
26
+ */
27
+ readonly vectorSupport: SqliteVectorSupport;
15
28
  private db;
16
29
  private driver;
17
30
  private customExec;
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.SqliteBrowserEngine = void 0;
4
4
  exports.createBrowserSqliteAdapter = createBrowserSqliteAdapter;
5
+ const vector_1 = require("./vector");
5
6
  // ─── Browser SQLite Engine ──────────────────────────────────────────────────────────
6
7
  class SqliteBrowserEngine {
7
8
  constructor(config) {
@@ -10,6 +11,18 @@ class SqliteBrowserEngine {
10
11
  this.driver = null;
11
12
  this.customExec = null;
12
13
  this.customExecuteRaw = null;
14
+ this.vectorSupport = new vector_1.SqliteVectorSupport({
15
+ native: () => this.db,
16
+ registerFunction: (name, fn) => {
17
+ const register = config?.registerFunction;
18
+ if (typeof register === 'function')
19
+ return register(name, fn);
20
+ const db = this.db;
21
+ if (db && typeof db.create_function === 'function')
22
+ return db.create_function(name, fn);
23
+ throw new Error('this SQLite build cannot register functions');
24
+ },
25
+ }, { sqliteVec: config?.sqliteVec, vectorStrategy: config?.vectorStrategy });
13
26
  if (!config)
14
27
  return;
15
28
  if (config.exec && typeof config.exec === 'function') {
@@ -1,9 +1,14 @@
1
1
  import type { An5AdapterConfig, Dialect, QueryEngine, TransactionHandle } from '../base/types';
2
+ import { SqliteVectorSupport } from './vector';
2
3
  export declare class SqliteEngine implements QueryEngine {
3
4
  dialect: Dialect;
5
+ /** Registers `an5_vec_*` and loads sqlite-vec for in-database vector search. */
6
+ readonly vectorSupport: SqliteVectorSupport;
4
7
  private db;
5
8
  private readonly filePath;
6
9
  constructor(adapterConfig: An5AdapterConfig);
10
+ /** The better-sqlite3 handle, needed for extension loading. */
11
+ native(): any;
7
12
  private getDb;
8
13
  exec<T = any>(query: string, params?: Record<string, any>): Promise<T[]>;
9
14
  executeRaw(query: string, params?: Record<string, any>): Promise<number>;