@an5/adapters 0.2.11 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +137 -5
  2. package/dist/an5Adapter.d.ts +42 -1
  3. package/dist/an5Adapter.js +146 -38
  4. package/dist/base/types.d.ts +14 -0
  5. package/dist/index.d.ts +2 -0
  6. package/dist/index.js +8 -1
  7. package/dist/sqlite/browserEngine.d.ts +13 -0
  8. package/dist/sqlite/browserEngine.js +13 -0
  9. package/dist/sqlite/engine.d.ts +5 -0
  10. package/dist/sqlite/engine.js +10 -0
  11. package/dist/sqlite/index.d.ts +1 -0
  12. package/dist/sqlite/index.js +1 -0
  13. package/dist/sqlite/vector.d.ts +155 -0
  14. package/dist/sqlite/vector.js +432 -0
  15. package/dotnet/An5Adapter.cs +99 -28
  16. package/dotnet/Base/Core.cs +14 -0
  17. package/dotnet/Mssql/MssqlEngine.cs +44 -1
  18. package/dotnet/Postgres/PostgresEngine.cs +2 -0
  19. package/dotnet/Sqlite/SqliteEngine.cs +82 -2
  20. package/dotnet/Sqlite/SqliteVectors.cs +401 -0
  21. package/golang/an5_adapter.go +84 -32
  22. package/golang/base/vectors.go +263 -0
  23. package/golang/sqlite/vector.go +248 -0
  24. package/golang/sqlite_vector_test.go +288 -0
  25. package/golang/table_client.go +67 -3
  26. package/java/pom.xml +177 -0
  27. package/java/src/main/java/an5/adapters/An5Adapter.java +374 -0
  28. package/java/src/main/java/an5/adapters/An5Aggregate.java +116 -0
  29. package/java/src/main/java/an5/adapters/An5GroupBy.java +97 -0
  30. package/java/src/main/java/an5/adapters/An5Query.java +100 -0
  31. package/java/src/main/java/an5/adapters/An5TableClient.java +1030 -0
  32. package/java/src/main/java/an5/adapters/An5ViewClient.java +107 -0
  33. package/java/src/main/java/an5/adapters/base/Dialect.java +103 -0
  34. package/java/src/main/java/an5/adapters/base/FieldMeta.java +42 -0
  35. package/java/src/main/java/an5/adapters/base/JdbcUrls.java +201 -0
  36. package/java/src/main/java/an5/adapters/base/Metadata.java +195 -0
  37. package/java/src/main/java/an5/adapters/base/RelationDef.java +26 -0
  38. package/java/src/main/java/an5/adapters/base/SqlBuilder.java +422 -0
  39. package/java/src/main/java/an5/adapters/base/SqliteVectors.java +375 -0
  40. package/java/src/main/java/an5/adapters/base/Vectors.java +123 -0
  41. package/java/src/test/java/An5AdaptersSmoke.java +733 -0
  42. package/java/src/test/java/Json.java +174 -0
  43. package/java/src/test/java/QuerySemantics.java +111 -0
  44. package/kotlin/build.gradle.kts +145 -0
  45. package/kotlin/settings.gradle.kts +10 -0
  46. package/kotlin/src/main/kotlin/an5/adapters/Aggregate.kt +89 -0
  47. package/kotlin/src/main/kotlin/an5/adapters/An5.kt +91 -0
  48. package/kotlin/src/main/kotlin/an5/adapters/Query.kt +130 -0
  49. package/kotlin/src/main/kotlin/an5/adapters/Rows.kt +166 -0
  50. package/kotlin/src/main/kotlin/an5/adapters/TableClient.kt +169 -0
  51. package/kotlin/src/main/kotlin/an5/adapters/Types.kt +77 -0
  52. package/kotlin/src/test/kotlin/an5/adapters/An5Smoke.kt +329 -0
  53. package/native/sqlite/CMakeLists.txt +12 -0
  54. package/native/sqlite/README.md +117 -0
  55. package/native/sqlite/an5_vector.c +209 -0
  56. package/package.json +33 -7
  57. package/python/an5_adapter.py +37 -6
  58. package/python/base/__init__.py +15 -0
  59. package/python/base/vectors.py +431 -0
  60. package/python/sqlite/__init__.py +2 -2
  61. package/python/sqlite/provider.py +21 -1
  62. package/python/table_client.py +129 -59
  63. package/rust/src/adapter.rs +342 -24
  64. package/rust/src/base/mod.rs +7 -0
  65. package/rust/src/base/vectors.rs +293 -0
  66. package/rust/src/lib.rs +3 -3
  67. package/scripts/sqlite-native-build.js +32 -0
  68. package/swift/Package.swift +22 -0
  69. package/swift/Sources/An5Adapters/An5Adapter.swift +160 -0
  70. package/swift/Sources/An5Adapters/Metadata.swift +183 -0
  71. package/swift/Sources/An5Adapters/Query.swift +105 -0
  72. package/swift/Sources/An5Adapters/Row+Accessors.swift +129 -0
  73. package/swift/Sources/An5Adapters/SQLBuilder.swift +359 -0
  74. package/swift/Sources/An5Adapters/SQLiteDriver.swift +301 -0
  75. package/swift/Sources/An5Adapters/SqliteVectors.swift +243 -0
  76. package/swift/Sources/An5Adapters/TableClient.swift +671 -0
  77. package/swift/Sources/An5Adapters/Types.swift +127 -0
  78. package/swift/Sources/An5Adapters/Vectors.swift +86 -0
  79. package/swift/Sources/An5Adapters/ViewClient.swift +50 -0
  80. package/swift/Sources/CSQLite/module.modulemap +8 -0
  81. package/swift/Sources/CSQLite/shim.h +9 -0
  82. package/swift/Tests/An5AdaptersTests/QuerySemanticsTests.swift +51 -0
  83. package/swift/Tests/An5AdaptersTests/RuntimeSmokeTests.swift +488 -0
  84. package/typescript/src/an5Adapter.ts +155 -31
  85. package/typescript/src/base/types.ts +15 -0
  86. package/typescript/src/index.ts +13 -0
  87. package/typescript/src/sqlite/browserEngine.ts +24 -0
  88. package/typescript/src/sqlite/engine.ts +13 -0
  89. package/typescript/src/sqlite/index.ts +1 -1
  90. package/typescript/src/sqlite/vector.ts +537 -0
  91. package/golang/sqlite/sqlite.go +0 -14
@@ -0,0 +1,329 @@
1
+ package an5.adapters
2
+
3
+ /**
4
+ * End-to-end exercise of the Kotlin runtime against a real in-memory SQLite database.
5
+ *
6
+ * Covers what a Java-only check cannot: that Kotlin's read-only maps reach the JDBC layer,
7
+ * that the query block builds the filter the other adapters build, and that a column read
8
+ * as `Long` from SQLite comes out of the typed accessors as `Int` without a cast at the
9
+ * call site.
10
+ */
11
+ object An5Smoke {
12
+
13
+ private var failures = 0
14
+
15
+ @JvmStatic
16
+ fun main(args: Array<String>) {
17
+ crud()
18
+ filters()
19
+ aggregates()
20
+ relations()
21
+ transactions()
22
+ vectorFallback()
23
+ sqliteVectorSearch()
24
+
25
+ if (failures > 0) {
26
+ println("kotlin smoke: $failures check(s) failed")
27
+ kotlin.system.exitProcess(1)
28
+ }
29
+ println("kotlin smoke: all checks passed")
30
+ }
31
+
32
+ private fun crud() {
33
+ val db = open()
34
+ db.use {
35
+ val users = it.table("User")
36
+ val created = users.create(mapOf("name" to "Ada", "score" to 10))
37
+ check("generated primary key") { created.string("id") != null }
38
+ check("create returns the stored row") { created.string("name") == "Ada" }
39
+ check("row accessors read a column") { created.int("score") == 10 }
40
+ check("row accessors widen a number") { created.long("score") == 10L }
41
+ check("missing column reads as the fallback") { created.int("nope", -1) == -1 }
42
+ check("null column reads as the fallback") { created.string("nope") == null }
43
+
44
+ users.create(mapOf("name" to "Grace", "score" to 20))
45
+ users.create(mapOf("name" to "Alan", "score" to 30))
46
+
47
+ check("count") { users.count() == 3L }
48
+ check("ordered results") { users.findMany { orderBy("score", Sort.DESC) }.first().string("name") == "Alan" }
49
+ check("paged results") {
50
+ users.findMany { orderBy("score"); skip(1); take(1) }.single().string("name") == "Grace"
51
+ }
52
+ check("select projects columns") {
53
+ users.findMany { select("name") }.all { it.keys == setOf("name") }
54
+ }
55
+ check("update") {
56
+ users.update(mapOf("id" to created.string("id")), mapOf("score" to 99))?.int("score") == 99
57
+ }
58
+ check("updateMany") {
59
+ users.updateMany(null, mapOf("score" to mapOf("increment" to 1))) == 3
60
+ }
61
+ check("upsert updates") {
62
+ users.upsert(mapOf("id" to created.string("id")), mapOf("name" to "x"), mapOf("name" to "Ada L"))
63
+ .string("name") == "Ada L"
64
+ }
65
+ check("upsert creates") {
66
+ users.upsert(mapOf("name" to "Katherine"), mapOf("name" to "Katherine"), mapOf("name" to "K"))
67
+ .string("name") == "Katherine"
68
+ }
69
+ check("delete") { users.delete(mapOf("id" to created.string("id"))) != null }
70
+ check("deleteMany") { users.deleteMany(null) == 3 }
71
+ }
72
+ }
73
+
74
+ private fun filters() {
75
+ val db = open()
76
+ db.use {
77
+ val users = it.table("User")
78
+ users.createMany(
79
+ listOf(
80
+ mapOf("name" to "Ada", "score" to 10),
81
+ mapOf("name" to "Grace", "score" to 20),
82
+ mapOf("name" to "Alan", "score" to 30),
83
+ )
84
+ )
85
+ check("gte") { users.findMany { where("score" gte 20) }.size == 2 }
86
+ check("lt") { users.findMany { where("score" lt 20) }.size == 1 }
87
+ check("eq") { users.findMany { where("name" eq "Ada") }.size == 1 }
88
+ check("neq") { users.findMany { where("name" neq "Ada") }.size == 2 }
89
+ check("contains") { users.findMany { where("name" contains "da") }.size == 1 }
90
+ check("endsWith") { users.findMany { where("name" endsWith "ce") }.size == 1 }
91
+ check("inList") { users.findMany { where("name" inList listOf("Ada", "Alan")) }.size == 2 }
92
+ check("empty inList matches nothing") {
93
+ users.findMany { where("name" inList emptyList<String>()) }.isEmpty()
94
+ }
95
+ check("empty notInList matches everything") {
96
+ users.findMany { where("name" notInList emptyList<String>()) }.size == 3
97
+ }
98
+ check("orOf") { users.findMany { where(orOf("name" eq "Ada", "name" eq "Alan")) }.size == 2 }
99
+ check("andOf") { users.findMany { where(andOf("score" gte 10, "score" lt 30)) }.size == 2 }
100
+ check("notOf") { users.findMany { where(notOf("name" eq "Ada", "name" eq "Grace")) }.size == 1 }
101
+ check("empty orOf matches nothing") { users.findMany { where(orOf()) }.isEmpty() }
102
+ check("two where calls combine with AND") {
103
+ users.findMany { where("score" gte 10); where("score" lte 20) }.size == 2
104
+ }
105
+ check("isNull") { users.findMany { where("nickname".isNull()) }.size == 3 }
106
+ check("isNotNull") { users.findMany { where("name".isNotNull()) }.size == 3 }
107
+ check("findFirst") { users.findFirst(mapOf("name" to "Ada"))?.int("score") == 10 }
108
+ check("findFirst misses") { users.findFirst(mapOf("name" to "Nobody")) == null }
109
+ }
110
+ }
111
+
112
+ private fun aggregates() {
113
+ val db = open()
114
+ db.use {
115
+ val users = it.table("User")
116
+ users.createMany(
117
+ listOf(
118
+ mapOf("name" to "Ada", "score" to 10),
119
+ mapOf("name" to "Grace", "score" to 20),
120
+ mapOf("name" to "Alan", "score" to 30),
121
+ )
122
+ )
123
+ val totals = users.aggregate { count(); sum("score"); avg("score"); max("score") }
124
+ check("aggregate count") { (totals["_count"] as Number).toLong() == 3L }
125
+ check("aggregate sum") { (totals["_sum_score"] as Number).toDouble() == 60.0 }
126
+ check("aggregate avg") { (totals["_avg_score"] as Number).toDouble() == 20.0 }
127
+ check("aggregate max") { (totals["_max_score"] as Number).toDouble() == 30.0 }
128
+
129
+ val filtered = users.aggregate { count(); where("score" gte 20) }
130
+ check("aggregate where") { (filtered["_count"] as Number).toLong() == 2L }
131
+
132
+ val groups = users.groupBy("name") { sum("score") }
133
+ check("groupBy rows") { groups.size == 3 }
134
+ check("groupBy counts") { groups.all { (it["_count"] as Number).toLong() == 1L } }
135
+ }
136
+ }
137
+
138
+ private fun relations() {
139
+ val db = open()
140
+ db.use {
141
+ val ada = it.table("User").create(mapOf("name" to "Ada", "score" to 1))
142
+ it.table("Post").create(mapOf("title" to "first", "userId" to ada.string("id")))
143
+ it.table("Post").create(mapOf("title" to "second", "userId" to ada.string("id")))
144
+
145
+ val rows = it.table("User").findMany {
146
+ include(mapOf("posts" to true, "_count" to true))
147
+ }
148
+ check("many relation eager-loaded") { rows.single().related("posts").size == 2 }
149
+ check("relation count eager-loaded") { rows.single().relationCount("posts") == 2 }
150
+ check("related rows are readable") {
151
+ rows.single().related("posts").first().string("title") == "first"
152
+ }
153
+ check("missing relation reads as empty") { rows.single().related("comments").isEmpty() }
154
+ check("missing relation count reads as zero") { rows.single().relationCount("comments") == 0 }
155
+ }
156
+ }
157
+
158
+ private fun transactions() {
159
+ val db = open()
160
+ db.use {
161
+ it.table("User").create(mapOf("name" to "outside"))
162
+ val failed = runCatching {
163
+ it.transaction { scoped ->
164
+ scoped.table("User").create(mapOf("name" to "inside"))
165
+ error("rollback")
166
+ }
167
+ }
168
+ check("transaction rethrows") { failed.isFailure && failed.exceptionOrNull()?.message == "rollback" }
169
+ check("rollback discarded the inner insert") { it.table("User").count() == 1L }
170
+
171
+ it.transaction { scoped -> scoped.table("User").create(mapOf("name" to "committed")) }
172
+ check("commit kept the insert") { it.table("User").count() == 2L }
173
+ }
174
+ }
175
+
176
+ private fun vectorFallback() {
177
+ val db = open()
178
+ db.use {
179
+ val docs = it.table("Document")
180
+ docs.create(mapOf("title" to "far", "embedding" to "[0.0, 1.0]"))
181
+ docs.create(mapOf("title" to "near", "embedding" to "[1.0, 0.1]"))
182
+ docs.create(mapOf("title" to "no vector", "embedding" to null))
183
+
184
+ val hits = docs.vectorSearch(doubleArrayOf(1.0, 0.0), take = 2)
185
+ check("rows without a vector are skipped") { hits.size == 2 }
186
+ check("nearest first") { hits.first().string("title") == "near" }
187
+ check("distance reported") { hits.first().containsKey("distance") }
188
+ }
189
+ }
190
+
191
+ /**
192
+ * SQLite ranks a `VECTOR(n)` column inside the database, and a `DoubleArray` written
193
+ * to one comes back as numbers.
194
+ *
195
+ * The docstring in the JVM adapter's `SqliteVectors` is the shared specification every
196
+ * runtime implements; test/sqlite-vector.test.js (TypeScript) is the mirror.
197
+ */
198
+ private fun sqliteVectorSearch() {
199
+ an5.adapters.base.Metadata.setAdapterMetadata(blobMetadata())
200
+ val db = An5(":memory:")
201
+ db.use {
202
+ it.executeRaw("CREATE TABLE documents (id TEXT PRIMARY KEY, title TEXT, embedding BLOB)")
203
+ val vectors = listOf(
204
+ "d1" to doubleArrayOf(1.0, 0.0, 0.0),
205
+ "d2" to doubleArrayOf(0.8, 0.2, 0.0),
206
+ "d3" to doubleArrayOf(0.0, 1.0, 0.0),
207
+ )
208
+ for ((id, vector) in vectors) {
209
+ it.executeRaw(
210
+ "INSERT INTO documents (id, title, embedding) VALUES (?, ?, ?)",
211
+ id, id, an5.adapters.base.SqliteVectors.encodeVector(vector),
212
+ )
213
+ }
214
+ // A row with another dimension must never rank against a 3-dimension query.
215
+ it.executeRaw(
216
+ "INSERT INTO documents (id, title, embedding) VALUES ('d4', 'd4', ?)",
217
+ an5.adapters.base.SqliteVectors.encodeVector(doubleArrayOf(1.0, 0.0, 0.0, 1.0)),
218
+ )
219
+
220
+ val docs = it.table("Document")
221
+ for (metric in DistanceMetric.entries) {
222
+ val hits = docs.vectorSearch(doubleArrayOf(1.0, 0.0, 0.0), take = 9, metric = metric)
223
+ check("${metric.sql}: row order") { hits.joinToString(",") { row -> row.string("title").orEmpty() } == "d1,d2,d3" }
224
+ check("${metric.sql}: the distance is a number") { hits.first()["distance"] is Double }
225
+ check("${metric.sql}: the vector decodes to numbers") { hits.first()["embedding"] is DoubleArray }
226
+ }
227
+
228
+ // `memory` pins the client-side fallback, which ranks the same rows.
229
+ it.vectorStrategy("memory")
230
+ val pinned = docs.vectorSearch(doubleArrayOf(1.0, 0.0, 0.0), take = 9)
231
+ check("the memory fallback ranks correctly") {
232
+ pinned.joinToString(",") { row -> row.string("title").orEmpty() } == "d1,d2,d3"
233
+ }
234
+
235
+ it.vectorStrategy(null)
236
+ val written = docs.create(
237
+ mapOf("id" to "d9", "title" to "written", "embedding" to doubleArrayOf(0.25, 0.5, 1.0)),
238
+ )
239
+ check("create writes a float32 BLOB") {
240
+ it.queryRaw("SELECT typeof(embedding) AS t FROM documents WHERE id = 'd9'")
241
+ .first().string("t") == "blob"
242
+ }
243
+ check("create returns numbers") { written["embedding"] is DoubleArray }
244
+ check("findMany returns numbers") {
245
+ docs.findMany(where = mapOf("id" to "d9")).first()["embedding"] is DoubleArray
246
+ }
247
+ docs.update(mapOf("id" to "d1"), mapOf("embedding" to doubleArrayOf(0.5, 0.5, 0.0)))
248
+ check("update encodes") {
249
+ docs.findMany(where = mapOf("id" to "d1")).first()["embedding"] is DoubleArray
250
+ }
251
+ docs.updateMany(mapOf("id" to "d2"), mapOf("embedding" to doubleArrayOf(0.0, 0.0, 1.0)))
252
+ check("updateMany encodes") {
253
+ docs.findMany(where = mapOf("id" to "d2")).first()["embedding"] is DoubleArray
254
+ }
255
+ }
256
+ }
257
+
258
+ private fun blobMetadata(): MutableMap<String, Any?> {
259
+ val metadata = metadata()
260
+ @Suppress("UNCHECKED_CAST")
261
+ val fields = metadata["modelFields"] as MutableMap<String, Any?>
262
+ fields["Document"] = listOf(
263
+ field("id", true),
264
+ field("title", false),
265
+ mutableMapOf<String, Any?>(
266
+ "name" to "embedding",
267
+ "type" to "number[] | string",
268
+ "sql" to "VECTOR(3)",
269
+ "isOptional" to true,
270
+ "hasDefault" to false,
271
+ "isId" to false,
272
+ ),
273
+ )
274
+ return metadata
275
+ }
276
+
277
+ // ─── Harness ──────────────────────────────────────────────────────────────────
278
+
279
+ private fun open(): An5 {
280
+ an5.adapters.base.Metadata.setAdapterMetadata(metadata())
281
+ val db = An5(":memory:")
282
+ db.executeRaw("CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT, score INTEGER, nickname TEXT)")
283
+ db.executeRaw("CREATE TABLE documents (id TEXT PRIMARY KEY, title TEXT, embedding TEXT)")
284
+ db.executeRaw("CREATE TABLE posts (id TEXT PRIMARY KEY, title TEXT, userId TEXT)")
285
+ return db
286
+ }
287
+
288
+ private fun metadata(): MutableMap<String, Any?> {
289
+ val posts = mutableMapOf<String, Any?>(
290
+ "modelName" to "Post",
291
+ "relationType" to "many",
292
+ "foreignKey" to "userId",
293
+ "localKey" to "id",
294
+ )
295
+ return mutableMapOf(
296
+ "modelToTable" to mutableMapOf(
297
+ "User" to "users",
298
+ "Post" to "posts",
299
+ "Document" to "documents",
300
+ ),
301
+ "modelFields" to mutableMapOf(
302
+ "User" to listOf(field("id", true), field("name", false), field("score", false)),
303
+ "Post" to listOf(field("id", true), field("title", false), field("userId", false)),
304
+ "Document" to listOf(field("id", true), field("title", false), field("embedding", false)),
305
+ ),
306
+ "relationMap" to mutableMapOf("User" to mutableMapOf("posts" to posts)),
307
+ )
308
+ }
309
+
310
+ private fun field(name: String, id: Boolean): MutableMap<String, Any?> = mutableMapOf(
311
+ "name" to name,
312
+ "type" to "string",
313
+ "sql" to "NVARCHAR(255)",
314
+ "isOptional" to false,
315
+ "hasDefault" to false,
316
+ "isId" to id,
317
+ )
318
+
319
+ private fun check(what: String, assertion: () -> Boolean) {
320
+ val outcome = runCatching(assertion)
321
+ when {
322
+ outcome.isSuccess && outcome.getOrNull() == true -> println("ok $what")
323
+ else -> {
324
+ failures++
325
+ println("FAIL $what: ${outcome.exceptionOrNull()?.message ?: "assertion returned false"}")
326
+ }
327
+ }
328
+ }
329
+ }
@@ -0,0 +1,12 @@
1
+ cmake_minimum_required(VERSION 3.16)
2
+ project(an5_vector LANGUAGES C)
3
+ find_path(SQLITE_INCLUDE_DIR sqlite3ext.h REQUIRED)
4
+ add_library(an5_vector MODULE an5_vector.c)
5
+ target_include_directories(an5_vector PRIVATE "${SQLITE_INCLUDE_DIR}")
6
+ set_target_properties(an5_vector PROPERTIES PREFIX "" C_STANDARD 99)
7
+ if(MSVC)
8
+ target_compile_options(an5_vector PRIVATE /O2 /W4)
9
+ else()
10
+ target_compile_options(an5_vector PRIVATE -O3 -Wall -Wextra -Werror)
11
+ target_link_libraries(an5_vector PRIVATE m)
12
+ endif()
@@ -0,0 +1,117 @@
1
+ # AN5 native SQLite vectors
2
+
3
+ `an5_vector.c` is a loadable SQLite extension, not a language-driver callback.
4
+ It implements all three AN5 metrics in C:
5
+
6
+ | SQL function | Result (smaller is closer) |
7
+ |---|---|
8
+ | `an5_vec_cosine(left, right)` | `1 - cosine_similarity` |
9
+ | `an5_vec_l2(left, right)` | Euclidean distance |
10
+ | `an5_vec_ip(left, right)` | Negative dot product |
11
+ | `an5_vector_version()` | Extension identifier |
12
+
13
+ BLOB inputs are little-endian float32. JSON text arrays are also accepted and
14
+ converted to float32, so native results have single-precision input values even
15
+ for legacy text columns. Empty, malformed, non-finite and dimension-mismatched
16
+ inputs return SQL NULL. Zero vectors have cosine distance 1.
17
+
18
+ The BLOB path reads the stored bytes directly without allocating a row vector.
19
+ The query operand is cached using SQLite auxdata for the statement. x86-64 uses
20
+ SSE2 with double-precision accumulators; other architectures use the portable C
21
+ loop. No `-march=native`, AVX dependency or fast-math flags are required.
22
+
23
+ This is **exact scanning**, not an ANN index. A normal relational index on a
24
+ filter column can narrow the candidates before distance calculation. Only the
25
+ top results are returned to the language runtime.
26
+
27
+ ## Build
28
+
29
+ In this workspace:
30
+
31
+ ```sh
32
+ npm run build:sqlite:native -w an5Adapters
33
+ ```
34
+
35
+ The command prints the absolute library path under `native/sqlite/build`.
36
+ Linux/macOS need a C compiler and `sqlite3ext.h`; on Debian/Ubuntu install
37
+ `build-essential` and `libsqlite3-dev`. `CC` and `SQLITE_INCLUDE_DIR` may be set
38
+ when using a different compiler or header directory.
39
+
40
+ For an installed npm package:
41
+
42
+ ```sh
43
+ node node_modules/@an5/adapters/scripts/sqlite-native-build.js ./native-build
44
+ ```
45
+
46
+ Windows builds through CMake and an installed C toolchain. The CMake project can
47
+ also be used directly on other platforms:
48
+
49
+ ```sh
50
+ cmake -S native/sqlite -B native/sqlite/build -DCMAKE_BUILD_TYPE=Release
51
+ cmake --build native/sqlite/build --config Release
52
+ ```
53
+
54
+ Build for the target OS and architecture. No precompiled binary is downloaded
55
+ or compiled during the package's ordinary install.
56
+
57
+ ## Load
58
+
59
+ The existing extension-path option accepts either sqlite-vec or this library:
60
+
61
+ ```typescript
62
+ const db = createAn5Adapter({
63
+ connectionString: 'sqlite:///app.db',
64
+ sqliteVec: '/absolute/path/an5_vector.so', // .dylib / .dll on other platforms
65
+ });
66
+ await db.document.vectorSearch({ vector: query, take: 10 });
67
+ ```
68
+
69
+ ```python
70
+ db = create_an5_adapter('sqlite:///app.db', sqlite_vec='/absolute/path/an5_vector.so')
71
+ ```
72
+
73
+ ```csharp
74
+ var db = new An5Adapter(new An5AdapterOptions {
75
+ ConnectionString = "app.db",
76
+ SqliteVec = "/absolute/path/an5_vector.so"
77
+ });
78
+ ```
79
+
80
+ TypeScript, Python, .NET and Swift recognize `an5_vector_version()` before
81
+ registering callbacks, preserving the native functions. In the existing strategy
82
+ API this uses `udf`; **the functions execute in C**, not in the host language.
83
+ On runtimes that take an existing connection/pool, load the extension on every
84
+ SQLite connection through the driver's extension-loading API. Go, Rust and JDBC
85
+ then discover the `an5_vec_*` functions through their existing probes. Swift's
86
+ loading hook is desktop-only; mobile platforms need a statically linked SQLite
87
+ extension/build supplied by the application.
88
+
89
+ Each gate builds the extension itself and checks that the native functions are
90
+ still the ones answering: `test:sqlite:native` covers TypeScript, Python and
91
+ (optionally) .NET, while `test:swift` builds the extension and runs the same
92
+ check through the Swift driver.
93
+
94
+ ## Verify and benchmark
95
+
96
+ ```sh
97
+ npm run test:sqlite:native -w an5Adapters
98
+ node an5Adapters/scripts/sqlite-native-test.js --dotnet
99
+ AN5_NATIVE_SANITIZE=1 npm run test:sqlite:native -w an5Adapters
100
+ npm run bench:sqlite:native -w an5Adapters
101
+ ```
102
+
103
+ The native test compiles into a disposable directory, tests the extension with
104
+ real SQLite, checks all metrics against the reference implementation, and verifies
105
+ that TypeScript and Python keep the C functions. `--dotnet` additionally tests
106
+ .NET. The sanitizer option enables UBSan on supporting Unix compilers.
107
+
108
+ `npm run test:swift -w an5Adapters` builds the extension too and asserts the
109
+ Swift driver keeps it; that case is skipped when `AN5_NATIVE_VECTOR_PATH` is
110
+ unset, as on a machine without a C compiler.
111
+
112
+ The benchmark compares the public adapter's native and JavaScript-UDF paths with
113
+ identical BLOB data and verifies equal top-10 IDs. It reports the median of seven
114
+ warmed runs over 4,000 vectors of 256 dimensions. On the development Linux x86-64
115
+ machine the native path measured about 4 ms per search versus 15–18 ms for the
116
+ JavaScript callback (roughly 4x faster). Timings are illustrative, not pass/fail
117
+ thresholds or a performance guarantee.
@@ -0,0 +1,209 @@
1
+ /* Portable SQLite extension for float32 vector distances. No driver callbacks.
2
+ * BLOB inputs are little-endian float32; JSON arrays remain readable.
3
+ * Query vectors use SQLite auxdata so parsing/copying happens once per statement.
4
+ */
5
+ #include <sqlite3ext.h>
6
+ SQLITE_EXTENSION_INIT1
7
+ #include <math.h>
8
+ #include <stdint.h>
9
+ #include <stdlib.h>
10
+ #include <string.h>
11
+ #if defined(__SSE2__)
12
+ #include <emmintrin.h>
13
+ #endif
14
+
15
+ typedef struct {
16
+ const unsigned char *bytes;
17
+ float *owned;
18
+ int count;
19
+ } Vector;
20
+
21
+ static void release_vector(void *ptr) {
22
+ Vector *v = (Vector *)ptr;
23
+ if (v) { sqlite3_free(v->owned); sqlite3_free(v); }
24
+ }
25
+
26
+ static int whitespace(unsigned char c) {
27
+ return c == ' ' || c == '\t' || c == '\r' || c == '\n';
28
+ }
29
+
30
+ /* Validate JSON number grammar before using strtod, which otherwise accepts
31
+ * non-JSON spellings such as NaN, Infinity and hexadecimal numbers. */
32
+ static const unsigned char *number_end(const unsigned char *p, const unsigned char *end) {
33
+ if (p < end && *p == '-') ++p;
34
+ if (p == end) return NULL;
35
+ if (*p == '0') ++p;
36
+ else {
37
+ if (*p < '1' || *p > '9') return NULL;
38
+ do { ++p; } while (p < end && *p >= '0' && *p <= '9');
39
+ }
40
+ if (p < end && *p == '.') {
41
+ ++p;
42
+ if (p == end || *p < '0' || *p > '9') return NULL;
43
+ do { ++p; } while (p < end && *p >= '0' && *p <= '9');
44
+ }
45
+ if (p < end && (*p == 'e' || *p == 'E')) {
46
+ ++p;
47
+ if (p < end && (*p == '+' || *p == '-')) ++p;
48
+ if (p == end || *p < '0' || *p > '9') return NULL;
49
+ do { ++p; } while (p < end && *p >= '0' && *p <= '9');
50
+ }
51
+ return p;
52
+ }
53
+
54
+ /* 1 = vector, 0 = invalid input, -1 = allocation failure. */
55
+ static int read_vector(sqlite3_value *arg, Vector *v, int copy_blob) {
56
+ memset(v, 0, sizeof(*v));
57
+ if (sqlite3_value_type(arg) == SQLITE_BLOB) {
58
+ int size = sqlite3_value_bytes(arg);
59
+ if (!size || size % 4) return 0;
60
+ v->count = size / 4;
61
+ v->bytes = sqlite3_value_blob(arg);
62
+ if (!v->bytes) return -1;
63
+ if (copy_blob) {
64
+ v->owned = sqlite3_malloc64((sqlite3_uint64)size);
65
+ if (!v->owned) return -1;
66
+ memcpy(v->owned, v->bytes, (size_t)size);
67
+ v->bytes = (const unsigned char *)v->owned;
68
+ }
69
+ return 1;
70
+ }
71
+ if (sqlite3_value_type(arg) != SQLITE_TEXT) return 0;
72
+ const unsigned char *p = sqlite3_value_text(arg);
73
+ if (!p) return -1;
74
+ const unsigned char *end = p + sqlite3_value_bytes(arg);
75
+ while (p < end && whitespace(*p)) ++p;
76
+ if (p == end || *p++ != '[') return 0;
77
+ int capacity = 0;
78
+ for (;;) {
79
+ while (p < end && whitespace(*p)) ++p;
80
+ const unsigned char *next = number_end(p, end);
81
+ if (!next) goto invalid;
82
+ char *parsed;
83
+ double n = strtod((const char *)p, &parsed);
84
+ if ((const unsigned char *)parsed != next || !isfinite(n) || !isfinite((float)n)) goto invalid;
85
+ if (v->count == capacity) {
86
+ if (capacity > 0x3fffffff) goto invalid;
87
+ capacity = capacity ? capacity * 2 : 32;
88
+ float *grown = sqlite3_realloc64(v->owned, (sqlite3_uint64)capacity * sizeof(float));
89
+ if (!grown) { sqlite3_free(v->owned); v->owned = NULL; return -1; }
90
+ v->owned = grown;
91
+ }
92
+ v->owned[v->count++] = (float)n;
93
+ p = next;
94
+ while (p < end && whitespace(*p)) ++p;
95
+ if (p == end) goto invalid;
96
+ if (*p == ']') { ++p; break; }
97
+ if (*p++ != ',') goto invalid;
98
+ }
99
+ while (p < end && whitespace(*p)) ++p;
100
+ if (p != end) goto invalid;
101
+ return 1;
102
+ invalid:
103
+ sqlite3_free(v->owned);
104
+ v->owned = NULL;
105
+ return 0;
106
+ }
107
+
108
+ static float element(const Vector *v, int i) {
109
+ if (!v->bytes) return v->owned[i];
110
+ const unsigned char *p = v->bytes + (size_t)i * 4;
111
+ uint32_t bits = (uint32_t)p[0] | ((uint32_t)p[1] << 8) |
112
+ ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24);
113
+ float value;
114
+ memcpy(&value, &bits, sizeof(value));
115
+ return value;
116
+ }
117
+
118
+ static int score(const Vector *a, const Vector *b, int metric, double *out) {
119
+ if (!a->count || a->count != b->count) return 0;
120
+ double dot = 0, norm_a = 0, norm_b = 0, squared = 0;
121
+ int i = 0;
122
+ #if defined(__SSE2__)
123
+ /* SSE2 is baseline on x86-64. Accumulate pairs in double precision; element()
124
+ * handles unaligned BLOB bytes and little-endian decoding before packing. */
125
+ if (a->bytes && b->bytes) {
126
+ __m128d d = _mm_setzero_pd(), na = d, nb = d, sq = d;
127
+ for (; i + 1 < a->count; i += 2) {
128
+ float af[2] = { element(a, i), element(a, i + 1) };
129
+ float bf[2] = { element(b, i), element(b, i + 1) };
130
+ if (!isfinite(af[0]) || !isfinite(af[1]) || !isfinite(bf[0]) || !isfinite(bf[1])) return 0;
131
+ __m128d av = _mm_set_pd(af[1], af[0]), bv = _mm_set_pd(bf[1], bf[0]);
132
+ if (metric == 1) {
133
+ __m128d delta = _mm_sub_pd(av, bv);
134
+ sq = _mm_add_pd(sq, _mm_mul_pd(delta, delta));
135
+ } else {
136
+ d = _mm_add_pd(d, _mm_mul_pd(av, bv));
137
+ if (metric == 0) {
138
+ na = _mm_add_pd(na, _mm_mul_pd(av, av));
139
+ nb = _mm_add_pd(nb, _mm_mul_pd(bv, bv));
140
+ }
141
+ }
142
+ }
143
+ double lanes[2];
144
+ _mm_storeu_pd(lanes, d); dot = lanes[0] + lanes[1];
145
+ _mm_storeu_pd(lanes, na); norm_a = lanes[0] + lanes[1];
146
+ _mm_storeu_pd(lanes, nb); norm_b = lanes[0] + lanes[1];
147
+ _mm_storeu_pd(lanes, sq); squared = lanes[0] + lanes[1];
148
+ }
149
+ #endif
150
+ for (; i < a->count; ++i) {
151
+ double av = element(a, i), bv = element(b, i);
152
+ if (!isfinite(av) || !isfinite(bv)) return 0;
153
+ if (metric == 1) { double delta = av - bv; squared += delta * delta; }
154
+ else {
155
+ dot += av * bv;
156
+ if (metric == 0) { norm_a += av * av; norm_b += bv * bv; }
157
+ }
158
+ }
159
+ *out = metric == 1 ? sqrt(squared) : metric == 2 ? -dot :
160
+ norm_a && norm_b ? 1.0 - dot / (sqrt(norm_a) * sqrt(norm_b)) : 1.0;
161
+ return 1;
162
+ }
163
+
164
+ static void distance(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
165
+ (void)argc;
166
+ Vector left;
167
+ int status = read_vector(argv[0], &left, 0);
168
+ if (status != 1) { if (status < 0) sqlite3_result_error_nomem(ctx); return; }
169
+ Vector *right = sqlite3_get_auxdata(ctx, 1);
170
+ int fresh = !right;
171
+ if (fresh) {
172
+ right = sqlite3_malloc64(sizeof(*right));
173
+ if (!right) { sqlite3_free(left.owned); sqlite3_result_error_nomem(ctx); return; }
174
+ status = read_vector(argv[1], right, 1);
175
+ if (status != 1) {
176
+ release_vector(right); sqlite3_free(left.owned);
177
+ if (status < 0) sqlite3_result_error_nomem(ctx);
178
+ return;
179
+ }
180
+ }
181
+ double value;
182
+ if (score(&left, right, (int)(intptr_t)sqlite3_user_data(ctx), &value)) sqlite3_result_double(ctx, value);
183
+ sqlite3_free(left.owned);
184
+ /* set_auxdata may immediately destroy the value: nothing accesses it after this. */
185
+ if (fresh) sqlite3_set_auxdata(ctx, 1, right, release_vector);
186
+ }
187
+
188
+ static void version(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
189
+ (void)argc; (void)argv;
190
+ sqlite3_result_text(ctx, "an5-vector/1", -1, SQLITE_STATIC);
191
+ }
192
+
193
+ #ifdef _WIN32
194
+ __declspec(dllexport)
195
+ #endif
196
+ int sqlite3_extension_init(sqlite3 *db, char **error, const sqlite3_api_routines *api) {
197
+ (void)error;
198
+ SQLITE_EXTENSION_INIT2(api);
199
+ const char *names[] = { "an5_vec_cosine", "an5_vec_l2", "an5_vec_ip" };
200
+ int flags = SQLITE_UTF8 | SQLITE_DETERMINISTIC;
201
+ #ifdef SQLITE_INNOCUOUS
202
+ flags |= SQLITE_INNOCUOUS;
203
+ #endif
204
+ for (int i = 0; i < 3; ++i) {
205
+ int rc = sqlite3_create_function_v2(db, names[i], 2, flags, (void *)(intptr_t)i, distance, NULL, NULL, NULL);
206
+ if (rc != SQLITE_OK) return rc;
207
+ }
208
+ return sqlite3_create_function_v2(db, "an5_vector_version", 0, flags, NULL, version, NULL, NULL, NULL);
209
+ }