@an5/adapters 0.2.11 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -5
- package/dist/an5Adapter.d.ts +42 -1
- package/dist/an5Adapter.js +146 -38
- package/dist/base/types.d.ts +14 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +8 -1
- package/dist/sqlite/browserEngine.d.ts +13 -0
- package/dist/sqlite/browserEngine.js +13 -0
- package/dist/sqlite/engine.d.ts +5 -0
- package/dist/sqlite/engine.js +10 -0
- package/dist/sqlite/index.d.ts +1 -0
- package/dist/sqlite/index.js +1 -0
- package/dist/sqlite/vector.d.ts +155 -0
- package/dist/sqlite/vector.js +432 -0
- package/dotnet/An5Adapter.cs +99 -28
- package/dotnet/Base/Core.cs +14 -0
- package/dotnet/Mssql/MssqlEngine.cs +44 -1
- package/dotnet/Postgres/PostgresEngine.cs +2 -0
- package/dotnet/Sqlite/SqliteEngine.cs +82 -2
- package/dotnet/Sqlite/SqliteVectors.cs +401 -0
- package/golang/an5_adapter.go +84 -32
- package/golang/base/vectors.go +263 -0
- package/golang/sqlite/vector.go +248 -0
- package/golang/sqlite_vector_test.go +288 -0
- package/golang/table_client.go +67 -3
- package/java/pom.xml +177 -0
- package/java/src/main/java/an5/adapters/An5Adapter.java +374 -0
- package/java/src/main/java/an5/adapters/An5Aggregate.java +116 -0
- package/java/src/main/java/an5/adapters/An5GroupBy.java +97 -0
- package/java/src/main/java/an5/adapters/An5Query.java +100 -0
- package/java/src/main/java/an5/adapters/An5TableClient.java +1030 -0
- package/java/src/main/java/an5/adapters/An5ViewClient.java +107 -0
- package/java/src/main/java/an5/adapters/base/Dialect.java +103 -0
- package/java/src/main/java/an5/adapters/base/FieldMeta.java +42 -0
- package/java/src/main/java/an5/adapters/base/JdbcUrls.java +201 -0
- package/java/src/main/java/an5/adapters/base/Metadata.java +195 -0
- package/java/src/main/java/an5/adapters/base/RelationDef.java +26 -0
- package/java/src/main/java/an5/adapters/base/SqlBuilder.java +422 -0
- package/java/src/main/java/an5/adapters/base/SqliteVectors.java +375 -0
- package/java/src/main/java/an5/adapters/base/Vectors.java +123 -0
- package/java/src/test/java/An5AdaptersSmoke.java +733 -0
- package/java/src/test/java/Json.java +174 -0
- package/java/src/test/java/QuerySemantics.java +111 -0
- package/kotlin/build.gradle.kts +145 -0
- package/kotlin/settings.gradle.kts +10 -0
- package/kotlin/src/main/kotlin/an5/adapters/Aggregate.kt +89 -0
- package/kotlin/src/main/kotlin/an5/adapters/An5.kt +91 -0
- package/kotlin/src/main/kotlin/an5/adapters/Query.kt +130 -0
- package/kotlin/src/main/kotlin/an5/adapters/Rows.kt +166 -0
- package/kotlin/src/main/kotlin/an5/adapters/TableClient.kt +169 -0
- package/kotlin/src/main/kotlin/an5/adapters/Types.kt +77 -0
- package/kotlin/src/test/kotlin/an5/adapters/An5Smoke.kt +329 -0
- package/native/sqlite/CMakeLists.txt +12 -0
- package/native/sqlite/README.md +117 -0
- package/native/sqlite/an5_vector.c +209 -0
- package/package.json +33 -7
- package/python/an5_adapter.py +37 -6
- package/python/base/__init__.py +15 -0
- package/python/base/vectors.py +431 -0
- package/python/sqlite/__init__.py +2 -2
- package/python/sqlite/provider.py +21 -1
- package/python/table_client.py +129 -59
- package/rust/src/adapter.rs +342 -24
- package/rust/src/base/mod.rs +7 -0
- package/rust/src/base/vectors.rs +293 -0
- package/rust/src/lib.rs +3 -3
- package/scripts/sqlite-native-build.js +32 -0
- package/swift/Package.swift +22 -0
- package/swift/Sources/An5Adapters/An5Adapter.swift +160 -0
- package/swift/Sources/An5Adapters/Metadata.swift +183 -0
- package/swift/Sources/An5Adapters/Query.swift +105 -0
- package/swift/Sources/An5Adapters/Row+Accessors.swift +129 -0
- package/swift/Sources/An5Adapters/SQLBuilder.swift +359 -0
- package/swift/Sources/An5Adapters/SQLiteDriver.swift +301 -0
- package/swift/Sources/An5Adapters/SqliteVectors.swift +243 -0
- package/swift/Sources/An5Adapters/TableClient.swift +671 -0
- package/swift/Sources/An5Adapters/Types.swift +127 -0
- package/swift/Sources/An5Adapters/Vectors.swift +86 -0
- package/swift/Sources/An5Adapters/ViewClient.swift +50 -0
- package/swift/Sources/CSQLite/module.modulemap +8 -0
- package/swift/Sources/CSQLite/shim.h +9 -0
- package/swift/Tests/An5AdaptersTests/QuerySemanticsTests.swift +51 -0
- package/swift/Tests/An5AdaptersTests/RuntimeSmokeTests.swift +488 -0
- package/typescript/src/an5Adapter.ts +155 -31
- package/typescript/src/base/types.ts +15 -0
- package/typescript/src/index.ts +13 -0
- package/typescript/src/sqlite/browserEngine.ts +24 -0
- package/typescript/src/sqlite/engine.ts +13 -0
- package/typescript/src/sqlite/index.ts +1 -1
- package/typescript/src/sqlite/vector.ts +537 -0
- package/golang/sqlite/sqlite.go +0 -14
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
package an5.adapters
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* End-to-end exercise of the Kotlin runtime against a real in-memory SQLite database.
|
|
5
|
+
*
|
|
6
|
+
* Covers what a Java-only check cannot: that Kotlin's read-only maps reach the JDBC layer,
|
|
7
|
+
* that the query block builds the filter the other adapters build, and that a column read
|
|
8
|
+
* as `Long` from SQLite comes out of the typed accessors as `Int` without a cast at the
|
|
9
|
+
* call site.
|
|
10
|
+
*/
|
|
11
|
+
object An5Smoke {
|
|
12
|
+
|
|
13
|
+
private var failures = 0
|
|
14
|
+
|
|
15
|
+
@JvmStatic
|
|
16
|
+
fun main(args: Array<String>) {
|
|
17
|
+
crud()
|
|
18
|
+
filters()
|
|
19
|
+
aggregates()
|
|
20
|
+
relations()
|
|
21
|
+
transactions()
|
|
22
|
+
vectorFallback()
|
|
23
|
+
sqliteVectorSearch()
|
|
24
|
+
|
|
25
|
+
if (failures > 0) {
|
|
26
|
+
println("kotlin smoke: $failures check(s) failed")
|
|
27
|
+
kotlin.system.exitProcess(1)
|
|
28
|
+
}
|
|
29
|
+
println("kotlin smoke: all checks passed")
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
private fun crud() {
|
|
33
|
+
val db = open()
|
|
34
|
+
db.use {
|
|
35
|
+
val users = it.table("User")
|
|
36
|
+
val created = users.create(mapOf("name" to "Ada", "score" to 10))
|
|
37
|
+
check("generated primary key") { created.string("id") != null }
|
|
38
|
+
check("create returns the stored row") { created.string("name") == "Ada" }
|
|
39
|
+
check("row accessors read a column") { created.int("score") == 10 }
|
|
40
|
+
check("row accessors widen a number") { created.long("score") == 10L }
|
|
41
|
+
check("missing column reads as the fallback") { created.int("nope", -1) == -1 }
|
|
42
|
+
check("null column reads as the fallback") { created.string("nope") == null }
|
|
43
|
+
|
|
44
|
+
users.create(mapOf("name" to "Grace", "score" to 20))
|
|
45
|
+
users.create(mapOf("name" to "Alan", "score" to 30))
|
|
46
|
+
|
|
47
|
+
check("count") { users.count() == 3L }
|
|
48
|
+
check("ordered results") { users.findMany { orderBy("score", Sort.DESC) }.first().string("name") == "Alan" }
|
|
49
|
+
check("paged results") {
|
|
50
|
+
users.findMany { orderBy("score"); skip(1); take(1) }.single().string("name") == "Grace"
|
|
51
|
+
}
|
|
52
|
+
check("select projects columns") {
|
|
53
|
+
users.findMany { select("name") }.all { it.keys == setOf("name") }
|
|
54
|
+
}
|
|
55
|
+
check("update") {
|
|
56
|
+
users.update(mapOf("id" to created.string("id")), mapOf("score" to 99))?.int("score") == 99
|
|
57
|
+
}
|
|
58
|
+
check("updateMany") {
|
|
59
|
+
users.updateMany(null, mapOf("score" to mapOf("increment" to 1))) == 3
|
|
60
|
+
}
|
|
61
|
+
check("upsert updates") {
|
|
62
|
+
users.upsert(mapOf("id" to created.string("id")), mapOf("name" to "x"), mapOf("name" to "Ada L"))
|
|
63
|
+
.string("name") == "Ada L"
|
|
64
|
+
}
|
|
65
|
+
check("upsert creates") {
|
|
66
|
+
users.upsert(mapOf("name" to "Katherine"), mapOf("name" to "Katherine"), mapOf("name" to "K"))
|
|
67
|
+
.string("name") == "Katherine"
|
|
68
|
+
}
|
|
69
|
+
check("delete") { users.delete(mapOf("id" to created.string("id"))) != null }
|
|
70
|
+
check("deleteMany") { users.deleteMany(null) == 3 }
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
private fun filters() {
|
|
75
|
+
val db = open()
|
|
76
|
+
db.use {
|
|
77
|
+
val users = it.table("User")
|
|
78
|
+
users.createMany(
|
|
79
|
+
listOf(
|
|
80
|
+
mapOf("name" to "Ada", "score" to 10),
|
|
81
|
+
mapOf("name" to "Grace", "score" to 20),
|
|
82
|
+
mapOf("name" to "Alan", "score" to 30),
|
|
83
|
+
)
|
|
84
|
+
)
|
|
85
|
+
check("gte") { users.findMany { where("score" gte 20) }.size == 2 }
|
|
86
|
+
check("lt") { users.findMany { where("score" lt 20) }.size == 1 }
|
|
87
|
+
check("eq") { users.findMany { where("name" eq "Ada") }.size == 1 }
|
|
88
|
+
check("neq") { users.findMany { where("name" neq "Ada") }.size == 2 }
|
|
89
|
+
check("contains") { users.findMany { where("name" contains "da") }.size == 1 }
|
|
90
|
+
check("endsWith") { users.findMany { where("name" endsWith "ce") }.size == 1 }
|
|
91
|
+
check("inList") { users.findMany { where("name" inList listOf("Ada", "Alan")) }.size == 2 }
|
|
92
|
+
check("empty inList matches nothing") {
|
|
93
|
+
users.findMany { where("name" inList emptyList<String>()) }.isEmpty()
|
|
94
|
+
}
|
|
95
|
+
check("empty notInList matches everything") {
|
|
96
|
+
users.findMany { where("name" notInList emptyList<String>()) }.size == 3
|
|
97
|
+
}
|
|
98
|
+
check("orOf") { users.findMany { where(orOf("name" eq "Ada", "name" eq "Alan")) }.size == 2 }
|
|
99
|
+
check("andOf") { users.findMany { where(andOf("score" gte 10, "score" lt 30)) }.size == 2 }
|
|
100
|
+
check("notOf") { users.findMany { where(notOf("name" eq "Ada", "name" eq "Grace")) }.size == 1 }
|
|
101
|
+
check("empty orOf matches nothing") { users.findMany { where(orOf()) }.isEmpty() }
|
|
102
|
+
check("two where calls combine with AND") {
|
|
103
|
+
users.findMany { where("score" gte 10); where("score" lte 20) }.size == 2
|
|
104
|
+
}
|
|
105
|
+
check("isNull") { users.findMany { where("nickname".isNull()) }.size == 3 }
|
|
106
|
+
check("isNotNull") { users.findMany { where("name".isNotNull()) }.size == 3 }
|
|
107
|
+
check("findFirst") { users.findFirst(mapOf("name" to "Ada"))?.int("score") == 10 }
|
|
108
|
+
check("findFirst misses") { users.findFirst(mapOf("name" to "Nobody")) == null }
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
private fun aggregates() {
|
|
113
|
+
val db = open()
|
|
114
|
+
db.use {
|
|
115
|
+
val users = it.table("User")
|
|
116
|
+
users.createMany(
|
|
117
|
+
listOf(
|
|
118
|
+
mapOf("name" to "Ada", "score" to 10),
|
|
119
|
+
mapOf("name" to "Grace", "score" to 20),
|
|
120
|
+
mapOf("name" to "Alan", "score" to 30),
|
|
121
|
+
)
|
|
122
|
+
)
|
|
123
|
+
val totals = users.aggregate { count(); sum("score"); avg("score"); max("score") }
|
|
124
|
+
check("aggregate count") { (totals["_count"] as Number).toLong() == 3L }
|
|
125
|
+
check("aggregate sum") { (totals["_sum_score"] as Number).toDouble() == 60.0 }
|
|
126
|
+
check("aggregate avg") { (totals["_avg_score"] as Number).toDouble() == 20.0 }
|
|
127
|
+
check("aggregate max") { (totals["_max_score"] as Number).toDouble() == 30.0 }
|
|
128
|
+
|
|
129
|
+
val filtered = users.aggregate { count(); where("score" gte 20) }
|
|
130
|
+
check("aggregate where") { (filtered["_count"] as Number).toLong() == 2L }
|
|
131
|
+
|
|
132
|
+
val groups = users.groupBy("name") { sum("score") }
|
|
133
|
+
check("groupBy rows") { groups.size == 3 }
|
|
134
|
+
check("groupBy counts") { groups.all { (it["_count"] as Number).toLong() == 1L } }
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
private fun relations() {
|
|
139
|
+
val db = open()
|
|
140
|
+
db.use {
|
|
141
|
+
val ada = it.table("User").create(mapOf("name" to "Ada", "score" to 1))
|
|
142
|
+
it.table("Post").create(mapOf("title" to "first", "userId" to ada.string("id")))
|
|
143
|
+
it.table("Post").create(mapOf("title" to "second", "userId" to ada.string("id")))
|
|
144
|
+
|
|
145
|
+
val rows = it.table("User").findMany {
|
|
146
|
+
include(mapOf("posts" to true, "_count" to true))
|
|
147
|
+
}
|
|
148
|
+
check("many relation eager-loaded") { rows.single().related("posts").size == 2 }
|
|
149
|
+
check("relation count eager-loaded") { rows.single().relationCount("posts") == 2 }
|
|
150
|
+
check("related rows are readable") {
|
|
151
|
+
rows.single().related("posts").first().string("title") == "first"
|
|
152
|
+
}
|
|
153
|
+
check("missing relation reads as empty") { rows.single().related("comments").isEmpty() }
|
|
154
|
+
check("missing relation count reads as zero") { rows.single().relationCount("comments") == 0 }
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
private fun transactions() {
|
|
159
|
+
val db = open()
|
|
160
|
+
db.use {
|
|
161
|
+
it.table("User").create(mapOf("name" to "outside"))
|
|
162
|
+
val failed = runCatching {
|
|
163
|
+
it.transaction { scoped ->
|
|
164
|
+
scoped.table("User").create(mapOf("name" to "inside"))
|
|
165
|
+
error("rollback")
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
check("transaction rethrows") { failed.isFailure && failed.exceptionOrNull()?.message == "rollback" }
|
|
169
|
+
check("rollback discarded the inner insert") { it.table("User").count() == 1L }
|
|
170
|
+
|
|
171
|
+
it.transaction { scoped -> scoped.table("User").create(mapOf("name" to "committed")) }
|
|
172
|
+
check("commit kept the insert") { it.table("User").count() == 2L }
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
private fun vectorFallback() {
|
|
177
|
+
val db = open()
|
|
178
|
+
db.use {
|
|
179
|
+
val docs = it.table("Document")
|
|
180
|
+
docs.create(mapOf("title" to "far", "embedding" to "[0.0, 1.0]"))
|
|
181
|
+
docs.create(mapOf("title" to "near", "embedding" to "[1.0, 0.1]"))
|
|
182
|
+
docs.create(mapOf("title" to "no vector", "embedding" to null))
|
|
183
|
+
|
|
184
|
+
val hits = docs.vectorSearch(doubleArrayOf(1.0, 0.0), take = 2)
|
|
185
|
+
check("rows without a vector are skipped") { hits.size == 2 }
|
|
186
|
+
check("nearest first") { hits.first().string("title") == "near" }
|
|
187
|
+
check("distance reported") { hits.first().containsKey("distance") }
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* SQLite ranks a `VECTOR(n)` column inside the database, and a `DoubleArray` written
|
|
193
|
+
* to one comes back as numbers.
|
|
194
|
+
*
|
|
195
|
+
* The docstring in the JVM adapter's `SqliteVectors` is the shared specification every
|
|
196
|
+
* runtime implements; test/sqlite-vector.test.js (TypeScript) is the mirror.
|
|
197
|
+
*/
|
|
198
|
+
private fun sqliteVectorSearch() {
|
|
199
|
+
an5.adapters.base.Metadata.setAdapterMetadata(blobMetadata())
|
|
200
|
+
val db = An5(":memory:")
|
|
201
|
+
db.use {
|
|
202
|
+
it.executeRaw("CREATE TABLE documents (id TEXT PRIMARY KEY, title TEXT, embedding BLOB)")
|
|
203
|
+
val vectors = listOf(
|
|
204
|
+
"d1" to doubleArrayOf(1.0, 0.0, 0.0),
|
|
205
|
+
"d2" to doubleArrayOf(0.8, 0.2, 0.0),
|
|
206
|
+
"d3" to doubleArrayOf(0.0, 1.0, 0.0),
|
|
207
|
+
)
|
|
208
|
+
for ((id, vector) in vectors) {
|
|
209
|
+
it.executeRaw(
|
|
210
|
+
"INSERT INTO documents (id, title, embedding) VALUES (?, ?, ?)",
|
|
211
|
+
id, id, an5.adapters.base.SqliteVectors.encodeVector(vector),
|
|
212
|
+
)
|
|
213
|
+
}
|
|
214
|
+
// A row with another dimension must never rank against a 3-dimension query.
|
|
215
|
+
it.executeRaw(
|
|
216
|
+
"INSERT INTO documents (id, title, embedding) VALUES ('d4', 'd4', ?)",
|
|
217
|
+
an5.adapters.base.SqliteVectors.encodeVector(doubleArrayOf(1.0, 0.0, 0.0, 1.0)),
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
val docs = it.table("Document")
|
|
221
|
+
for (metric in DistanceMetric.entries) {
|
|
222
|
+
val hits = docs.vectorSearch(doubleArrayOf(1.0, 0.0, 0.0), take = 9, metric = metric)
|
|
223
|
+
check("${metric.sql}: row order") { hits.joinToString(",") { row -> row.string("title").orEmpty() } == "d1,d2,d3" }
|
|
224
|
+
check("${metric.sql}: the distance is a number") { hits.first()["distance"] is Double }
|
|
225
|
+
check("${metric.sql}: the vector decodes to numbers") { hits.first()["embedding"] is DoubleArray }
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// `memory` pins the client-side fallback, which ranks the same rows.
|
|
229
|
+
it.vectorStrategy("memory")
|
|
230
|
+
val pinned = docs.vectorSearch(doubleArrayOf(1.0, 0.0, 0.0), take = 9)
|
|
231
|
+
check("the memory fallback ranks correctly") {
|
|
232
|
+
pinned.joinToString(",") { row -> row.string("title").orEmpty() } == "d1,d2,d3"
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
it.vectorStrategy(null)
|
|
236
|
+
val written = docs.create(
|
|
237
|
+
mapOf("id" to "d9", "title" to "written", "embedding" to doubleArrayOf(0.25, 0.5, 1.0)),
|
|
238
|
+
)
|
|
239
|
+
check("create writes a float32 BLOB") {
|
|
240
|
+
it.queryRaw("SELECT typeof(embedding) AS t FROM documents WHERE id = 'd9'")
|
|
241
|
+
.first().string("t") == "blob"
|
|
242
|
+
}
|
|
243
|
+
check("create returns numbers") { written["embedding"] is DoubleArray }
|
|
244
|
+
check("findMany returns numbers") {
|
|
245
|
+
docs.findMany(where = mapOf("id" to "d9")).first()["embedding"] is DoubleArray
|
|
246
|
+
}
|
|
247
|
+
docs.update(mapOf("id" to "d1"), mapOf("embedding" to doubleArrayOf(0.5, 0.5, 0.0)))
|
|
248
|
+
check("update encodes") {
|
|
249
|
+
docs.findMany(where = mapOf("id" to "d1")).first()["embedding"] is DoubleArray
|
|
250
|
+
}
|
|
251
|
+
docs.updateMany(mapOf("id" to "d2"), mapOf("embedding" to doubleArrayOf(0.0, 0.0, 1.0)))
|
|
252
|
+
check("updateMany encodes") {
|
|
253
|
+
docs.findMany(where = mapOf("id" to "d2")).first()["embedding"] is DoubleArray
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
private fun blobMetadata(): MutableMap<String, Any?> {
|
|
259
|
+
val metadata = metadata()
|
|
260
|
+
@Suppress("UNCHECKED_CAST")
|
|
261
|
+
val fields = metadata["modelFields"] as MutableMap<String, Any?>
|
|
262
|
+
fields["Document"] = listOf(
|
|
263
|
+
field("id", true),
|
|
264
|
+
field("title", false),
|
|
265
|
+
mutableMapOf<String, Any?>(
|
|
266
|
+
"name" to "embedding",
|
|
267
|
+
"type" to "number[] | string",
|
|
268
|
+
"sql" to "VECTOR(3)",
|
|
269
|
+
"isOptional" to true,
|
|
270
|
+
"hasDefault" to false,
|
|
271
|
+
"isId" to false,
|
|
272
|
+
),
|
|
273
|
+
)
|
|
274
|
+
return metadata
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
// ─── Harness ──────────────────────────────────────────────────────────────────
|
|
278
|
+
|
|
279
|
+
private fun open(): An5 {
|
|
280
|
+
an5.adapters.base.Metadata.setAdapterMetadata(metadata())
|
|
281
|
+
val db = An5(":memory:")
|
|
282
|
+
db.executeRaw("CREATE TABLE users (id TEXT PRIMARY KEY, name TEXT, score INTEGER, nickname TEXT)")
|
|
283
|
+
db.executeRaw("CREATE TABLE documents (id TEXT PRIMARY KEY, title TEXT, embedding TEXT)")
|
|
284
|
+
db.executeRaw("CREATE TABLE posts (id TEXT PRIMARY KEY, title TEXT, userId TEXT)")
|
|
285
|
+
return db
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
private fun metadata(): MutableMap<String, Any?> {
|
|
289
|
+
val posts = mutableMapOf<String, Any?>(
|
|
290
|
+
"modelName" to "Post",
|
|
291
|
+
"relationType" to "many",
|
|
292
|
+
"foreignKey" to "userId",
|
|
293
|
+
"localKey" to "id",
|
|
294
|
+
)
|
|
295
|
+
return mutableMapOf(
|
|
296
|
+
"modelToTable" to mutableMapOf(
|
|
297
|
+
"User" to "users",
|
|
298
|
+
"Post" to "posts",
|
|
299
|
+
"Document" to "documents",
|
|
300
|
+
),
|
|
301
|
+
"modelFields" to mutableMapOf(
|
|
302
|
+
"User" to listOf(field("id", true), field("name", false), field("score", false)),
|
|
303
|
+
"Post" to listOf(field("id", true), field("title", false), field("userId", false)),
|
|
304
|
+
"Document" to listOf(field("id", true), field("title", false), field("embedding", false)),
|
|
305
|
+
),
|
|
306
|
+
"relationMap" to mutableMapOf("User" to mutableMapOf("posts" to posts)),
|
|
307
|
+
)
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
private fun field(name: String, id: Boolean): MutableMap<String, Any?> = mutableMapOf(
|
|
311
|
+
"name" to name,
|
|
312
|
+
"type" to "string",
|
|
313
|
+
"sql" to "NVARCHAR(255)",
|
|
314
|
+
"isOptional" to false,
|
|
315
|
+
"hasDefault" to false,
|
|
316
|
+
"isId" to id,
|
|
317
|
+
)
|
|
318
|
+
|
|
319
|
+
private fun check(what: String, assertion: () -> Boolean) {
|
|
320
|
+
val outcome = runCatching(assertion)
|
|
321
|
+
when {
|
|
322
|
+
outcome.isSuccess && outcome.getOrNull() == true -> println("ok $what")
|
|
323
|
+
else -> {
|
|
324
|
+
failures++
|
|
325
|
+
println("FAIL $what: ${outcome.exceptionOrNull()?.message ?: "assertion returned false"}")
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
cmake_minimum_required(VERSION 3.16)
|
|
2
|
+
project(an5_vector LANGUAGES C)
|
|
3
|
+
find_path(SQLITE_INCLUDE_DIR sqlite3ext.h REQUIRED)
|
|
4
|
+
add_library(an5_vector MODULE an5_vector.c)
|
|
5
|
+
target_include_directories(an5_vector PRIVATE "${SQLITE_INCLUDE_DIR}")
|
|
6
|
+
set_target_properties(an5_vector PROPERTIES PREFIX "" C_STANDARD 99)
|
|
7
|
+
if(MSVC)
|
|
8
|
+
target_compile_options(an5_vector PRIVATE /O2 /W4)
|
|
9
|
+
else()
|
|
10
|
+
target_compile_options(an5_vector PRIVATE -O3 -Wall -Wextra -Werror)
|
|
11
|
+
target_link_libraries(an5_vector PRIVATE m)
|
|
12
|
+
endif()
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# AN5 native SQLite vectors
|
|
2
|
+
|
|
3
|
+
`an5_vector.c` is a loadable SQLite extension, not a language-driver callback.
|
|
4
|
+
It implements all three AN5 metrics in C:
|
|
5
|
+
|
|
6
|
+
| SQL function | Result (smaller is closer) |
|
|
7
|
+
|---|---|
|
|
8
|
+
| `an5_vec_cosine(left, right)` | `1 - cosine_similarity` |
|
|
9
|
+
| `an5_vec_l2(left, right)` | Euclidean distance |
|
|
10
|
+
| `an5_vec_ip(left, right)` | Negative dot product |
|
|
11
|
+
| `an5_vector_version()` | Extension identifier |
|
|
12
|
+
|
|
13
|
+
BLOB inputs are little-endian float32. JSON text arrays are also accepted and
|
|
14
|
+
converted to float32, so native results have single-precision input values even
|
|
15
|
+
for legacy text columns. Empty, malformed, non-finite and dimension-mismatched
|
|
16
|
+
inputs return SQL NULL. Zero vectors have cosine distance 1.
|
|
17
|
+
|
|
18
|
+
The BLOB path reads the stored bytes directly without allocating a row vector.
|
|
19
|
+
The query operand is cached using SQLite auxdata for the statement. x86-64 uses
|
|
20
|
+
SSE2 with double-precision accumulators; other architectures use the portable C
|
|
21
|
+
loop. No `-march=native`, AVX dependency or fast-math flags are required.
|
|
22
|
+
|
|
23
|
+
This is **exact scanning**, not an ANN index. A normal relational index on a
|
|
24
|
+
filter column can narrow the candidates before distance calculation. Only the
|
|
25
|
+
top results are returned to the language runtime.
|
|
26
|
+
|
|
27
|
+
## Build
|
|
28
|
+
|
|
29
|
+
In this workspace:
|
|
30
|
+
|
|
31
|
+
```sh
|
|
32
|
+
npm run build:sqlite:native -w an5Adapters
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
The command prints the absolute library path under `native/sqlite/build`.
|
|
36
|
+
Linux/macOS need a C compiler and `sqlite3ext.h`; on Debian/Ubuntu install
|
|
37
|
+
`build-essential` and `libsqlite3-dev`. `CC` and `SQLITE_INCLUDE_DIR` may be set
|
|
38
|
+
when using a different compiler or header directory.
|
|
39
|
+
|
|
40
|
+
For an installed npm package:
|
|
41
|
+
|
|
42
|
+
```sh
|
|
43
|
+
node node_modules/@an5/adapters/scripts/sqlite-native-build.js ./native-build
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Windows builds through CMake and an installed C toolchain. The CMake project can
|
|
47
|
+
also be used directly on other platforms:
|
|
48
|
+
|
|
49
|
+
```sh
|
|
50
|
+
cmake -S native/sqlite -B native/sqlite/build -DCMAKE_BUILD_TYPE=Release
|
|
51
|
+
cmake --build native/sqlite/build --config Release
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Build for the target OS and architecture. No precompiled binary is downloaded
|
|
55
|
+
or compiled during the package's ordinary install.
|
|
56
|
+
|
|
57
|
+
## Load
|
|
58
|
+
|
|
59
|
+
The existing extension-path option accepts either sqlite-vec or this library:
|
|
60
|
+
|
|
61
|
+
```typescript
|
|
62
|
+
const db = createAn5Adapter({
|
|
63
|
+
connectionString: 'sqlite:///app.db',
|
|
64
|
+
sqliteVec: '/absolute/path/an5_vector.so', // .dylib / .dll on other platforms
|
|
65
|
+
});
|
|
66
|
+
await db.document.vectorSearch({ vector: query, take: 10 });
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
db = create_an5_adapter('sqlite:///app.db', sqlite_vec='/absolute/path/an5_vector.so')
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
```csharp
|
|
74
|
+
var db = new An5Adapter(new An5AdapterOptions {
|
|
75
|
+
ConnectionString = "app.db",
|
|
76
|
+
SqliteVec = "/absolute/path/an5_vector.so"
|
|
77
|
+
});
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
TypeScript, Python, .NET and Swift recognize `an5_vector_version()` before
|
|
81
|
+
registering callbacks, preserving the native functions. In the existing strategy
|
|
82
|
+
API this uses `udf`; **the functions execute in C**, not in the host language.
|
|
83
|
+
On runtimes that take an existing connection/pool, load the extension on every
|
|
84
|
+
SQLite connection through the driver's extension-loading API. Go, Rust and JDBC
|
|
85
|
+
then discover the `an5_vec_*` functions through their existing probes. Swift's
|
|
86
|
+
loading hook is desktop-only; mobile platforms need a statically linked SQLite
|
|
87
|
+
extension/build supplied by the application.
|
|
88
|
+
|
|
89
|
+
Each gate builds the extension itself and checks that the native functions are
|
|
90
|
+
still the ones answering: `test:sqlite:native` covers TypeScript, Python and
|
|
91
|
+
(optionally) .NET, while `test:swift` builds the extension and runs the same
|
|
92
|
+
check through the Swift driver.
|
|
93
|
+
|
|
94
|
+
## Verify and benchmark
|
|
95
|
+
|
|
96
|
+
```sh
|
|
97
|
+
npm run test:sqlite:native -w an5Adapters
|
|
98
|
+
node an5Adapters/scripts/sqlite-native-test.js --dotnet
|
|
99
|
+
AN5_NATIVE_SANITIZE=1 npm run test:sqlite:native -w an5Adapters
|
|
100
|
+
npm run bench:sqlite:native -w an5Adapters
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
The native test compiles into a disposable directory, tests the extension with
|
|
104
|
+
real SQLite, checks all metrics against the reference implementation, and verifies
|
|
105
|
+
that TypeScript and Python keep the C functions. `--dotnet` additionally tests
|
|
106
|
+
.NET. The sanitizer option enables UBSan on supporting Unix compilers.
|
|
107
|
+
|
|
108
|
+
`npm run test:swift -w an5Adapters` builds the extension too and asserts the
|
|
109
|
+
Swift driver keeps it; that case is skipped when `AN5_NATIVE_VECTOR_PATH` is
|
|
110
|
+
unset, as on a machine without a C compiler.
|
|
111
|
+
|
|
112
|
+
The benchmark compares the public adapter's native and JavaScript-UDF paths with
|
|
113
|
+
identical BLOB data and verifies equal top-10 IDs. It reports the median of seven
|
|
114
|
+
warmed runs over 4,000 vectors of 256 dimensions. On the development Linux x86-64
|
|
115
|
+
machine the native path measured about 4 ms per search versus 15–18 ms for the
|
|
116
|
+
JavaScript callback (roughly 4x faster). Timings are illustrative, not pass/fail
|
|
117
|
+
thresholds or a performance guarantee.
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/* Portable SQLite extension for float32 vector distances. No driver callbacks.
|
|
2
|
+
* BLOB inputs are little-endian float32; JSON arrays remain readable.
|
|
3
|
+
* Query vectors use SQLite auxdata so parsing/copying happens once per statement.
|
|
4
|
+
*/
|
|
5
|
+
#include <sqlite3ext.h>
|
|
6
|
+
SQLITE_EXTENSION_INIT1
|
|
7
|
+
#include <math.h>
|
|
8
|
+
#include <stdint.h>
|
|
9
|
+
#include <stdlib.h>
|
|
10
|
+
#include <string.h>
|
|
11
|
+
#if defined(__SSE2__)
|
|
12
|
+
#include <emmintrin.h>
|
|
13
|
+
#endif
|
|
14
|
+
|
|
15
|
+
typedef struct {
|
|
16
|
+
const unsigned char *bytes;
|
|
17
|
+
float *owned;
|
|
18
|
+
int count;
|
|
19
|
+
} Vector;
|
|
20
|
+
|
|
21
|
+
static void release_vector(void *ptr) {
|
|
22
|
+
Vector *v = (Vector *)ptr;
|
|
23
|
+
if (v) { sqlite3_free(v->owned); sqlite3_free(v); }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
static int whitespace(unsigned char c) {
|
|
27
|
+
return c == ' ' || c == '\t' || c == '\r' || c == '\n';
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/* Validate JSON number grammar before using strtod, which otherwise accepts
|
|
31
|
+
* non-JSON spellings such as NaN, Infinity and hexadecimal numbers. */
|
|
32
|
+
static const unsigned char *number_end(const unsigned char *p, const unsigned char *end) {
|
|
33
|
+
if (p < end && *p == '-') ++p;
|
|
34
|
+
if (p == end) return NULL;
|
|
35
|
+
if (*p == '0') ++p;
|
|
36
|
+
else {
|
|
37
|
+
if (*p < '1' || *p > '9') return NULL;
|
|
38
|
+
do { ++p; } while (p < end && *p >= '0' && *p <= '9');
|
|
39
|
+
}
|
|
40
|
+
if (p < end && *p == '.') {
|
|
41
|
+
++p;
|
|
42
|
+
if (p == end || *p < '0' || *p > '9') return NULL;
|
|
43
|
+
do { ++p; } while (p < end && *p >= '0' && *p <= '9');
|
|
44
|
+
}
|
|
45
|
+
if (p < end && (*p == 'e' || *p == 'E')) {
|
|
46
|
+
++p;
|
|
47
|
+
if (p < end && (*p == '+' || *p == '-')) ++p;
|
|
48
|
+
if (p == end || *p < '0' || *p > '9') return NULL;
|
|
49
|
+
do { ++p; } while (p < end && *p >= '0' && *p <= '9');
|
|
50
|
+
}
|
|
51
|
+
return p;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/* 1 = vector, 0 = invalid input, -1 = allocation failure. */
|
|
55
|
+
static int read_vector(sqlite3_value *arg, Vector *v, int copy_blob) {
|
|
56
|
+
memset(v, 0, sizeof(*v));
|
|
57
|
+
if (sqlite3_value_type(arg) == SQLITE_BLOB) {
|
|
58
|
+
int size = sqlite3_value_bytes(arg);
|
|
59
|
+
if (!size || size % 4) return 0;
|
|
60
|
+
v->count = size / 4;
|
|
61
|
+
v->bytes = sqlite3_value_blob(arg);
|
|
62
|
+
if (!v->bytes) return -1;
|
|
63
|
+
if (copy_blob) {
|
|
64
|
+
v->owned = sqlite3_malloc64((sqlite3_uint64)size);
|
|
65
|
+
if (!v->owned) return -1;
|
|
66
|
+
memcpy(v->owned, v->bytes, (size_t)size);
|
|
67
|
+
v->bytes = (const unsigned char *)v->owned;
|
|
68
|
+
}
|
|
69
|
+
return 1;
|
|
70
|
+
}
|
|
71
|
+
if (sqlite3_value_type(arg) != SQLITE_TEXT) return 0;
|
|
72
|
+
const unsigned char *p = sqlite3_value_text(arg);
|
|
73
|
+
if (!p) return -1;
|
|
74
|
+
const unsigned char *end = p + sqlite3_value_bytes(arg);
|
|
75
|
+
while (p < end && whitespace(*p)) ++p;
|
|
76
|
+
if (p == end || *p++ != '[') return 0;
|
|
77
|
+
int capacity = 0;
|
|
78
|
+
for (;;) {
|
|
79
|
+
while (p < end && whitespace(*p)) ++p;
|
|
80
|
+
const unsigned char *next = number_end(p, end);
|
|
81
|
+
if (!next) goto invalid;
|
|
82
|
+
char *parsed;
|
|
83
|
+
double n = strtod((const char *)p, &parsed);
|
|
84
|
+
if ((const unsigned char *)parsed != next || !isfinite(n) || !isfinite((float)n)) goto invalid;
|
|
85
|
+
if (v->count == capacity) {
|
|
86
|
+
if (capacity > 0x3fffffff) goto invalid;
|
|
87
|
+
capacity = capacity ? capacity * 2 : 32;
|
|
88
|
+
float *grown = sqlite3_realloc64(v->owned, (sqlite3_uint64)capacity * sizeof(float));
|
|
89
|
+
if (!grown) { sqlite3_free(v->owned); v->owned = NULL; return -1; }
|
|
90
|
+
v->owned = grown;
|
|
91
|
+
}
|
|
92
|
+
v->owned[v->count++] = (float)n;
|
|
93
|
+
p = next;
|
|
94
|
+
while (p < end && whitespace(*p)) ++p;
|
|
95
|
+
if (p == end) goto invalid;
|
|
96
|
+
if (*p == ']') { ++p; break; }
|
|
97
|
+
if (*p++ != ',') goto invalid;
|
|
98
|
+
}
|
|
99
|
+
while (p < end && whitespace(*p)) ++p;
|
|
100
|
+
if (p != end) goto invalid;
|
|
101
|
+
return 1;
|
|
102
|
+
invalid:
|
|
103
|
+
sqlite3_free(v->owned);
|
|
104
|
+
v->owned = NULL;
|
|
105
|
+
return 0;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
static float element(const Vector *v, int i) {
|
|
109
|
+
if (!v->bytes) return v->owned[i];
|
|
110
|
+
const unsigned char *p = v->bytes + (size_t)i * 4;
|
|
111
|
+
uint32_t bits = (uint32_t)p[0] | ((uint32_t)p[1] << 8) |
|
|
112
|
+
((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24);
|
|
113
|
+
float value;
|
|
114
|
+
memcpy(&value, &bits, sizeof(value));
|
|
115
|
+
return value;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
static int score(const Vector *a, const Vector *b, int metric, double *out) {
|
|
119
|
+
if (!a->count || a->count != b->count) return 0;
|
|
120
|
+
double dot = 0, norm_a = 0, norm_b = 0, squared = 0;
|
|
121
|
+
int i = 0;
|
|
122
|
+
#if defined(__SSE2__)
|
|
123
|
+
/* SSE2 is baseline on x86-64. Accumulate pairs in double precision; element()
|
|
124
|
+
* handles unaligned BLOB bytes and little-endian decoding before packing. */
|
|
125
|
+
if (a->bytes && b->bytes) {
|
|
126
|
+
__m128d d = _mm_setzero_pd(), na = d, nb = d, sq = d;
|
|
127
|
+
for (; i + 1 < a->count; i += 2) {
|
|
128
|
+
float af[2] = { element(a, i), element(a, i + 1) };
|
|
129
|
+
float bf[2] = { element(b, i), element(b, i + 1) };
|
|
130
|
+
if (!isfinite(af[0]) || !isfinite(af[1]) || !isfinite(bf[0]) || !isfinite(bf[1])) return 0;
|
|
131
|
+
__m128d av = _mm_set_pd(af[1], af[0]), bv = _mm_set_pd(bf[1], bf[0]);
|
|
132
|
+
if (metric == 1) {
|
|
133
|
+
__m128d delta = _mm_sub_pd(av, bv);
|
|
134
|
+
sq = _mm_add_pd(sq, _mm_mul_pd(delta, delta));
|
|
135
|
+
} else {
|
|
136
|
+
d = _mm_add_pd(d, _mm_mul_pd(av, bv));
|
|
137
|
+
if (metric == 0) {
|
|
138
|
+
na = _mm_add_pd(na, _mm_mul_pd(av, av));
|
|
139
|
+
nb = _mm_add_pd(nb, _mm_mul_pd(bv, bv));
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
double lanes[2];
|
|
144
|
+
_mm_storeu_pd(lanes, d); dot = lanes[0] + lanes[1];
|
|
145
|
+
_mm_storeu_pd(lanes, na); norm_a = lanes[0] + lanes[1];
|
|
146
|
+
_mm_storeu_pd(lanes, nb); norm_b = lanes[0] + lanes[1];
|
|
147
|
+
_mm_storeu_pd(lanes, sq); squared = lanes[0] + lanes[1];
|
|
148
|
+
}
|
|
149
|
+
#endif
|
|
150
|
+
for (; i < a->count; ++i) {
|
|
151
|
+
double av = element(a, i), bv = element(b, i);
|
|
152
|
+
if (!isfinite(av) || !isfinite(bv)) return 0;
|
|
153
|
+
if (metric == 1) { double delta = av - bv; squared += delta * delta; }
|
|
154
|
+
else {
|
|
155
|
+
dot += av * bv;
|
|
156
|
+
if (metric == 0) { norm_a += av * av; norm_b += bv * bv; }
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
*out = metric == 1 ? sqrt(squared) : metric == 2 ? -dot :
|
|
160
|
+
norm_a && norm_b ? 1.0 - dot / (sqrt(norm_a) * sqrt(norm_b)) : 1.0;
|
|
161
|
+
return 1;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
static void distance(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
|
|
165
|
+
(void)argc;
|
|
166
|
+
Vector left;
|
|
167
|
+
int status = read_vector(argv[0], &left, 0);
|
|
168
|
+
if (status != 1) { if (status < 0) sqlite3_result_error_nomem(ctx); return; }
|
|
169
|
+
Vector *right = sqlite3_get_auxdata(ctx, 1);
|
|
170
|
+
int fresh = !right;
|
|
171
|
+
if (fresh) {
|
|
172
|
+
right = sqlite3_malloc64(sizeof(*right));
|
|
173
|
+
if (!right) { sqlite3_free(left.owned); sqlite3_result_error_nomem(ctx); return; }
|
|
174
|
+
status = read_vector(argv[1], right, 1);
|
|
175
|
+
if (status != 1) {
|
|
176
|
+
release_vector(right); sqlite3_free(left.owned);
|
|
177
|
+
if (status < 0) sqlite3_result_error_nomem(ctx);
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
double value;
|
|
182
|
+
if (score(&left, right, (int)(intptr_t)sqlite3_user_data(ctx), &value)) sqlite3_result_double(ctx, value);
|
|
183
|
+
sqlite3_free(left.owned);
|
|
184
|
+
/* set_auxdata may immediately destroy the value: nothing accesses it after this. */
|
|
185
|
+
if (fresh) sqlite3_set_auxdata(ctx, 1, right, release_vector);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
static void version(sqlite3_context *ctx, int argc, sqlite3_value **argv) {
|
|
189
|
+
(void)argc; (void)argv;
|
|
190
|
+
sqlite3_result_text(ctx, "an5-vector/1", -1, SQLITE_STATIC);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
#ifdef _WIN32
|
|
194
|
+
__declspec(dllexport)
|
|
195
|
+
#endif
|
|
196
|
+
int sqlite3_extension_init(sqlite3 *db, char **error, const sqlite3_api_routines *api) {
|
|
197
|
+
(void)error;
|
|
198
|
+
SQLITE_EXTENSION_INIT2(api);
|
|
199
|
+
const char *names[] = { "an5_vec_cosine", "an5_vec_l2", "an5_vec_ip" };
|
|
200
|
+
int flags = SQLITE_UTF8 | SQLITE_DETERMINISTIC;
|
|
201
|
+
#ifdef SQLITE_INNOCUOUS
|
|
202
|
+
flags |= SQLITE_INNOCUOUS;
|
|
203
|
+
#endif
|
|
204
|
+
for (int i = 0; i < 3; ++i) {
|
|
205
|
+
int rc = sqlite3_create_function_v2(db, names[i], 2, flags, (void *)(intptr_t)i, distance, NULL, NULL, NULL);
|
|
206
|
+
if (rc != SQLITE_OK) return rc;
|
|
207
|
+
}
|
|
208
|
+
return sqlite3_create_function_v2(db, "an5_vector_version", 0, flags, NULL, version, NULL, NULL, NULL);
|
|
209
|
+
}
|