@an5/adapters 0.2.11 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -5
- package/dist/an5Adapter.d.ts +42 -1
- package/dist/an5Adapter.js +146 -38
- package/dist/base/types.d.ts +14 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +8 -1
- package/dist/sqlite/browserEngine.d.ts +13 -0
- package/dist/sqlite/browserEngine.js +13 -0
- package/dist/sqlite/engine.d.ts +5 -0
- package/dist/sqlite/engine.js +10 -0
- package/dist/sqlite/index.d.ts +1 -0
- package/dist/sqlite/index.js +1 -0
- package/dist/sqlite/vector.d.ts +155 -0
- package/dist/sqlite/vector.js +432 -0
- package/dotnet/An5Adapter.cs +99 -28
- package/dotnet/Base/Core.cs +14 -0
- package/dotnet/Mssql/MssqlEngine.cs +44 -1
- package/dotnet/Postgres/PostgresEngine.cs +2 -0
- package/dotnet/Sqlite/SqliteEngine.cs +82 -2
- package/dotnet/Sqlite/SqliteVectors.cs +401 -0
- package/golang/an5_adapter.go +84 -32
- package/golang/base/vectors.go +263 -0
- package/golang/sqlite/vector.go +248 -0
- package/golang/sqlite_vector_test.go +288 -0
- package/golang/table_client.go +67 -3
- package/java/pom.xml +177 -0
- package/java/src/main/java/an5/adapters/An5Adapter.java +374 -0
- package/java/src/main/java/an5/adapters/An5Aggregate.java +116 -0
- package/java/src/main/java/an5/adapters/An5GroupBy.java +97 -0
- package/java/src/main/java/an5/adapters/An5Query.java +100 -0
- package/java/src/main/java/an5/adapters/An5TableClient.java +1030 -0
- package/java/src/main/java/an5/adapters/An5ViewClient.java +107 -0
- package/java/src/main/java/an5/adapters/base/Dialect.java +103 -0
- package/java/src/main/java/an5/adapters/base/FieldMeta.java +42 -0
- package/java/src/main/java/an5/adapters/base/JdbcUrls.java +201 -0
- package/java/src/main/java/an5/adapters/base/Metadata.java +195 -0
- package/java/src/main/java/an5/adapters/base/RelationDef.java +26 -0
- package/java/src/main/java/an5/adapters/base/SqlBuilder.java +422 -0
- package/java/src/main/java/an5/adapters/base/SqliteVectors.java +375 -0
- package/java/src/main/java/an5/adapters/base/Vectors.java +123 -0
- package/java/src/test/java/An5AdaptersSmoke.java +733 -0
- package/java/src/test/java/Json.java +174 -0
- package/java/src/test/java/QuerySemantics.java +111 -0
- package/kotlin/build.gradle.kts +145 -0
- package/kotlin/settings.gradle.kts +10 -0
- package/kotlin/src/main/kotlin/an5/adapters/Aggregate.kt +89 -0
- package/kotlin/src/main/kotlin/an5/adapters/An5.kt +91 -0
- package/kotlin/src/main/kotlin/an5/adapters/Query.kt +130 -0
- package/kotlin/src/main/kotlin/an5/adapters/Rows.kt +166 -0
- package/kotlin/src/main/kotlin/an5/adapters/TableClient.kt +169 -0
- package/kotlin/src/main/kotlin/an5/adapters/Types.kt +77 -0
- package/kotlin/src/test/kotlin/an5/adapters/An5Smoke.kt +329 -0
- package/native/sqlite/CMakeLists.txt +12 -0
- package/native/sqlite/README.md +117 -0
- package/native/sqlite/an5_vector.c +209 -0
- package/package.json +33 -7
- package/python/an5_adapter.py +37 -6
- package/python/base/__init__.py +15 -0
- package/python/base/vectors.py +431 -0
- package/python/sqlite/__init__.py +2 -2
- package/python/sqlite/provider.py +21 -1
- package/python/table_client.py +129 -59
- package/rust/src/adapter.rs +342 -24
- package/rust/src/base/mod.rs +7 -0
- package/rust/src/base/vectors.rs +293 -0
- package/rust/src/lib.rs +3 -3
- package/scripts/sqlite-native-build.js +32 -0
- package/swift/Package.swift +22 -0
- package/swift/Sources/An5Adapters/An5Adapter.swift +160 -0
- package/swift/Sources/An5Adapters/Metadata.swift +183 -0
- package/swift/Sources/An5Adapters/Query.swift +105 -0
- package/swift/Sources/An5Adapters/Row+Accessors.swift +129 -0
- package/swift/Sources/An5Adapters/SQLBuilder.swift +359 -0
- package/swift/Sources/An5Adapters/SQLiteDriver.swift +301 -0
- package/swift/Sources/An5Adapters/SqliteVectors.swift +243 -0
- package/swift/Sources/An5Adapters/TableClient.swift +671 -0
- package/swift/Sources/An5Adapters/Types.swift +127 -0
- package/swift/Sources/An5Adapters/Vectors.swift +86 -0
- package/swift/Sources/An5Adapters/ViewClient.swift +50 -0
- package/swift/Sources/CSQLite/module.modulemap +8 -0
- package/swift/Sources/CSQLite/shim.h +9 -0
- package/swift/Tests/An5AdaptersTests/QuerySemanticsTests.swift +51 -0
- package/swift/Tests/An5AdaptersTests/RuntimeSmokeTests.swift +488 -0
- package/typescript/src/an5Adapter.ts +155 -31
- package/typescript/src/base/types.ts +15 -0
- package/typescript/src/index.ts +13 -0
- package/typescript/src/sqlite/browserEngine.ts +24 -0
- package/typescript/src/sqlite/engine.ts +13 -0
- package/typescript/src/sqlite/index.ts +1 -1
- package/typescript/src/sqlite/vector.ts +537 -0
- package/golang/sqlite/sqlite.go +0 -14
|
@@ -0,0 +1,432 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
// ─── SQLite vector search ──────────────────────────────────────────────────────────
|
|
3
|
+
//
|
|
4
|
+
// SQLite has no vector type, so `VECTOR(n)` columns are stored as a BLOB of
|
|
5
|
+
// little-endian float32 values (see `encodeVector`) and searched in one of four
|
|
6
|
+
// ways. The order is fixed so every runtime that speaks SQLite can implement it:
|
|
7
|
+
//
|
|
8
|
+
// 1. sqlite-vec native scalar distance functions, if the extension loads.
|
|
9
|
+
// 2. udf `an5_vec_cosine` / `an5_vec_l2` / `an5_vec_ip` registered by
|
|
10
|
+
// the driver. Reads the BLOB and the legacy JSON text.
|
|
11
|
+
// 3. sql `json_each` brute force in plain SQL. No UDF needed, so it
|
|
12
|
+
// works in browsers and on JDBC, but only for JSON text.
|
|
13
|
+
// 4. memory the adapter loads the column and scores it in the client.
|
|
14
|
+
//
|
|
15
|
+
// Strategies 1-3 rank in the database and only transfer the `take` rows that
|
|
16
|
+
// match, which is the reason they are preferred over the in-memory path.
|
|
17
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
+
exports.SqliteVectorSupport = exports.AN5_VECTOR_FUNCTIONS = exports.SQLITE_VEC_FUNCTIONS = void 0;
|
|
19
|
+
exports.parseVector = parseVector;
|
|
20
|
+
exports.decodeVector = decodeVector;
|
|
21
|
+
exports.encodeVector = encodeVector;
|
|
22
|
+
exports.needsVectorEncoding = needsVectorEncoding;
|
|
23
|
+
exports.isVectorField = isVectorField;
|
|
24
|
+
exports.vectorDistance = vectorDistance;
|
|
25
|
+
exports.buildSqliteVectorQuery = buildSqliteVectorQuery;
|
|
26
|
+
exports.planSqliteVectorStrategies = planSqliteVectorStrategies;
|
|
27
|
+
exports.readSqliteColumnType = readSqliteColumnType;
|
|
28
|
+
exports.runSqliteVectorSearch = runSqliteVectorSearch;
|
|
29
|
+
/** Scalar functions shipped by the sqlite-vec extension. */
|
|
30
|
+
exports.SQLITE_VEC_FUNCTIONS = {
|
|
31
|
+
cosine: 'vec_distance_cosine',
|
|
32
|
+
euclidean: 'vec_distance_l2',
|
|
33
|
+
dot: 'vec_distance_ip',
|
|
34
|
+
};
|
|
35
|
+
/** Scalar functions the adapter registers itself when the driver allows it. */
|
|
36
|
+
exports.AN5_VECTOR_FUNCTIONS = {
|
|
37
|
+
cosine: 'an5_vec_cosine',
|
|
38
|
+
euclidean: 'an5_vec_l2',
|
|
39
|
+
dot: 'an5_vec_ip',
|
|
40
|
+
};
|
|
41
|
+
const BYTES_PER_FLOAT = 4;
|
|
42
|
+
function normalizeMetric(metric) {
|
|
43
|
+
return metric === 'euclidean' || metric === 'dot' ? metric : 'cosine';
|
|
44
|
+
}
|
|
45
|
+
// ─── Codec ─────────────────────────────────────────────────────────────────────────
|
|
46
|
+
function toBytes(value) {
|
|
47
|
+
if (value instanceof Uint8Array)
|
|
48
|
+
return value;
|
|
49
|
+
if (typeof Buffer !== 'undefined' && Buffer.isBuffer(value))
|
|
50
|
+
return value;
|
|
51
|
+
if (ArrayBuffer.isView(value)) {
|
|
52
|
+
const view = value;
|
|
53
|
+
return new Uint8Array(view.buffer, view.byteOffset, view.byteLength);
|
|
54
|
+
}
|
|
55
|
+
if (value instanceof ArrayBuffer)
|
|
56
|
+
return new Uint8Array(value);
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
/** Read a numeric array out of the shapes a driver can hand back. */
|
|
60
|
+
function parseVector(value) {
|
|
61
|
+
if (value === null || value === undefined)
|
|
62
|
+
return null;
|
|
63
|
+
if (Array.isArray(value)) {
|
|
64
|
+
const out = new Array(value.length);
|
|
65
|
+
for (let i = 0; i < value.length; i++) {
|
|
66
|
+
const n = Number(value[i]);
|
|
67
|
+
if (!Number.isFinite(n))
|
|
68
|
+
return null;
|
|
69
|
+
out[i] = n;
|
|
70
|
+
}
|
|
71
|
+
return out.length ? out : null;
|
|
72
|
+
}
|
|
73
|
+
if (typeof value === 'string') {
|
|
74
|
+
const trimmed = value.trim();
|
|
75
|
+
if (!trimmed.startsWith('['))
|
|
76
|
+
return null;
|
|
77
|
+
try {
|
|
78
|
+
return parseVector(JSON.parse(trimmed));
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Decode a stored vector: a float32 BLOB, a legacy JSON text column, or an
|
|
88
|
+
* array the caller already decoded.
|
|
89
|
+
*/
|
|
90
|
+
function decodeVector(value) {
|
|
91
|
+
const direct = parseVector(value);
|
|
92
|
+
if (direct)
|
|
93
|
+
return direct;
|
|
94
|
+
const bytes = toBytes(value);
|
|
95
|
+
if (!bytes || bytes.length === 0 || bytes.length % BYTES_PER_FLOAT !== 0)
|
|
96
|
+
return null;
|
|
97
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
98
|
+
const out = new Array(bytes.length / BYTES_PER_FLOAT);
|
|
99
|
+
for (let i = 0; i < out.length; i++) {
|
|
100
|
+
out[i] = view.getFloat32(i * BYTES_PER_FLOAT, true);
|
|
101
|
+
if (!Number.isFinite(out[i]))
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
return out;
|
|
105
|
+
}
|
|
106
|
+
/** Encode a vector as the little-endian float32 BLOB `VECTOR(n)` columns store. */
|
|
107
|
+
function encodeVector(values) {
|
|
108
|
+
const buf = typeof Buffer !== 'undefined'
|
|
109
|
+
? Buffer.alloc(values.length * BYTES_PER_FLOAT)
|
|
110
|
+
: new Uint8Array(values.length * BYTES_PER_FLOAT);
|
|
111
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
112
|
+
for (let i = 0; i < values.length; i++) {
|
|
113
|
+
if (!Number.isFinite(values[i]))
|
|
114
|
+
throw new TypeError('Vector elements must be finite numbers');
|
|
115
|
+
view.setFloat32(i * BYTES_PER_FLOAT, values[i], true);
|
|
116
|
+
if (!Number.isFinite(view.getFloat32(i * BYTES_PER_FLOAT, true))) {
|
|
117
|
+
throw new RangeError('Vector element exceeds float32 range');
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
return buf;
|
|
121
|
+
}
|
|
122
|
+
/** True when the value still needs encoding on its way into a `VECTOR(n)` column. */
|
|
123
|
+
function needsVectorEncoding(value) {
|
|
124
|
+
return Array.isArray(value) || value instanceof Float32Array || value instanceof Float64Array;
|
|
125
|
+
}
|
|
126
|
+
/** True when a generated metadata entry describes a `VECTOR(n)` column. */
|
|
127
|
+
function isVectorField(def) {
|
|
128
|
+
if (!def)
|
|
129
|
+
return false;
|
|
130
|
+
if (typeof def === 'string')
|
|
131
|
+
return def.includes('[]') || /^VECTOR\b/i.test(def.trim());
|
|
132
|
+
const record = def;
|
|
133
|
+
if (record.kind === 'vector')
|
|
134
|
+
return true;
|
|
135
|
+
const sql = String(record.sql ?? record.type ?? '');
|
|
136
|
+
if (/^VECTOR\s*(\(|$)/i.test(sql.trim()))
|
|
137
|
+
return true;
|
|
138
|
+
return String(record.ts ?? '').includes('[]');
|
|
139
|
+
}
|
|
140
|
+
// ─── Distance ──────────────────────────────────────────────────────────────────────
|
|
141
|
+
/**
|
|
142
|
+
* Distance between two equal-length vectors, lower is closer. Mirrors the
|
|
143
|
+
* functions registered on the drivers so a search ranks the same way whichever
|
|
144
|
+
* strategy ran.
|
|
145
|
+
*/
|
|
146
|
+
function vectorDistance(a, b, metric) {
|
|
147
|
+
if (!Array.isArray(a) || !Array.isArray(b) || a.length === 0 || a.length !== b.length)
|
|
148
|
+
return null;
|
|
149
|
+
let dot = 0;
|
|
150
|
+
let magA = 0;
|
|
151
|
+
let magB = 0;
|
|
152
|
+
for (let i = 0; i < a.length; i++) {
|
|
153
|
+
const av = Number(a[i]) || 0;
|
|
154
|
+
const bv = Number(b[i]) || 0;
|
|
155
|
+
dot += av * bv;
|
|
156
|
+
magA += av * av;
|
|
157
|
+
magB += bv * bv;
|
|
158
|
+
}
|
|
159
|
+
const resolved = normalizeMetric(metric);
|
|
160
|
+
if (resolved === 'euclidean') {
|
|
161
|
+
let sum = 0;
|
|
162
|
+
for (let i = 0; i < a.length; i++) {
|
|
163
|
+
const diff = (Number(a[i]) || 0) - (Number(b[i]) || 0);
|
|
164
|
+
sum += diff * diff;
|
|
165
|
+
}
|
|
166
|
+
return Math.sqrt(sum);
|
|
167
|
+
}
|
|
168
|
+
if (resolved === 'dot')
|
|
169
|
+
return -dot;
|
|
170
|
+
return magA && magB ? 1 - dot / (Math.sqrt(magA) * Math.sqrt(magB)) : 1;
|
|
171
|
+
}
|
|
172
|
+
// ─── Query builders ────────────────────────────────────────────────────────────────
|
|
173
|
+
/** `json_valid` fails on a float32 BLOB, so the JSON path needs this guard. */
|
|
174
|
+
function jsonText(column) {
|
|
175
|
+
return `CASE WHEN json_valid(${column}) THEN ${column} ELSE '[]' END`;
|
|
176
|
+
}
|
|
177
|
+
function jsonDistanceExpr(metric, column) {
|
|
178
|
+
const row = jsonText(column);
|
|
179
|
+
const sameLength = `(SELECT COUNT(*) FROM json_each(${row})) = (SELECT COUNT(*) FROM q)`;
|
|
180
|
+
const terms = {
|
|
181
|
+
cosine: `1.0 - (SELECT SUM(je.value * q.v) FROM json_each(${row}) je JOIN q ON q.k = je.key)`
|
|
182
|
+
+ ` / NULLIF(sqrt((SELECT SUM(q.v * q.v) FROM q)) * sqrt((SELECT SUM(je.value * je.value) FROM json_each(${row}) je)), 0)`,
|
|
183
|
+
euclidean: `sqrt((SELECT SUM((je.value - q.v) * (je.value - q.v)) FROM json_each(${row}) je JOIN q ON q.k = je.key))`,
|
|
184
|
+
dot: `-(SELECT SUM(je.value * q.v) FROM json_each(${row}) je JOIN q ON q.k = je.key)`,
|
|
185
|
+
};
|
|
186
|
+
return `CASE WHEN ${sameLength} THEN ${terms[metric]} END`;
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Ranking SQL for one strategy. `sqlite-vec` and `udf` bind the query vector as
|
|
190
|
+
* a float32 BLOB; `sql` binds it as JSON text because `json_each` reads text.
|
|
191
|
+
*/
|
|
192
|
+
function buildSqliteVectorQuery(args) {
|
|
193
|
+
const { strategy, metric, table, column, take } = args;
|
|
194
|
+
const params = args.params;
|
|
195
|
+
const tail = args.tail ?? '';
|
|
196
|
+
const queryVector = args.vector;
|
|
197
|
+
// A row whose stored vector cannot be scored (a NULL, a text column, another
|
|
198
|
+
// dimension) yields a NULL distance; those rows are dropped rather than
|
|
199
|
+
// reported, so a caller never has to sort them out of the result.
|
|
200
|
+
const rank = (inner) => `SELECT * FROM (${inner}) AS an5_ranked WHERE distance IS NOT NULL ORDER BY distance ASC LIMIT ${take}`;
|
|
201
|
+
if (strategy === 'sqlite-vec') {
|
|
202
|
+
params.an5_vector = encodeVector(queryVector);
|
|
203
|
+
params.an5_vector_bytes = queryVector.length * BYTES_PER_FLOAT;
|
|
204
|
+
const fn = exports.SQLITE_VEC_FUNCTIONS[metric];
|
|
205
|
+
const stored = `vec_f32(${column})`;
|
|
206
|
+
const distance = `CASE WHEN vec_length(${stored}) = ${queryVector.length} THEN ${fn}(${stored}, @an5_vector) END`;
|
|
207
|
+
// vec_f32 accepts JSON text and float32 BLOBs, preserving legacy rows.
|
|
208
|
+
return {
|
|
209
|
+
sql: rank(`SELECT *, ${distance} AS distance FROM ${table}${tail ? `${tail} AND` : ' WHERE'} ${column} IS NOT NULL`),
|
|
210
|
+
params,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
if (strategy === 'udf') {
|
|
214
|
+
params.an5_vector = encodeVector(queryVector);
|
|
215
|
+
const distance = `${exports.AN5_VECTOR_FUNCTIONS[metric]}(${column}, @an5_vector)`;
|
|
216
|
+
return { sql: rank(`SELECT *, ${distance} AS distance FROM ${table}${tail}`), params };
|
|
217
|
+
}
|
|
218
|
+
params.an5_vector_json = JSON.stringify(queryVector);
|
|
219
|
+
const distance = jsonDistanceExpr(metric, column);
|
|
220
|
+
return {
|
|
221
|
+
sql: 'WITH q AS (SELECT je.key AS k, CAST(je.value AS REAL) AS v FROM json_each(@an5_vector_json) je) '
|
|
222
|
+
+ rank(`SELECT *, ${distance} AS distance FROM ${table}${tail}`),
|
|
223
|
+
params,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
function probed(fn) {
|
|
227
|
+
try {
|
|
228
|
+
return fn();
|
|
229
|
+
}
|
|
230
|
+
catch {
|
|
231
|
+
return null;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
/**
|
|
235
|
+
* Resolve the ordered list of strategies to try for this connection.
|
|
236
|
+
*
|
|
237
|
+
* `declaredType` is the column's DDL type: the JSON strategy only reaches rows
|
|
238
|
+
* stored as text, so a `BLOB` column drops it from the order.
|
|
239
|
+
*/
|
|
240
|
+
function planSqliteVectorStrategies(caps, declaredType, preference) {
|
|
241
|
+
const wanted = preference && preference !== 'auto' ? [preference] : [];
|
|
242
|
+
const available = [];
|
|
243
|
+
if (caps.native && caps.udf)
|
|
244
|
+
available.push('udf');
|
|
245
|
+
if (caps.vec)
|
|
246
|
+
available.push('sqlite-vec');
|
|
247
|
+
if (caps.udf && !caps.native)
|
|
248
|
+
available.push('udf');
|
|
249
|
+
if (caps.json1)
|
|
250
|
+
available.push('sql');
|
|
251
|
+
if (wanted.length > 0)
|
|
252
|
+
return wanted.filter((s) => s !== 'memory');
|
|
253
|
+
const plan = available;
|
|
254
|
+
// A BLOB column has no JSON to read, so `json_each` can only produce NULLs.
|
|
255
|
+
const type = String(declaredType ?? '').toUpperCase();
|
|
256
|
+
if (type && !/TEXT|CHAR|CLOB|STRING|JSON/.test(type)) {
|
|
257
|
+
return plan.filter((s) => s !== 'sql');
|
|
258
|
+
}
|
|
259
|
+
return plan;
|
|
260
|
+
}
|
|
261
|
+
/**
|
|
262
|
+
* Register `an5_vec_*` on a driver and load sqlite-vec when a path is configured.
|
|
263
|
+
* Safe to call repeatedly; each driver hook is used at most once.
|
|
264
|
+
*/
|
|
265
|
+
class SqliteVectorSupport {
|
|
266
|
+
constructor(hooks, options = {}) {
|
|
267
|
+
this.hooks = hooks;
|
|
268
|
+
this.options = options;
|
|
269
|
+
this.caps = null;
|
|
270
|
+
/**
|
|
271
|
+
* Declared column types and the "any binary row" probe, keyed by column.
|
|
272
|
+
*
|
|
273
|
+
* Both cost a query, and neither changes while a schema is in use. A stale
|
|
274
|
+
* answer is still safe: the probe only decides which strategy to try first, and
|
|
275
|
+
* a strategy that fails falls through to the next one rather than reporting a
|
|
276
|
+
* wrong result.
|
|
277
|
+
*/
|
|
278
|
+
this.columnTypes = new Map();
|
|
279
|
+
this.binaryRows = new Map();
|
|
280
|
+
}
|
|
281
|
+
async capabilities() {
|
|
282
|
+
if (this.caps)
|
|
283
|
+
return this.caps;
|
|
284
|
+
const caps = { vec: false, udf: false, json1: false };
|
|
285
|
+
const native = probed(() => this.hooks.native?.());
|
|
286
|
+
if (native) {
|
|
287
|
+
if (this.options.sqliteVec) {
|
|
288
|
+
probed(() => native.loadExtension(this.options.sqliteVec));
|
|
289
|
+
}
|
|
290
|
+
if (probed(() => native.prepare('SELECT vec_version()').get()) !== null)
|
|
291
|
+
caps.vec = true;
|
|
292
|
+
if (probed(() => native.prepare('SELECT an5_vector_version()').get()) !== null) {
|
|
293
|
+
caps.native = true;
|
|
294
|
+
caps.udf = true;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
const register = this.hooks.registerFunction;
|
|
298
|
+
if (register && !caps.native) {
|
|
299
|
+
// better-sqlite3 takes an options object where every other driver takes a
|
|
300
|
+
// plain function, so both shapes are offered and the driver picks.
|
|
301
|
+
const names = Object.values(exports.AN5_VECTOR_FUNCTIONS);
|
|
302
|
+
const impls = Object.keys(exports.AN5_VECTOR_FUNCTIONS).map((metric) => {
|
|
303
|
+
const fn = (...args) => {
|
|
304
|
+
const a = decodeVector(args[0]);
|
|
305
|
+
const b = decodeVector(args[1]);
|
|
306
|
+
return a && b ? vectorDistance(a, b, metric) : null;
|
|
307
|
+
};
|
|
308
|
+
return { name: exports.AN5_VECTOR_FUNCTIONS[metric], fn };
|
|
309
|
+
});
|
|
310
|
+
let all = true;
|
|
311
|
+
for (const { name, fn } of impls) {
|
|
312
|
+
if (probed(() => register(name, fn)) === null) {
|
|
313
|
+
all = false;
|
|
314
|
+
break;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
// A driver that accepts the registration is trusted here; a query that
|
|
318
|
+
// still fails falls through to the next strategy.
|
|
319
|
+
caps.udf = all && names.length > 0;
|
|
320
|
+
}
|
|
321
|
+
this.caps = caps;
|
|
322
|
+
return caps;
|
|
323
|
+
}
|
|
324
|
+
/** Confirms JSON1 and a usable sqlite-vec/UDF before a query is built for them. */
|
|
325
|
+
async probe(exec) {
|
|
326
|
+
const caps = await this.capabilities();
|
|
327
|
+
if (!caps.json1) {
|
|
328
|
+
try {
|
|
329
|
+
await exec("SELECT json_valid('[1]') AS an5_json_ok");
|
|
330
|
+
caps.json1 = true;
|
|
331
|
+
}
|
|
332
|
+
catch {
|
|
333
|
+
caps.json1 = false;
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
return caps;
|
|
337
|
+
}
|
|
338
|
+
/** The column's declared DDL type, remembered per column. */
|
|
339
|
+
async columnType(exec, table, column) {
|
|
340
|
+
const key = `${table}.${column}`;
|
|
341
|
+
if (!this.columnTypes.has(key)) {
|
|
342
|
+
const found = await readSqliteColumnType(exec, table, column).catch(() => null);
|
|
343
|
+
this.columnTypes.set(key, found);
|
|
344
|
+
}
|
|
345
|
+
return this.columnTypes.get(key) ?? null;
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Whether any row stores raw bytes, which `json_each` cannot read.
|
|
349
|
+
*
|
|
350
|
+
* Checked only for the `sql` strategy, where a legacy JSON column that has
|
|
351
|
+
* since taken new float32 rows would otherwise rank just the old ones.
|
|
352
|
+
*/
|
|
353
|
+
async hasBinaryRows(exec, table, column, tail, params) {
|
|
354
|
+
const rows = await exec(`SELECT 1 FROM ${table}${tail ? `${tail} AND` : ' WHERE'} typeof(${column}) = 'blob' LIMIT 1`, params).catch(() => []);
|
|
355
|
+
return rows.length > 0;
|
|
356
|
+
}
|
|
357
|
+
reset() {
|
|
358
|
+
this.caps = null;
|
|
359
|
+
this.columnTypes.clear();
|
|
360
|
+
this.binaryRows.clear();
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
exports.SqliteVectorSupport = SqliteVectorSupport;
|
|
364
|
+
/** Reads a vector column's declared DDL type, or null when it cannot be found. */
|
|
365
|
+
async function readSqliteColumnType(exec, table, column) {
|
|
366
|
+
const rows = await exec("SELECT type FROM pragma_table_info(@an5_table) WHERE name = @an5_column", {
|
|
367
|
+
an5_table: table,
|
|
368
|
+
an5_column: column,
|
|
369
|
+
});
|
|
370
|
+
const value = rows?.[0]?.type;
|
|
371
|
+
return value ? String(value) : null;
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* Rank a SQLite table by distance inside the database, trying each available
|
|
375
|
+
* strategy in order. Returns null when the connection offers none, so the
|
|
376
|
+
* caller can fall back to scoring in memory.
|
|
377
|
+
*/
|
|
378
|
+
async function runSqliteVectorSearch(args) {
|
|
379
|
+
if (!Array.isArray(args.vector) || args.vector.length === 0)
|
|
380
|
+
return null;
|
|
381
|
+
const caps = args.vectorSupport
|
|
382
|
+
? await args.vectorSupport.probe(args.exec)
|
|
383
|
+
: { vec: false, udf: false, json1: false };
|
|
384
|
+
if (!args.vectorSupport) {
|
|
385
|
+
try {
|
|
386
|
+
await args.exec("SELECT json_valid('[1]') AS an5_json_ok");
|
|
387
|
+
caps.json1 = true;
|
|
388
|
+
}
|
|
389
|
+
catch {
|
|
390
|
+
caps.json1 = false;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
let declaredType = null;
|
|
394
|
+
try {
|
|
395
|
+
declaredType = args.vectorSupport
|
|
396
|
+
? await args.vectorSupport.columnType(args.exec, args.rawTable, args.rawColumn)
|
|
397
|
+
: await readSqliteColumnType(args.exec, args.rawTable, args.rawColumn);
|
|
398
|
+
}
|
|
399
|
+
catch {
|
|
400
|
+
declaredType = null;
|
|
401
|
+
}
|
|
402
|
+
for (const strategy of planSqliteVectorStrategies(caps, declaredType, args.preference)) {
|
|
403
|
+
if (strategy === 'sql') {
|
|
404
|
+
const binary = args.vectorSupport
|
|
405
|
+
? await args.vectorSupport.hasBinaryRows(args.exec, args.table, args.column, args.tail ?? '', args.params)
|
|
406
|
+
: (await args.exec(`SELECT 1 FROM ${args.table}${args.tail ? `${args.tail} AND` : ' WHERE'} typeof(${args.column}) = 'blob' LIMIT 1`, args.params)).length > 0;
|
|
407
|
+
if (binary)
|
|
408
|
+
continue;
|
|
409
|
+
}
|
|
410
|
+
const params = { ...args.params };
|
|
411
|
+
const query = buildSqliteVectorQuery({
|
|
412
|
+
strategy,
|
|
413
|
+
metric: normalizeMetric(args.metric),
|
|
414
|
+
table: args.table,
|
|
415
|
+
column: args.column,
|
|
416
|
+
vector: args.vector,
|
|
417
|
+
take: args.take,
|
|
418
|
+
tail: args.tail ?? '',
|
|
419
|
+
params,
|
|
420
|
+
});
|
|
421
|
+
try {
|
|
422
|
+
const rows = await args.exec(query.sql, params);
|
|
423
|
+
return { rows: rows ?? [], strategy };
|
|
424
|
+
}
|
|
425
|
+
catch {
|
|
426
|
+
// The driver advertised the capability but the query failed, e.g. an
|
|
427
|
+
// extension that did not really load. The next strategy is cheaper than
|
|
428
|
+
// reporting the failure, and memory is always available as a last resort.
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
return null;
|
|
432
|
+
}
|
package/dotnet/An5Adapter.cs
CHANGED
|
@@ -19,21 +19,34 @@ namespace An5Orm
|
|
|
19
19
|
private readonly int _commandTimeout;
|
|
20
20
|
|
|
21
21
|
public An5Adapter(string connectionString, int commandTimeout = 60)
|
|
22
|
+
: this(connectionString, commandTimeout, null) { }
|
|
23
|
+
|
|
24
|
+
public An5Adapter(string connectionString, int commandTimeout, SqliteVectorSupport vectorSupport)
|
|
22
25
|
{
|
|
23
26
|
ConnectionString = connectionString ?? throw new ArgumentNullException(nameof(connectionString));
|
|
24
27
|
_commandTimeout = commandTimeout;
|
|
25
28
|
_engine = DialectDetector.Detect(connectionString) switch
|
|
26
29
|
{
|
|
27
30
|
Dialect.Postgres => new PostgresEngine(connectionString, commandTimeout),
|
|
28
|
-
Dialect.Sqlite => new SqliteEngine(connectionString, commandTimeout),
|
|
31
|
+
Dialect.Sqlite => new SqliteEngine(connectionString, commandTimeout, vectorSupport),
|
|
29
32
|
_ => new MssqlEngine(connectionString, commandTimeout)
|
|
30
33
|
};
|
|
31
34
|
}
|
|
32
35
|
|
|
33
|
-
public An5Adapter(An5AdapterOptions options)
|
|
36
|
+
public An5Adapter(An5AdapterOptions options)
|
|
37
|
+
: this(options.ConnectionString, options.CommandTimeout,
|
|
38
|
+
new SqliteVectorSupport { SqliteVecPath = options.SqliteVec, Preference = options.VectorStrategy })
|
|
39
|
+
{ }
|
|
34
40
|
|
|
35
41
|
// ── Raw query execution ────────────────────────────────────────────────
|
|
36
42
|
|
|
43
|
+
/// <summary>
|
|
44
|
+
/// SQLite only: the vector hooks of the configured connection, or null on
|
|
45
|
+
/// the other providers. `VectorSearch` needs them to rank a `VECTOR(n)`
|
|
46
|
+
/// column inside the database.
|
|
47
|
+
/// </summary>
|
|
48
|
+
public SqliteVectorSupport VectorSupport => _engine.VectorSupport;
|
|
49
|
+
|
|
37
50
|
public List<Dictionary<string, object>> QueryRaw(string sql, Dictionary<string, object> parameters = null)
|
|
38
51
|
=> _engine.QueryRaw(sql, parameters);
|
|
39
52
|
|
|
@@ -895,7 +908,20 @@ namespace An5Orm
|
|
|
895
908
|
public List<(T Item, double Distance)> VectorSearch(VectorSearchArgs args)
|
|
896
909
|
{
|
|
897
910
|
args ??= new VectorSearchArgs();
|
|
898
|
-
|
|
911
|
+
Dictionary<string, object> parameters = null;
|
|
912
|
+
string whereSql = null;
|
|
913
|
+
if (args.Where != null)
|
|
914
|
+
{
|
|
915
|
+
parameters = new Dictionary<string, object>();
|
|
916
|
+
whereSql = SqlBuilder.ParseWhere(args.Where, parameters, _dialect);
|
|
917
|
+
}
|
|
918
|
+
return VectorSearch(
|
|
919
|
+
args.Vector,
|
|
920
|
+
args.Take,
|
|
921
|
+
string.IsNullOrEmpty(whereSql) ? null : whereSql,
|
|
922
|
+
parameters,
|
|
923
|
+
args.VectorField,
|
|
924
|
+
args.DistanceMetric);
|
|
899
925
|
}
|
|
900
926
|
|
|
901
927
|
public T FindFirst(string whereClause = null, Dictionary<string, object> parameters = null, string orderBy = null)
|
|
@@ -1006,10 +1032,29 @@ namespace An5Orm
|
|
|
1006
1032
|
string whereClause = null, Dictionary<string, object> parameters = null,
|
|
1007
1033
|
string vectorField = "Embedding", string distanceMetric = "cosine")
|
|
1008
1034
|
{
|
|
1009
|
-
// 1.
|
|
1010
|
-
//
|
|
1011
|
-
//
|
|
1012
|
-
|
|
1035
|
+
// 1. SQLite ranks through sqlite-vec, a driver function or json_each,
|
|
1036
|
+
// whichever this connection supports, so the table never has to
|
|
1037
|
+
// leave the database.
|
|
1038
|
+
if (_dialect == Dialect.Sqlite)
|
|
1039
|
+
{
|
|
1040
|
+
var p = parameters != null ? new Dictionary<string, object>(parameters) : new Dictionary<string, object>();
|
|
1041
|
+
var quotedVectorField = SqlQuote.QuoteName(vectorField, _dialect);
|
|
1042
|
+
var ranked = SqliteVectors.RunSearch(
|
|
1043
|
+
_adapter.QueryRaw,
|
|
1044
|
+
_tableName,
|
|
1045
|
+
_rawTable.Trim('[', ']'),
|
|
1046
|
+
quotedVectorField,
|
|
1047
|
+
vectorField,
|
|
1048
|
+
vector,
|
|
1049
|
+
distanceMetric,
|
|
1050
|
+
take,
|
|
1051
|
+
string.IsNullOrWhiteSpace(whereClause) ? "" : $" WHERE {whereClause}",
|
|
1052
|
+
p,
|
|
1053
|
+
_adapter.VectorSupport);
|
|
1054
|
+
if (ranked != null) return ToRanked(ranked, take);
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1057
|
+
// 2. Primary path: Native database SQL vector query execution (MSSQL VECTOR_DISTANCE / Postgres pgvector)
|
|
1013
1058
|
if (_dialect != Dialect.Sqlite)
|
|
1014
1059
|
try
|
|
1015
1060
|
{
|
|
@@ -1063,7 +1108,7 @@ namespace An5Orm
|
|
|
1063
1108
|
// Secondary fallback: In-memory similarity computation if DB instance lacks native vector extension
|
|
1064
1109
|
}
|
|
1065
1110
|
|
|
1066
|
-
//
|
|
1111
|
+
// 3. Secondary fallback: In-memory similarity computation
|
|
1067
1112
|
var rows = FindMany(whereClause, parameters);
|
|
1068
1113
|
var results = new List<(T, double)>();
|
|
1069
1114
|
var vectorProp = typeof(T).GetProperty(vectorField, BindingFlags.Public | BindingFlags.Instance | BindingFlags.IgnoreCase);
|
|
@@ -1071,33 +1116,59 @@ namespace An5Orm
|
|
|
1071
1116
|
|
|
1072
1117
|
foreach (var row in rows)
|
|
1073
1118
|
{
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
try
|
|
1078
|
-
{
|
|
1079
|
-
if (rawVal is string s) vec = JsonSerializer.Deserialize<List<double>>(s);
|
|
1080
|
-
}
|
|
1081
|
-
catch { continue; }
|
|
1119
|
+
// A float32 BLOB, a legacy JSON text column or an already-decoded
|
|
1120
|
+
// array all decode the same way.
|
|
1121
|
+
var vec = SqliteVectors.DecodeVector(vectorProp.GetValue(row));
|
|
1082
1122
|
if (vec == null || vec.Count != vector.Count) continue;
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
{
|
|
1087
|
-
dot += vector[i] * vec[i];
|
|
1088
|
-
m1 += vector[i] * vector[i];
|
|
1089
|
-
m2 += vec[i] * vec[i];
|
|
1090
|
-
}
|
|
1091
|
-
double cosine = (m1 > 0 && m2 > 0) ? dot / (Math.Sqrt(m1) * Math.Sqrt(m2)) : 0;
|
|
1092
|
-
double dist = distanceMetric.Equals("cosine", StringComparison.OrdinalIgnoreCase) ? 1.0 - cosine :
|
|
1093
|
-
distanceMetric.Equals("dot", StringComparison.OrdinalIgnoreCase) ? -dot : EuclideanDistance(vector, vec);
|
|
1094
|
-
results.Add((row, dist));
|
|
1123
|
+
var dist = SqliteVectors.VectorDistance(vector, vec, distanceMetric);
|
|
1124
|
+
if (!dist.HasValue) continue;
|
|
1125
|
+
results.Add((row, dist.Value));
|
|
1095
1126
|
}
|
|
1096
1127
|
|
|
1097
1128
|
results.Sort((a, b) => a.Item2.CompareTo(b.Item2));
|
|
1098
1129
|
return results.GetRange(0, Math.Min(take, results.Count));
|
|
1099
1130
|
}
|
|
1100
1131
|
|
|
1132
|
+
/// <summary>
|
|
1133
|
+
/// Projects the ranked rows the in-database strategies return into the
|
|
1134
|
+
/// caller's entity type, taking `distance` from the added column.
|
|
1135
|
+
/// </summary>
|
|
1136
|
+
private List<(T, double)> ToRanked(List<Dictionary<string, object>> rows, int take)
|
|
1137
|
+
{
|
|
1138
|
+
var results = new List<(T, double)>();
|
|
1139
|
+
foreach (var row in rows)
|
|
1140
|
+
{
|
|
1141
|
+
var item = Activator.CreateInstance<T>();
|
|
1142
|
+
foreach (var prop in typeof(T).GetProperties(BindingFlags.Public | BindingFlags.Instance))
|
|
1143
|
+
{
|
|
1144
|
+
if (!row.TryGetValue(prop.Name, out var value) || value == null) continue;
|
|
1145
|
+
if (prop.Name.Equals("Distance", StringComparison.OrdinalIgnoreCase))
|
|
1146
|
+
{
|
|
1147
|
+
if (value is double d) prop.SetValue(item, d);
|
|
1148
|
+
continue;
|
|
1149
|
+
}
|
|
1150
|
+
// A vector column arrives as float32 bytes; the entity declares
|
|
1151
|
+
// it as float[].
|
|
1152
|
+
if (prop.PropertyType == typeof(float[]))
|
|
1153
|
+
{
|
|
1154
|
+
var vec = SqliteVectors.DecodeVector(value);
|
|
1155
|
+
if (vec != null)
|
|
1156
|
+
{
|
|
1157
|
+
prop.SetValue(item, vec.Select(v => (float)v).ToArray());
|
|
1158
|
+
continue;
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
try { prop.SetValue(item, Convert.ChangeType(value, prop.PropertyType)); } catch { }
|
|
1162
|
+
}
|
|
1163
|
+
double distance = row.TryGetValue("distance", out var raw) && raw != null
|
|
1164
|
+
? Convert.ToDouble(raw)
|
|
1165
|
+
: 0;
|
|
1166
|
+
results.Add((item, distance));
|
|
1167
|
+
if (results.Count >= take) break;
|
|
1168
|
+
}
|
|
1169
|
+
return results;
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1101
1172
|
private static double EuclideanDistance(List<double> left, List<double> right)
|
|
1102
1173
|
{
|
|
1103
1174
|
double sum = 0;
|
package/dotnet/Base/Core.cs
CHANGED
|
@@ -18,6 +18,17 @@ public enum Dialect
|
|
|
18
18
|
public string ConnectionString { get; set; }
|
|
19
19
|
public int CommandTimeout { get; set; } = 60;
|
|
20
20
|
public int ConnectRetryCount { get; set; } = 3;
|
|
21
|
+
/// <summary>
|
|
22
|
+
/// SQLite only: path to the sqlite-vec extension binary. Loaded on every
|
|
23
|
+
/// connection so vector search ranks inside the database. Other
|
|
24
|
+
/// providers ignore it.
|
|
25
|
+
/// </summary>
|
|
26
|
+
public string SqliteVec { get; set; }
|
|
27
|
+
/// <summary>
|
|
28
|
+
/// SQLite only: pin one of "sqlite-vec" | "udf" | "sql" | "memory"
|
|
29
|
+
/// instead of probing for the fastest strategy available.
|
|
30
|
+
/// </summary>
|
|
31
|
+
public string VectorStrategy { get; set; }
|
|
21
32
|
}
|
|
22
33
|
|
|
23
34
|
// ─── Query Engine Interface ────────────────────────────────────────────────
|
|
@@ -29,6 +40,9 @@ public enum Dialect
|
|
|
29
40
|
List<T> QueryRaw<T>(string sql, Dictionary<string, object> parameters) where T : new();
|
|
30
41
|
int ExecuteRaw(string sql, Dictionary<string, object> parameters);
|
|
31
42
|
An5TransactionBase BeginTransaction();
|
|
43
|
+
|
|
44
|
+
/// <summary>SQLite only: extension loading and distance-function registration.</summary>
|
|
45
|
+
SqliteVectorSupport VectorSupport { get; }
|
|
32
46
|
}
|
|
33
47
|
|
|
34
48
|
// ─── Transaction base ──────────────────────────────────────────────────────
|