@an5/adapters 0.2.11 → 0.2.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +137 -5
- package/dist/an5Adapter.d.ts +42 -1
- package/dist/an5Adapter.js +146 -38
- package/dist/base/types.d.ts +14 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +8 -1
- package/dist/sqlite/browserEngine.d.ts +13 -0
- package/dist/sqlite/browserEngine.js +13 -0
- package/dist/sqlite/engine.d.ts +5 -0
- package/dist/sqlite/engine.js +10 -0
- package/dist/sqlite/index.d.ts +1 -0
- package/dist/sqlite/index.js +1 -0
- package/dist/sqlite/vector.d.ts +155 -0
- package/dist/sqlite/vector.js +432 -0
- package/dotnet/An5Adapter.cs +99 -28
- package/dotnet/Base/Core.cs +14 -0
- package/dotnet/Mssql/MssqlEngine.cs +44 -1
- package/dotnet/Postgres/PostgresEngine.cs +2 -0
- package/dotnet/Sqlite/SqliteEngine.cs +82 -2
- package/dotnet/Sqlite/SqliteVectors.cs +401 -0
- package/golang/an5_adapter.go +84 -32
- package/golang/base/vectors.go +263 -0
- package/golang/sqlite/vector.go +248 -0
- package/golang/sqlite_vector_test.go +288 -0
- package/golang/table_client.go +67 -3
- package/java/pom.xml +177 -0
- package/java/src/main/java/an5/adapters/An5Adapter.java +374 -0
- package/java/src/main/java/an5/adapters/An5Aggregate.java +116 -0
- package/java/src/main/java/an5/adapters/An5GroupBy.java +97 -0
- package/java/src/main/java/an5/adapters/An5Query.java +100 -0
- package/java/src/main/java/an5/adapters/An5TableClient.java +1030 -0
- package/java/src/main/java/an5/adapters/An5ViewClient.java +107 -0
- package/java/src/main/java/an5/adapters/base/Dialect.java +103 -0
- package/java/src/main/java/an5/adapters/base/FieldMeta.java +42 -0
- package/java/src/main/java/an5/adapters/base/JdbcUrls.java +201 -0
- package/java/src/main/java/an5/adapters/base/Metadata.java +195 -0
- package/java/src/main/java/an5/adapters/base/RelationDef.java +26 -0
- package/java/src/main/java/an5/adapters/base/SqlBuilder.java +422 -0
- package/java/src/main/java/an5/adapters/base/SqliteVectors.java +375 -0
- package/java/src/main/java/an5/adapters/base/Vectors.java +123 -0
- package/java/src/test/java/An5AdaptersSmoke.java +733 -0
- package/java/src/test/java/Json.java +174 -0
- package/java/src/test/java/QuerySemantics.java +111 -0
- package/kotlin/build.gradle.kts +145 -0
- package/kotlin/settings.gradle.kts +10 -0
- package/kotlin/src/main/kotlin/an5/adapters/Aggregate.kt +89 -0
- package/kotlin/src/main/kotlin/an5/adapters/An5.kt +91 -0
- package/kotlin/src/main/kotlin/an5/adapters/Query.kt +130 -0
- package/kotlin/src/main/kotlin/an5/adapters/Rows.kt +166 -0
- package/kotlin/src/main/kotlin/an5/adapters/TableClient.kt +169 -0
- package/kotlin/src/main/kotlin/an5/adapters/Types.kt +77 -0
- package/kotlin/src/test/kotlin/an5/adapters/An5Smoke.kt +329 -0
- package/native/sqlite/CMakeLists.txt +12 -0
- package/native/sqlite/README.md +117 -0
- package/native/sqlite/an5_vector.c +209 -0
- package/package.json +33 -7
- package/python/an5_adapter.py +37 -6
- package/python/base/__init__.py +15 -0
- package/python/base/vectors.py +431 -0
- package/python/sqlite/__init__.py +2 -2
- package/python/sqlite/provider.py +21 -1
- package/python/table_client.py +129 -59
- package/rust/src/adapter.rs +342 -24
- package/rust/src/base/mod.rs +7 -0
- package/rust/src/base/vectors.rs +293 -0
- package/rust/src/lib.rs +3 -3
- package/scripts/sqlite-native-build.js +32 -0
- package/swift/Package.swift +22 -0
- package/swift/Sources/An5Adapters/An5Adapter.swift +160 -0
- package/swift/Sources/An5Adapters/Metadata.swift +183 -0
- package/swift/Sources/An5Adapters/Query.swift +105 -0
- package/swift/Sources/An5Adapters/Row+Accessors.swift +129 -0
- package/swift/Sources/An5Adapters/SQLBuilder.swift +359 -0
- package/swift/Sources/An5Adapters/SQLiteDriver.swift +301 -0
- package/swift/Sources/An5Adapters/SqliteVectors.swift +243 -0
- package/swift/Sources/An5Adapters/TableClient.swift +671 -0
- package/swift/Sources/An5Adapters/Types.swift +127 -0
- package/swift/Sources/An5Adapters/Vectors.swift +86 -0
- package/swift/Sources/An5Adapters/ViewClient.swift +50 -0
- package/swift/Sources/CSQLite/module.modulemap +8 -0
- package/swift/Sources/CSQLite/shim.h +9 -0
- package/swift/Tests/An5AdaptersTests/QuerySemanticsTests.swift +51 -0
- package/swift/Tests/An5AdaptersTests/RuntimeSmokeTests.swift +488 -0
- package/typescript/src/an5Adapter.ts +155 -31
- package/typescript/src/base/types.ts +15 -0
- package/typescript/src/index.ts +13 -0
- package/typescript/src/sqlite/browserEngine.ts +24 -0
- package/typescript/src/sqlite/engine.ts +13 -0
- package/typescript/src/sqlite/index.ts +1 -1
- package/typescript/src/sqlite/vector.ts +537 -0
- package/golang/sqlite/sqlite.go +0 -14
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
using System;
|
|
2
2
|
using System.Collections.Generic;
|
|
3
3
|
using System.Data;
|
|
4
|
+
using System.Globalization;
|
|
4
5
|
using System.Reflection;
|
|
5
6
|
using Microsoft.Data.SqlClient;
|
|
6
7
|
|
|
@@ -11,6 +12,8 @@ namespace An5Orm
|
|
|
11
12
|
internal class MssqlEngine : IQueryEngine
|
|
12
13
|
{
|
|
13
14
|
public Dialect Dialect => Dialect.Mssql;
|
|
15
|
+
// Only SQLite has a vector hook; this provider has none to offer.
|
|
16
|
+
public SqliteVectorSupport VectorSupport => null;
|
|
14
17
|
private readonly string _connectionString;
|
|
15
18
|
private readonly int _commandTimeout;
|
|
16
19
|
|
|
@@ -32,6 +35,33 @@ namespace An5Orm
|
|
|
32
35
|
return conn;
|
|
33
36
|
}
|
|
34
37
|
|
|
38
|
+
/// <summary>
|
|
39
|
+
/// Converts a value into something SqlClient can bind.
|
|
40
|
+
/// </summary>
|
|
41
|
+
/// <remarks>
|
|
42
|
+
/// A <c>VECTOR</c> column is transmitted as a JSON array over TDS, so a float or
|
|
43
|
+
/// double array is written in that form. Binding the array itself is not an option:
|
|
44
|
+
/// SqlClient has no mapping for it, and the failure names the array rather than the
|
|
45
|
+
/// column. Driver versions before 6.1 have no <c>SqlVector</c> either, so the JSON
|
|
46
|
+
/// form is what every version accepts.
|
|
47
|
+
/// </remarks>
|
|
48
|
+
private static object BindableValue(object value)
|
|
49
|
+
{
|
|
50
|
+
if (value is float[] floats)
|
|
51
|
+
{
|
|
52
|
+
var parts = new string[floats.Length];
|
|
53
|
+
for (int i = 0; i < floats.Length; i++) parts[i] = floats[i].ToString("R", CultureInfo.InvariantCulture);
|
|
54
|
+
return "[" + string.Join(", ", parts) + "]";
|
|
55
|
+
}
|
|
56
|
+
if (value is double[] doubles)
|
|
57
|
+
{
|
|
58
|
+
var parts = new string[doubles.Length];
|
|
59
|
+
for (int i = 0; i < doubles.Length; i++) parts[i] = doubles[i].ToString("R", CultureInfo.InvariantCulture);
|
|
60
|
+
return "[" + string.Join(", ", parts) + "]";
|
|
61
|
+
}
|
|
62
|
+
return value;
|
|
63
|
+
}
|
|
64
|
+
|
|
35
65
|
private SqlCommand BuildCommand(
|
|
36
66
|
SqlConnection conn, string sql, Dictionary<string, object> parameters)
|
|
37
67
|
{
|
|
@@ -42,7 +72,7 @@ namespace An5Orm
|
|
|
42
72
|
foreach (var kv in parameters)
|
|
43
73
|
{
|
|
44
74
|
var paramName = kv.Key.StartsWith("@") ? kv.Key : "@" + kv.Key;
|
|
45
|
-
cmd.Parameters.AddWithValue(paramName, kv.Value ?? DBNull.Value);
|
|
75
|
+
cmd.Parameters.AddWithValue(paramName, BindableValue(kv.Value) ?? DBNull.Value);
|
|
46
76
|
}
|
|
47
77
|
}
|
|
48
78
|
return cmd;
|
|
@@ -87,6 +117,19 @@ namespace An5Orm
|
|
|
87
117
|
if (!HasColumn(reader, prop.Name)) continue;
|
|
88
118
|
var val = reader[prop.Name];
|
|
89
119
|
if (val == DBNull.Value) continue;
|
|
120
|
+
// A `VECTOR` column arrives as a JSON array, which no converter
|
|
121
|
+
// bridges to the `float[]` the entity declares.
|
|
122
|
+
if (prop.PropertyType == typeof(float[]))
|
|
123
|
+
{
|
|
124
|
+
var vector = SqliteVectors.DecodeVector(val);
|
|
125
|
+
if (vector != null)
|
|
126
|
+
{
|
|
127
|
+
var floats = new float[vector.Count];
|
|
128
|
+
for (int i = 0; i < vector.Count; i++) floats[i] = (float)vector[i];
|
|
129
|
+
prop.SetValue(item, floats);
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
90
133
|
try { prop.SetValue(item, Convert.ChangeType(val, prop.PropertyType)); } catch { }
|
|
91
134
|
}
|
|
92
135
|
results.Add(item);
|
|
@@ -10,6 +10,8 @@ namespace An5Orm
|
|
|
10
10
|
internal class PostgresEngine : IQueryEngine
|
|
11
11
|
{
|
|
12
12
|
public Dialect Dialect => Dialect.Postgres;
|
|
13
|
+
// Only SQLite has a vector hook; this provider has none to offer.
|
|
14
|
+
public SqliteVectorSupport VectorSupport => null;
|
|
13
15
|
private readonly string _connectionString;
|
|
14
16
|
private readonly int _commandTimeout;
|
|
15
17
|
|
|
@@ -18,13 +18,15 @@ namespace An5Orm
|
|
|
18
18
|
internal class SqliteEngine : IQueryEngine
|
|
19
19
|
{
|
|
20
20
|
public Dialect Dialect => Dialect.Sqlite;
|
|
21
|
+
public SqliteVectorSupport VectorSupport { get; }
|
|
21
22
|
private readonly string _connectionString;
|
|
22
23
|
private readonly int _commandTimeout;
|
|
23
24
|
|
|
24
|
-
public SqliteEngine(string connectionString, int commandTimeout)
|
|
25
|
+
public SqliteEngine(string connectionString, int commandTimeout, SqliteVectorSupport vectorSupport = null)
|
|
25
26
|
{
|
|
26
27
|
_connectionString = SqliteConnectionString.Normalize(connectionString);
|
|
27
28
|
_commandTimeout = commandTimeout;
|
|
29
|
+
VectorSupport = vectorSupport ?? new SqliteVectorSupport();
|
|
28
30
|
}
|
|
29
31
|
|
|
30
32
|
// Set while a transaction is open; statements then join it instead of
|
|
@@ -40,9 +42,71 @@ namespace An5Orm
|
|
|
40
42
|
using var pragma = conn.CreateCommand();
|
|
41
43
|
pragma.CommandText = "PRAGMA journal_mode=WAL; PRAGMA foreign_keys=ON;";
|
|
42
44
|
pragma.ExecuteNonQuery();
|
|
45
|
+
PrepareVectorSearch(conn);
|
|
43
46
|
return conn;
|
|
44
47
|
}
|
|
45
48
|
|
|
49
|
+
/// <summary>
|
|
50
|
+
/// Registers the distance functions and loads sqlite-vec so a `VECTOR(n)`
|
|
51
|
+
/// column is ranked by SQLite instead of by the client. Neither is
|
|
52
|
+
/// required: a connection that refuses both falls back to `json_each`, or
|
|
53
|
+
/// to scoring in C#.
|
|
54
|
+
///
|
|
55
|
+
/// This runs per connection because the engine opens one per statement,
|
|
56
|
+
/// and a user function only exists for the connection that declared it.
|
|
57
|
+
/// </summary>
|
|
58
|
+
private void PrepareVectorSearch(Microsoft.Data.Sqlite.SqliteConnection conn)
|
|
59
|
+
{
|
|
60
|
+
var support = VectorSupport;
|
|
61
|
+
if (support == null) return;
|
|
62
|
+
|
|
63
|
+
var hasVec = false;
|
|
64
|
+
if (!string.IsNullOrWhiteSpace(support.SqliteVecPath))
|
|
65
|
+
{
|
|
66
|
+
try { conn.LoadExtension(support.SqliteVecPath); }
|
|
67
|
+
catch { /* the extension is optional */ }
|
|
68
|
+
}
|
|
69
|
+
if (!hasVec)
|
|
70
|
+
{
|
|
71
|
+
try
|
|
72
|
+
{
|
|
73
|
+
using var probe = conn.CreateCommand();
|
|
74
|
+
probe.CommandText = "SELECT vec_version()";
|
|
75
|
+
probe.ExecuteScalar();
|
|
76
|
+
hasVec = true;
|
|
77
|
+
}
|
|
78
|
+
catch { /* not loaded */ }
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
var hasUdf = false;
|
|
82
|
+
try {
|
|
83
|
+
using var native = conn.CreateCommand();
|
|
84
|
+
native.CommandText = "SELECT an5_vector_version()";
|
|
85
|
+
native.ExecuteScalar();
|
|
86
|
+
support.Report(false, true, true);
|
|
87
|
+
return;
|
|
88
|
+
} catch { /* no AN5 native extension */ }
|
|
89
|
+
try
|
|
90
|
+
{
|
|
91
|
+
foreach (var metric in SqliteVectors.An5Functions.Keys)
|
|
92
|
+
{
|
|
93
|
+
var name = SqliteVectors.An5Functions[metric];
|
|
94
|
+
var distance = SqliteVectors.MakeDistanceFunction(metric);
|
|
95
|
+
// The metric is baked into `distance`; the empty state is
|
|
96
|
+
// only there to satisfy the delegate shape.
|
|
97
|
+
conn.CreateFunction<object[], object>(
|
|
98
|
+
name,
|
|
99
|
+
Array.Empty<object>(),
|
|
100
|
+
(_state, args) => distance(args),
|
|
101
|
+
true);
|
|
102
|
+
}
|
|
103
|
+
hasUdf = true;
|
|
104
|
+
}
|
|
105
|
+
catch { /* the bundle may not allow user functions */ }
|
|
106
|
+
|
|
107
|
+
support.Report(hasVec, hasUdf, true);
|
|
108
|
+
}
|
|
109
|
+
|
|
46
110
|
private ConnectionScope<Microsoft.Data.Sqlite.SqliteConnection> Acquire()
|
|
47
111
|
=> _txnConn != null
|
|
48
112
|
? new ConnectionScope<Microsoft.Data.Sqlite.SqliteConnection>(_txnConn, owned: false)
|
|
@@ -67,7 +131,9 @@ namespace An5Orm
|
|
|
67
131
|
param.ParameterName = paramName;
|
|
68
132
|
// A JSON vector or a bool needs a type SQLite will store in a
|
|
69
133
|
// column rather than coerce to an integer.
|
|
70
|
-
if (value is
|
|
134
|
+
if (value is float[] || value is double[] || value is List<double>)
|
|
135
|
+
param.Value = SqliteVectors.EncodeVector(SqliteVectors.DecodeVector(value));
|
|
136
|
+
else if (value is bool flag) param.Value = flag ? 1 : 0;
|
|
71
137
|
else param.Value = value;
|
|
72
138
|
cmd.Parameters.Add(param);
|
|
73
139
|
}
|
|
@@ -108,6 +174,20 @@ namespace An5Orm
|
|
|
108
174
|
if (!HasColumn(reader, prop.Name)) continue;
|
|
109
175
|
var val = reader[prop.Name];
|
|
110
176
|
if (val == DBNull.Value) continue;
|
|
177
|
+
// A `VECTOR(n)` column stores a float32 BLOB (and older ones a
|
|
178
|
+
// JSON text column) while the entity declares it as float[],
|
|
179
|
+
// which no converter bridges.
|
|
180
|
+
if (prop.PropertyType == typeof(float[]))
|
|
181
|
+
{
|
|
182
|
+
var vector = SqliteVectors.DecodeVector(val);
|
|
183
|
+
if (vector != null)
|
|
184
|
+
{
|
|
185
|
+
var floats = new float[vector.Count];
|
|
186
|
+
for (int i = 0; i < vector.Count; i++) floats[i] = (float)vector[i];
|
|
187
|
+
prop.SetValue(item, floats);
|
|
188
|
+
continue;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
111
191
|
try { prop.SetValue(item, Convert.ChangeType(val, prop.PropertyType)); } catch { }
|
|
112
192
|
}
|
|
113
193
|
results.Add(item);
|
|
@@ -0,0 +1,401 @@
|
|
|
1
|
+
using System;
|
|
2
|
+
using System.Collections.Generic;
|
|
3
|
+
using System.Linq;
|
|
4
|
+
using System.Text.Json;
|
|
5
|
+
|
|
6
|
+
namespace An5Orm
|
|
7
|
+
{
|
|
8
|
+
// ─── SQLite vector search ────────────────────────────────────────────────────
|
|
9
|
+
//
|
|
10
|
+
// SQLite has no vector type, so a `VECTOR(n)` column stores a BLOB of
|
|
11
|
+
// little-endian float32 values (see `EncodeVector`) and is ranked in one of
|
|
12
|
+
// four ways, in this order:
|
|
13
|
+
//
|
|
14
|
+
// 1. sqlite-vec native scalar distances, when the extension loads.
|
|
15
|
+
// 2. udf an5_vec_cosine / an5_vec_l2 / an5_vec_ip registered with
|
|
16
|
+
// the connection. Reads the BLOB and the legacy JSON text.
|
|
17
|
+
// 3. sql json_each brute force in plain SQL. Needs no user
|
|
18
|
+
// function, but only reaches rows stored as JSON text.
|
|
19
|
+
// 4. memory the column is loaded and scored in C#.
|
|
20
|
+
//
|
|
21
|
+
// Strategies 1-3 rank inside the database and transfer only the matching
|
|
22
|
+
// rows, which is why they are preferred over the in-memory path.
|
|
23
|
+
|
|
24
|
+
public static class SqliteVectors
|
|
25
|
+
{
|
|
26
|
+
/// <summary>Scalar functions the adapter registers itself.</summary>
|
|
27
|
+
public static readonly Dictionary<string, string> An5Functions = new()
|
|
28
|
+
{
|
|
29
|
+
["cosine"] = "an5_vec_cosine",
|
|
30
|
+
["euclidean"] = "an5_vec_l2",
|
|
31
|
+
["dot"] = "an5_vec_ip",
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
/// <summary>Scalar functions shipped by the sqlite-vec extension.</summary>
|
|
35
|
+
public static readonly Dictionary<string, string> SqliteVecFunctions = new()
|
|
36
|
+
{
|
|
37
|
+
["cosine"] = "vec_distance_cosine",
|
|
38
|
+
["euclidean"] = "vec_distance_l2",
|
|
39
|
+
["dot"] = "vec_distance_ip",
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
public const string StrategySqliteVec = "sqlite-vec";
|
|
43
|
+
public const string StrategyUdf = "udf";
|
|
44
|
+
public const string StrategySql = "sql";
|
|
45
|
+
public const string StrategyMemory = "memory";
|
|
46
|
+
|
|
47
|
+
private const int BytesPerFloat = 4;
|
|
48
|
+
|
|
49
|
+
public static string NormalizeMetric(string metric)
|
|
50
|
+
=> metric == "euclidean" || metric == "dot" ? metric : "cosine";
|
|
51
|
+
|
|
52
|
+
// ── Codec ───────────────────────────────────────────────────────────────
|
|
53
|
+
|
|
54
|
+
/// <summary>
|
|
55
|
+
/// Reads a numeric array out of the shapes a driver can hand back: a
|
|
56
|
+
/// list, a JSON text column, or raw float32 bytes.
|
|
57
|
+
/// </summary>
|
|
58
|
+
public static List<double> ParseVector(object value)
|
|
59
|
+
{
|
|
60
|
+
switch (value)
|
|
61
|
+
{
|
|
62
|
+
case null:
|
|
63
|
+
return null;
|
|
64
|
+
case List<double> list:
|
|
65
|
+
return list.Count > 0 ? list : null;
|
|
66
|
+
case double[] array:
|
|
67
|
+
return array.Length > 0 ? array.ToList() : null;
|
|
68
|
+
case float[] floats:
|
|
69
|
+
return floats.Length > 0 ? floats.Select(f => (double)f).ToList() : null;
|
|
70
|
+
case byte[] bytes:
|
|
71
|
+
return DecodeVector(bytes);
|
|
72
|
+
case string text:
|
|
73
|
+
var trimmed = text.Trim();
|
|
74
|
+
if (!trimmed.StartsWith("[")) return null;
|
|
75
|
+
try { return ParseVector(JsonSerializer.Deserialize<double[]>(trimmed)); }
|
|
76
|
+
catch { return null; }
|
|
77
|
+
case System.Collections.IEnumerable seq when value is not string:
|
|
78
|
+
try { return ParseVector(seq.Cast<object>().Select(o => Convert.ToDouble(o)).ToList()); }
|
|
79
|
+
catch { return null; }
|
|
80
|
+
default:
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/// <summary>
|
|
86
|
+
/// Decodes a stored vector: a float32 BLOB, a legacy JSON text column,
|
|
87
|
+
/// or an array the caller already decoded.
|
|
88
|
+
/// </summary>
|
|
89
|
+
public static List<double> DecodeVector(object value)
|
|
90
|
+
{
|
|
91
|
+
var direct = value is byte[] ? null : ParseVector(value);
|
|
92
|
+
if (direct != null) return direct;
|
|
93
|
+
if (!(value is byte[] bytes) || bytes.Length == 0 || bytes.Length % BytesPerFloat != 0) return null;
|
|
94
|
+
var outVec = new List<double>(bytes.Length / BytesPerFloat);
|
|
95
|
+
for (int i = 0; i < bytes.Length; i += BytesPerFloat)
|
|
96
|
+
outVec.Add(BitConverter.ToSingle(bytes, i));
|
|
97
|
+
return outVec;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/// <summary>Encodes a vector as the little-endian float32 BLOB the column stores.</summary>
|
|
101
|
+
public static byte[] EncodeVector(IReadOnlyList<double> values)
|
|
102
|
+
{
|
|
103
|
+
var bytes = new byte[values.Count * BytesPerFloat];
|
|
104
|
+
for (int i = 0; i < values.Count; i++)
|
|
105
|
+
{
|
|
106
|
+
// Little-endian, written through the integer view of the float:
|
|
107
|
+
// SQLite reads the bytes in memory order.
|
|
108
|
+
var bits = BitConverter.SingleToInt32Bits((float)values[i]);
|
|
109
|
+
bytes[i * BytesPerFloat] = (byte)(bits & 0xFF);
|
|
110
|
+
bytes[i * BytesPerFloat + 1] = (byte)((bits >> 8) & 0xFF);
|
|
111
|
+
bytes[i * BytesPerFloat + 2] = (byte)((bits >> 16) & 0xFF);
|
|
112
|
+
bytes[i * BytesPerFloat + 3] = (byte)((bits >> 24) & 0xFF);
|
|
113
|
+
}
|
|
114
|
+
return bytes;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/// <summary>True when the value still has to become bytes on its way into a column.</summary>
|
|
118
|
+
public static bool NeedsVectorEncoding(object value)
|
|
119
|
+
{
|
|
120
|
+
if (value == null || value is string || value is byte[]) return false;
|
|
121
|
+
if (value is System.Collections.IEnumerable seq) return !(value is System.Collections.IDictionary);
|
|
122
|
+
return false;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/// <summary>True when generated metadata describes a `VECTOR(n)` column.</summary>
|
|
126
|
+
public static bool IsVectorField(object definition)
|
|
127
|
+
{
|
|
128
|
+
if (definition == null) return false;
|
|
129
|
+
if (definition is string text)
|
|
130
|
+
return text.Contains("[]") || text.TrimStart().StartsWith("VECTOR", StringComparison.OrdinalIgnoreCase);
|
|
131
|
+
|
|
132
|
+
// The metadata carries a small record per field.
|
|
133
|
+
var kind = ReadMember(definition, "kind") as string;
|
|
134
|
+
if (kind == "vector") return true;
|
|
135
|
+
var sql = ReadMember(definition, "sql") as string ?? ReadMember(definition, "type") as string;
|
|
136
|
+
if (!string.IsNullOrWhiteSpace(sql))
|
|
137
|
+
{
|
|
138
|
+
var trimmed = sql.Trim();
|
|
139
|
+
if (trimmed.StartsWith("VECTOR", StringComparison.OrdinalIgnoreCase)
|
|
140
|
+
&& (trimmed.Length == 6 || trimmed[6] == '(' || trimmed[6] == ' '))
|
|
141
|
+
return true;
|
|
142
|
+
}
|
|
143
|
+
var ts = ReadMember(definition, "ts") as string;
|
|
144
|
+
return ts != null && ts.Contains("[]");
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
private static object ReadMember(object target, string name)
|
|
148
|
+
{
|
|
149
|
+
if (target is System.Collections.IDictionary dict)
|
|
150
|
+
return dict.Contains(name) ? dict[name] : null;
|
|
151
|
+
var prop = target.GetType().GetProperty(name);
|
|
152
|
+
return prop != null ? prop.GetValue(target) : null;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// ── Distance ────────────────────────────────────────────────────────────
|
|
156
|
+
|
|
157
|
+
/// <summary>
|
|
158
|
+
/// Distance between two equal-length vectors, lower is closer. Mirrors
|
|
159
|
+
/// the SQL functions so a search ranks the same way whichever strategy ran.
|
|
160
|
+
/// </summary>
|
|
161
|
+
public static double? VectorDistance(IReadOnlyList<double> a, IReadOnlyList<double> b, string metric)
|
|
162
|
+
{
|
|
163
|
+
if (a == null || b == null || a.Count == 0 || a.Count != b.Count) return null;
|
|
164
|
+
var resolved = NormalizeMetric(metric);
|
|
165
|
+
if (resolved == "euclidean")
|
|
166
|
+
{
|
|
167
|
+
double sum = 0;
|
|
168
|
+
for (int i = 0; i < a.Count; i++)
|
|
169
|
+
{
|
|
170
|
+
var diff = a[i] - b[i];
|
|
171
|
+
sum += diff * diff;
|
|
172
|
+
}
|
|
173
|
+
return Math.Sqrt(sum);
|
|
174
|
+
}
|
|
175
|
+
double dot = 0, m1 = 0, m2 = 0;
|
|
176
|
+
for (int i = 0; i < a.Count; i++)
|
|
177
|
+
{
|
|
178
|
+
dot += a[i] * b[i];
|
|
179
|
+
m1 += a[i] * a[i];
|
|
180
|
+
m2 += b[i] * b[i];
|
|
181
|
+
}
|
|
182
|
+
if (resolved == "dot") return -dot;
|
|
183
|
+
return m1 > 0 && m2 > 0 ? 1.0 - dot / (Math.Sqrt(m1) * Math.Sqrt(m2)) : 1.0;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/// <summary>The scalar function registered with the driver for one metric.</summary>
|
|
187
|
+
public static Func<object[], object> MakeDistanceFunction(string metric)
|
|
188
|
+
=> args =>
|
|
189
|
+
{
|
|
190
|
+
var a = DecodeVector(args.Length > 0 ? args[0] : null);
|
|
191
|
+
var b = DecodeVector(args.Length > 1 ? args[1] : null);
|
|
192
|
+
if (a == null || b == null) return DBNull.Value;
|
|
193
|
+
var distance = VectorDistance(a, b, metric);
|
|
194
|
+
return distance.HasValue ? (object)distance.Value : DBNull.Value;
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
// ── Query building ──────────────────────────────────────────────────────
|
|
198
|
+
|
|
199
|
+
/// <summary>`json_valid` fails on a float32 BLOB, so the JSON path needs this guard.</summary>
|
|
200
|
+
private static string JsonText(string column)
|
|
201
|
+
=> $"CASE WHEN json_valid({column}) THEN {column} ELSE '[]' END";
|
|
202
|
+
|
|
203
|
+
private static string JsonDistanceExpr(string metric, string column)
|
|
204
|
+
{
|
|
205
|
+
var row = JsonText(column);
|
|
206
|
+
var sameLength = $"(SELECT COUNT(*) FROM json_each({row})) = (SELECT COUNT(*) FROM q)";
|
|
207
|
+
string terms;
|
|
208
|
+
switch (metric)
|
|
209
|
+
{
|
|
210
|
+
case "euclidean":
|
|
211
|
+
terms = $"sqrt((SELECT SUM((je.value - q.v) * (je.value - q.v)) FROM json_each({row}) je JOIN q ON q.k = je.key))";
|
|
212
|
+
break;
|
|
213
|
+
case "dot":
|
|
214
|
+
terms = $"-(SELECT SUM(je.value * q.v) FROM json_each({row}) je JOIN q ON q.k = je.key)";
|
|
215
|
+
break;
|
|
216
|
+
default:
|
|
217
|
+
terms = $"1.0 - (SELECT SUM(je.value * q.v) FROM json_each({row}) je JOIN q ON q.k = je.key)"
|
|
218
|
+
+ $" / NULLIF(sqrt((SELECT SUM(q.v * q.v) FROM q)) * sqrt((SELECT SUM(je.value * je.value) FROM json_each({row}) je)), 0)";
|
|
219
|
+
break;
|
|
220
|
+
}
|
|
221
|
+
return $"CASE WHEN {sameLength} THEN {terms} END";
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/// <summary>
|
|
225
|
+
/// Ranking SQL for one strategy, with the parameters it needs appended to
|
|
226
|
+
/// `parameters`. `sqlite-vec` and `udf` bind the query vector as a float32
|
|
227
|
+
/// BLOB; `sql` binds it as JSON text because `json_each` reads text.
|
|
228
|
+
/// </summary>
|
|
229
|
+
public static string BuildQuery(
|
|
230
|
+
string strategy, string metric, string table, string column,
|
|
231
|
+
IReadOnlyList<double> vector, int take, string tail,
|
|
232
|
+
Dictionary<string, object> parameters)
|
|
233
|
+
{
|
|
234
|
+
metric = NormalizeMetric(metric);
|
|
235
|
+
// A row whose stored vector cannot be scored (a NULL, a text column,
|
|
236
|
+
// another dimension) yields a NULL distance; those rows are dropped
|
|
237
|
+
// rather than reported, so a caller never has to sort them out.
|
|
238
|
+
string Rank(string inner)
|
|
239
|
+
=> $"SELECT * FROM ({inner}) AS an5_ranked WHERE distance IS NOT NULL ORDER BY distance ASC LIMIT {take}";
|
|
240
|
+
|
|
241
|
+
if (strategy == StrategySqliteVec)
|
|
242
|
+
{
|
|
243
|
+
parameters["an5_vector"] = EncodeVector(vector);
|
|
244
|
+
parameters["an5_vector_bytes"] = vector.Count * BytesPerFloat;
|
|
245
|
+
// sqlite-vec only understands float32 BLOB operands, so rows stored
|
|
246
|
+
// any other way are excluded instead of aborting the query.
|
|
247
|
+
return Rank($"SELECT *, CASE WHEN vec_length(vec_f32({column})) = {vector.Count}"
|
|
248
|
+
+ $" THEN {SqliteVecFunctions[metric]}(vec_f32({column}), @an5_vector) END AS distance FROM {table}"
|
|
249
|
+
+ (string.IsNullOrWhiteSpace(tail) ? " WHERE " : tail + " AND ")
|
|
250
|
+
+ $"{column} IS NOT NULL");
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
if (strategy == StrategyUdf)
|
|
254
|
+
{
|
|
255
|
+
parameters["an5_vector"] = EncodeVector(vector);
|
|
256
|
+
return Rank($"SELECT *, {An5Functions[metric]}({column}, @an5_vector) AS distance FROM {table}{tail}");
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
parameters["an5_vector_json"] = JsonSerializer.Serialize(vector);
|
|
260
|
+
return "WITH q AS (SELECT je.key AS k, CAST(je.value AS REAL) AS v FROM json_each(@an5_vector_json) je) "
|
|
261
|
+
+ Rank($"SELECT *, {JsonDistanceExpr(metric, column)} AS distance FROM {table}{tail}");
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// ── Capabilities ────────────────────────────────────────────────────────
|
|
265
|
+
|
|
266
|
+
/// <summary>
|
|
267
|
+
/// The ordered strategies to try. `declaredType` is the column's DDL type:
|
|
268
|
+
/// the JSON strategy only reaches rows stored as text, so a `BLOB` column
|
|
269
|
+
/// drops it from the order.
|
|
270
|
+
/// </summary>
|
|
271
|
+
public static List<string> PlanStrategies(
|
|
272
|
+
bool hasVec, bool hasUdf, bool hasJson1, string declaredType, string preference)
|
|
273
|
+
{
|
|
274
|
+
if (!string.IsNullOrWhiteSpace(preference) && preference != "auto")
|
|
275
|
+
return preference == StrategyMemory ? new List<string>() : new List<string> { preference };
|
|
276
|
+
|
|
277
|
+
var available = new List<string>();
|
|
278
|
+
if (hasVec) available.Add(StrategySqliteVec);
|
|
279
|
+
if (hasUdf) available.Add(StrategyUdf);
|
|
280
|
+
if (hasJson1) available.Add(StrategySql);
|
|
281
|
+
|
|
282
|
+
var declared = (declaredType ?? "").ToUpperInvariant();
|
|
283
|
+
if (declared.Length > 0
|
|
284
|
+
&& !new[] { "TEXT", "CHAR", "CLOB", "STRING", "JSON" }.Any(t => declared.Contains(t)))
|
|
285
|
+
{
|
|
286
|
+
// A BLOB column has no JSON to read, so json_each can only produce NULLs.
|
|
287
|
+
available.Remove(StrategySql);
|
|
288
|
+
}
|
|
289
|
+
return available;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/// <summary>Reads a vector column's declared DDL type, or null when not found.</summary>
|
|
293
|
+
public static string ReadColumnType(
|
|
294
|
+
Func<string, Dictionary<string, object>, List<Dictionary<string, object>>> exec,
|
|
295
|
+
string table, string column)
|
|
296
|
+
{
|
|
297
|
+
try
|
|
298
|
+
{
|
|
299
|
+
var rows = exec(
|
|
300
|
+
"SELECT type FROM pragma_table_info(@an5_table) WHERE name = @an5_column",
|
|
301
|
+
new Dictionary<string, object> { ["an5_table"] = table, ["an5_column"] = column });
|
|
302
|
+
var value = rows.Count > 0 ? rows[0].GetValueOrDefault("type") : null;
|
|
303
|
+
return value == null ? null : value.ToString();
|
|
304
|
+
}
|
|
305
|
+
catch { return null; }
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
/// <summary>
|
|
309
|
+
/// Ranks a SQLite table inside the database. Returns null when the
|
|
310
|
+
/// connection offers no strategy, so the caller can fall back to scoring
|
|
311
|
+
/// in memory.
|
|
312
|
+
/// </summary>
|
|
313
|
+
public static List<Dictionary<string, object>> RunSearch(
|
|
314
|
+
Func<string, Dictionary<string, object>, List<Dictionary<string, object>>> exec,
|
|
315
|
+
string table, string rawTable, string column, string rawColumn,
|
|
316
|
+
IReadOnlyList<double> vector, string metric, int take, string tail,
|
|
317
|
+
Dictionary<string, object> parameters, SqliteVectorSupport support)
|
|
318
|
+
{
|
|
319
|
+
if (vector == null || vector.Count == 0) return null;
|
|
320
|
+
|
|
321
|
+
var caps = support?.Capabilities() ?? new SqliteVectorCapabilities { Json1 = true };
|
|
322
|
+
if (support != null) support.ProbeJson1(exec);
|
|
323
|
+
|
|
324
|
+
var declaredType = ReadColumnType(exec, rawTable, rawColumn);
|
|
325
|
+
foreach (var strategy in PlanStrategies(caps.Vec, caps.Udf, caps.Json1, declaredType, support?.Preference))
|
|
326
|
+
{
|
|
327
|
+
if (strategy == StrategySql) {
|
|
328
|
+
var binary = exec($"SELECT 1 FROM {table}" + (string.IsNullOrWhiteSpace(tail) ? " WHERE " : tail + " AND ")
|
|
329
|
+
+ $"typeof({column}) = 'blob' LIMIT 1", parameters);
|
|
330
|
+
if (binary.Count > 0) continue;
|
|
331
|
+
}
|
|
332
|
+
var bound = new Dictionary<string, object>();
|
|
333
|
+
foreach (var kv in parameters) bound[kv.Key] = kv.Value;
|
|
334
|
+
var sql = BuildQuery(strategy, metric, table, column, vector, take, tail, bound);
|
|
335
|
+
try
|
|
336
|
+
{
|
|
337
|
+
var rows = exec(sql, bound);
|
|
338
|
+
if (rows != null) return rows;
|
|
339
|
+
}
|
|
340
|
+
catch
|
|
341
|
+
{
|
|
342
|
+
// The connection advertised the capability but the query
|
|
343
|
+
// failed, e.g. an extension that did not really load. The
|
|
344
|
+
// next strategy is cheaper than reporting the failure.
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return null;
|
|
349
|
+
}
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
/// <summary>What a SQLite connection can do, resolved once and cached.</summary>
|
|
353
|
+
public class SqliteVectorCapabilities
|
|
354
|
+
{
|
|
355
|
+
/// <summary>`vec_version()` answered, so the sqlite-vec extension is loaded.</summary>
|
|
356
|
+
public bool Vec { get; set; }
|
|
357
|
+
/// <summary>The adapter's own distance functions are registered.</summary>
|
|
358
|
+
public bool Udf { get; set; }
|
|
359
|
+
/// <summary>`json_each()` is available, which the pure-SQL strategy needs.</summary>
|
|
360
|
+
public bool Json1 { get; set; }
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
/// <summary>
|
|
364
|
+
/// The settings one SQLite connection needs for in-database vector search.
|
|
365
|
+
/// Filled in by the engine, which knows how to open connections.
|
|
366
|
+
/// </summary>
|
|
367
|
+
public class SqliteVectorSupport
|
|
368
|
+
{
|
|
369
|
+
/// <summary>Path to the sqlite-vec extension binary to load.</summary>
|
|
370
|
+
public string SqliteVecPath { get; set; }
|
|
371
|
+
/// <summary>Pin a strategy instead of probing for the fastest one available.</summary>
|
|
372
|
+
public string Preference { get; set; }
|
|
373
|
+
|
|
374
|
+
private SqliteVectorCapabilities _capabilities;
|
|
375
|
+
|
|
376
|
+
public SqliteVectorCapabilities Capabilities()
|
|
377
|
+
=> _capabilities ??= new SqliteVectorCapabilities { Udf = true, Json1 = true };
|
|
378
|
+
|
|
379
|
+
/// <summary>Confirms JSON1 before a query is built that needs it.</summary>
|
|
380
|
+
public void ProbeJson1(Func<string, Dictionary<string, object>, List<Dictionary<string, object>>> exec)
|
|
381
|
+
{
|
|
382
|
+
var caps = Capabilities();
|
|
383
|
+
if (caps.Json1) return;
|
|
384
|
+
try
|
|
385
|
+
{
|
|
386
|
+
exec("SELECT json_valid('[1]') AS an5_json_ok", new Dictionary<string, object>());
|
|
387
|
+
caps.Json1 = true;
|
|
388
|
+
}
|
|
389
|
+
catch { caps.Json1 = false; }
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/// <summary>Records what the connection actually manages, after the engine tried.</summary>
|
|
393
|
+
internal void Report(bool vec, bool udf, bool json1)
|
|
394
|
+
{
|
|
395
|
+
var caps = Capabilities();
|
|
396
|
+
caps.Vec = vec;
|
|
397
|
+
caps.Udf = udf;
|
|
398
|
+
caps.Json1 = json1;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|