@an5/adapters 0.2.11 → 0.2.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +137 -5
  2. package/dist/an5Adapter.d.ts +42 -1
  3. package/dist/an5Adapter.js +146 -38
  4. package/dist/base/types.d.ts +14 -0
  5. package/dist/index.d.ts +2 -0
  6. package/dist/index.js +8 -1
  7. package/dist/sqlite/browserEngine.d.ts +13 -0
  8. package/dist/sqlite/browserEngine.js +13 -0
  9. package/dist/sqlite/engine.d.ts +5 -0
  10. package/dist/sqlite/engine.js +10 -0
  11. package/dist/sqlite/index.d.ts +1 -0
  12. package/dist/sqlite/index.js +1 -0
  13. package/dist/sqlite/vector.d.ts +155 -0
  14. package/dist/sqlite/vector.js +432 -0
  15. package/dotnet/An5Adapter.cs +99 -28
  16. package/dotnet/Base/Core.cs +14 -0
  17. package/dotnet/Mssql/MssqlEngine.cs +44 -1
  18. package/dotnet/Postgres/PostgresEngine.cs +2 -0
  19. package/dotnet/Sqlite/SqliteEngine.cs +82 -2
  20. package/dotnet/Sqlite/SqliteVectors.cs +401 -0
  21. package/golang/an5_adapter.go +84 -32
  22. package/golang/base/vectors.go +263 -0
  23. package/golang/sqlite/vector.go +248 -0
  24. package/golang/sqlite_vector_test.go +288 -0
  25. package/golang/table_client.go +67 -3
  26. package/java/pom.xml +177 -0
  27. package/java/src/main/java/an5/adapters/An5Adapter.java +374 -0
  28. package/java/src/main/java/an5/adapters/An5Aggregate.java +116 -0
  29. package/java/src/main/java/an5/adapters/An5GroupBy.java +97 -0
  30. package/java/src/main/java/an5/adapters/An5Query.java +100 -0
  31. package/java/src/main/java/an5/adapters/An5TableClient.java +1030 -0
  32. package/java/src/main/java/an5/adapters/An5ViewClient.java +107 -0
  33. package/java/src/main/java/an5/adapters/base/Dialect.java +103 -0
  34. package/java/src/main/java/an5/adapters/base/FieldMeta.java +42 -0
  35. package/java/src/main/java/an5/adapters/base/JdbcUrls.java +201 -0
  36. package/java/src/main/java/an5/adapters/base/Metadata.java +195 -0
  37. package/java/src/main/java/an5/adapters/base/RelationDef.java +26 -0
  38. package/java/src/main/java/an5/adapters/base/SqlBuilder.java +422 -0
  39. package/java/src/main/java/an5/adapters/base/SqliteVectors.java +375 -0
  40. package/java/src/main/java/an5/adapters/base/Vectors.java +123 -0
  41. package/java/src/test/java/An5AdaptersSmoke.java +733 -0
  42. package/java/src/test/java/Json.java +174 -0
  43. package/java/src/test/java/QuerySemantics.java +111 -0
  44. package/kotlin/build.gradle.kts +145 -0
  45. package/kotlin/settings.gradle.kts +10 -0
  46. package/kotlin/src/main/kotlin/an5/adapters/Aggregate.kt +89 -0
  47. package/kotlin/src/main/kotlin/an5/adapters/An5.kt +91 -0
  48. package/kotlin/src/main/kotlin/an5/adapters/Query.kt +130 -0
  49. package/kotlin/src/main/kotlin/an5/adapters/Rows.kt +166 -0
  50. package/kotlin/src/main/kotlin/an5/adapters/TableClient.kt +169 -0
  51. package/kotlin/src/main/kotlin/an5/adapters/Types.kt +77 -0
  52. package/kotlin/src/test/kotlin/an5/adapters/An5Smoke.kt +329 -0
  53. package/native/sqlite/CMakeLists.txt +12 -0
  54. package/native/sqlite/README.md +117 -0
  55. package/native/sqlite/an5_vector.c +209 -0
  56. package/package.json +33 -7
  57. package/python/an5_adapter.py +37 -6
  58. package/python/base/__init__.py +15 -0
  59. package/python/base/vectors.py +431 -0
  60. package/python/sqlite/__init__.py +2 -2
  61. package/python/sqlite/provider.py +21 -1
  62. package/python/table_client.py +129 -59
  63. package/rust/src/adapter.rs +342 -24
  64. package/rust/src/base/mod.rs +7 -0
  65. package/rust/src/base/vectors.rs +293 -0
  66. package/rust/src/lib.rs +3 -3
  67. package/scripts/sqlite-native-build.js +32 -0
  68. package/swift/Package.swift +22 -0
  69. package/swift/Sources/An5Adapters/An5Adapter.swift +160 -0
  70. package/swift/Sources/An5Adapters/Metadata.swift +183 -0
  71. package/swift/Sources/An5Adapters/Query.swift +105 -0
  72. package/swift/Sources/An5Adapters/Row+Accessors.swift +129 -0
  73. package/swift/Sources/An5Adapters/SQLBuilder.swift +359 -0
  74. package/swift/Sources/An5Adapters/SQLiteDriver.swift +301 -0
  75. package/swift/Sources/An5Adapters/SqliteVectors.swift +243 -0
  76. package/swift/Sources/An5Adapters/TableClient.swift +671 -0
  77. package/swift/Sources/An5Adapters/Types.swift +127 -0
  78. package/swift/Sources/An5Adapters/Vectors.swift +86 -0
  79. package/swift/Sources/An5Adapters/ViewClient.swift +50 -0
  80. package/swift/Sources/CSQLite/module.modulemap +8 -0
  81. package/swift/Sources/CSQLite/shim.h +9 -0
  82. package/swift/Tests/An5AdaptersTests/QuerySemanticsTests.swift +51 -0
  83. package/swift/Tests/An5AdaptersTests/RuntimeSmokeTests.swift +488 -0
  84. package/typescript/src/an5Adapter.ts +155 -31
  85. package/typescript/src/base/types.ts +15 -0
  86. package/typescript/src/index.ts +13 -0
  87. package/typescript/src/sqlite/browserEngine.ts +24 -0
  88. package/typescript/src/sqlite/engine.ts +13 -0
  89. package/typescript/src/sqlite/index.ts +1 -1
  90. package/typescript/src/sqlite/vector.ts +537 -0
  91. package/golang/sqlite/sqlite.go +0 -14
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@an5/adapters",
3
- "version": "0.2.11",
4
- "description": "Standalone language adapters for AN5 ORM - TypeScript, Python, .NET, Go, Rust, and Google Sheets query execution.",
3
+ "version": "0.2.14",
4
+ "description": "Standalone language adapters for AN5 ORM - TypeScript, Python, .NET, Go, Rust, Java, Kotlin, Swift, and Google Sheets query execution.",
5
5
  "license": "MIT",
6
6
  "main": "dist/index.js",
7
7
  "types": "dist/index.d.ts",
@@ -23,6 +23,9 @@
23
23
  "./dotnet": "./dotnet/An5Adapter.cs",
24
24
  "./golang": "./golang/an5_adapter.go",
25
25
  "./rust": "./rust/Cargo.toml",
26
+ "./java": "./java/src/main/java",
27
+ "./kotlin": "./kotlin/src/main/kotlin",
28
+ "./swift": "./swift/Sources/An5Adapters",
26
29
  "./nbase": "./dist/nbase/index.js"
27
30
  },
28
31
  "author": "an5ORM",
@@ -40,31 +43,54 @@
40
43
  "dotnet",
41
44
  "go",
42
45
  "golang",
43
- "rust"
46
+ "rust",
47
+ "java",
48
+ "kotlin",
49
+ "swift"
44
50
  ],
45
51
  "files": [
46
52
  "dist/**/*.{js,d.ts}",
53
+ "native/sqlite/**/*",
54
+ "!native/sqlite/build/**/*",
55
+ "scripts/sqlite-native-build.js",
47
56
  "typescript/src/**/*.ts",
57
+ "typescript/src/nbase/**/*.ts",
48
58
  "python/**/*.py",
49
59
  "dotnet/**/*",
50
60
  "golang/**/*",
51
61
  "rust/Cargo.toml",
52
62
  "rust/src/**/*",
53
- "README.md",
54
- "typescript/src/nbase/**/*.ts"
63
+ "java/pom.xml",
64
+ "java/src/main/java/**/*",
65
+ "java/src/test/java/**/*",
66
+ "kotlin/build.gradle.kts",
67
+ "kotlin/settings.gradle.kts",
68
+ "kotlin/src/main/kotlin/**/*",
69
+ "kotlin/src/test/kotlin/**/*",
70
+ "swift/Package.swift",
71
+ "swift/Sources/**/*",
72
+ "swift/Tests/**/*",
73
+ "README.md"
55
74
  ],
56
75
  "scripts": {
57
76
  "build": "tsc -p tsconfig.json",
58
77
  "pretest": "npm run build",
59
78
  "prepack": "npm run build",
60
79
  "build:python": "(python -m build --outdir dist-py || python3 -m build --outdir dist-py)",
61
- "test": "node test/unit.test.js && node test/package.test.js && node test/nbase.test.js && node --test test/version-sync.test.js && node --test test/query-semantics.test.js && node test/query-relations.integration.test.js && node --test test/sheets-oauth.test.js",
80
+ "test": "node test/unit.test.js && node test/package.test.js && node test/nbase.test.js && node test/sqlite-vector.test.js && node --test test/version-sync.test.js && node --test test/query-semantics.test.js && node test/query-relations.integration.test.js && node --test test/sheets-oauth.test.js",
62
81
  "test:package:smoke": "node scripts/package-smoke.js",
63
82
  "test:integration:live": "node test/live-db.integration.test.js && node test/query-relations.integration.test.js",
64
83
  "test:python": "node scripts/python-tests.js",
65
84
  "test:dotnet": "node scripts/dotnet-compile-check.js && node scripts/dotnet-sqlite-smoke.js",
66
85
  "test:go": "cd golang && go test ./...",
67
- "test:rust": "node scripts/rust-compile-check.js"
86
+ "test:rust": "node scripts/rust-compile-check.js",
87
+ "test:java": "node scripts/java-compile-check.js && node scripts/java-sqlite-smoke.js",
88
+ "test:kotlin": "node scripts/kotlin-compile-check.js",
89
+ "test:gradle": "node scripts/kotlin-gradle-build.js",
90
+ "test:swift": "node scripts/swift-test.js",
91
+ "build:sqlite:native": "node scripts/sqlite-native-build.js",
92
+ "test:sqlite:native": "node scripts/sqlite-native-test.js",
93
+ "bench:sqlite:native": "node scripts/sqlite-native-test.js --benchmark"
68
94
  },
69
95
  "dependencies": {
70
96
  "googleapis": "^183.0.0",
@@ -4,15 +4,17 @@ from typing import Dict, List, Optional, Any
4
4
 
5
5
  try:
6
6
  from .base import DIALECT_MSSQL, DIALECT_POSTGRES, DIALECT_SQLITE, detect_dialect, set_adapter_metadata
7
+ from .base.vectors import decode_vector, encode_vector, vector_distance
7
8
  from .mssql import connect as connect_mssql
8
9
  from .postgres import connect as connect_postgres
9
- from .sqlite import connect as connect_sqlite, is_memory as is_sqlite_memory
10
+ from .sqlite import connect as connect_sqlite, is_memory as is_sqlite_memory, register_vector_functions as register_sqlite_vector_functions
10
11
  from .table_client import AdapterTableClient, ViewClient
11
12
  except ImportError:
12
13
  from base import DIALECT_MSSQL, DIALECT_POSTGRES, DIALECT_SQLITE, detect_dialect, set_adapter_metadata
14
+ from base.vectors import decode_vector, encode_vector, vector_distance
13
15
  from mssql import connect as connect_mssql
14
16
  from postgres import connect as connect_postgres
15
- from sqlite import connect as connect_sqlite, is_memory as is_sqlite_memory
17
+ from sqlite import connect as connect_sqlite, is_memory as is_sqlite_memory, register_vector_functions as register_sqlite_vector_functions
16
18
  from table_client import AdapterTableClient, ViewClient
17
19
 
18
20
  # Backward-compatible aliases used by tests and older imports.
@@ -23,9 +25,15 @@ except ImportError:
23
25
  from mssql import parse_connection_string as _parse_connection_string
24
26
 
25
27
  class An5Adapter:
26
- def __init__(self, connection_string: str):
28
+ def __init__(self, connection_string: str, sqlite_vec: Optional[str] = None, vector_strategy: Optional[str] = None):
27
29
  self._dialect = detect_dialect(connection_string)
28
30
  self._conn_str = connection_string
31
+ # SQLite only. `sqlite_vec` names the extension binary to load;
32
+ # `vector_strategy` pins one of "sqlite-vec" | "udf" | "sql" | "memory"
33
+ # instead of probing for the fastest one available.
34
+ self._sqlite_vec = sqlite_vec
35
+ self.vector_strategy = vector_strategy
36
+ self._vector_supports: Dict[int, Any] = {}
29
37
  # The open connection of `transaction()`, when there is one. Every
30
38
  # statement in the callback has to use this very connection, otherwise a
31
39
  # rollback returns to a different, already committed connection — that
@@ -39,9 +47,29 @@ class An5Adapter:
39
47
  if self._dialect == DIALECT_POSTGRES:
40
48
  return connect_postgres(self._conn_str)
41
49
  if self._dialect == DIALECT_SQLITE:
42
- return connect_sqlite(self._conn_str)
50
+ conn = connect_sqlite(self._conn_str)
51
+ support = register_sqlite_vector_functions(conn, self._sqlite_vec)
52
+ support.capabilities()
53
+ self._sqlite_vector_support = support
54
+ return conn
43
55
  return connect_mssql(self._conn_str)
44
56
 
57
+ def vector_support(self):
58
+ """The SQLite vector hooks of the current connection, or None.
59
+
60
+ They register the distance functions SQLite ranks with and load the
61
+ sqlite-vec extension. A connection with none still searches, by falling
62
+ back to `json_each` or to Python.
63
+ """
64
+ if self._dialect != DIALECT_SQLITE:
65
+ return None
66
+ conn, owned = self._acquire()
67
+ try:
68
+ return self._sqlite_vector_support
69
+ finally:
70
+ if owned:
71
+ conn.close()
72
+
45
73
  def _acquire(self):
46
74
  """Returns (connection, whether we opened it and must close it)."""
47
75
  if self._txn_conn is not None:
@@ -167,11 +195,14 @@ class An5Adapter:
167
195
  if owned:
168
196
  conn.close()
169
197
 
170
- def create_an5_adapter(connection_string: str) -> An5Adapter:
171
- return An5Adapter(connection_string)
198
+ def create_an5_adapter(connection_string: str, sqlite_vec: Optional[str] = None, vector_strategy: Optional[str] = None) -> An5Adapter:
199
+ return An5Adapter(connection_string, sqlite_vec=sqlite_vec, vector_strategy=vector_strategy)
172
200
 
173
201
  __all__ = [
174
202
  "An5Adapter",
203
+ "encode_vector",
204
+ "decode_vector",
205
+ "vector_distance",
175
206
  "AdapterTableClient",
176
207
  "ViewClient",
177
208
  "create_an5_adapter",
@@ -1,3 +1,18 @@
1
1
  from .dialects import DIALECT_MSSQL, DIALECT_POSTGRES, DIALECT_SQLITE, detect_dialect
2
2
  from .metadata import model_to_table, model_fields, relation_map, set_adapter_metadata, get_relations_for_model, get_model_to_table, get_fields_for_model
3
3
  from .sql import _quote, _parse_where, _build_order_by, _resolve_table, _quote_table
4
+ from .vectors import (
5
+ AN5_VECTOR_FUNCTIONS,
6
+ SQLITE_VEC_FUNCTIONS,
7
+ SqliteVectorSupport,
8
+ decode_vector,
9
+ encode_vector,
10
+ is_vector_field,
11
+ make_distance_function,
12
+ needs_vector_encoding,
13
+ parse_vector,
14
+ plan_sqlite_vector_strategies,
15
+ read_sqlite_column_type,
16
+ run_sqlite_vector_search,
17
+ vector_distance,
18
+ )
@@ -0,0 +1,431 @@
1
+ """SQLite vector search — codec, distance functions and the strategy plan.
2
+
3
+ SQLite has no vector type, so a `VECTOR(n)` column stores a BLOB of little-endian
4
+ float32 values (see `encode_vector`) and is ranked in one of four ways, in this
5
+ order:
6
+
7
+ 1. ``sqlite-vec`` native scalar distance functions, when the extension loads.
8
+ 2. ``udf`` ``an5_vec_cosine`` / ``an5_vec_l2`` / ``an5_vec_ip``
9
+ registered with the driver. Reads the BLOB and the legacy JSON text.
10
+ 3. ``sql`` ``json_each`` brute force in plain SQL. Needs no user
11
+ function, but only reaches rows stored as JSON text.
12
+ 4. ``memory`` the column is loaded and scored in Python.
13
+
14
+ Strategies 1-3 rank inside the database and transfer only the matching rows,
15
+ which is why they are preferred over the in-memory path.
16
+ """
17
+
18
+ import json
19
+ import math
20
+ import struct
21
+ from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
22
+
23
+ METRIC_COSINE = "cosine"
24
+ METRIC_EUCLIDEAN = "euclidean"
25
+ METRIC_DOT = "dot"
26
+
27
+ # Scalar functions the adapter registers itself.
28
+ AN5_VECTOR_FUNCTIONS: Dict[str, str] = {
29
+ METRIC_COSINE: "an5_vec_cosine",
30
+ METRIC_EUCLIDEAN: "an5_vec_l2",
31
+ METRIC_DOT: "an5_vec_ip",
32
+ }
33
+
34
+ # Scalar functions shipped by the sqlite-vec extension.
35
+ SQLITE_VEC_FUNCTIONS: Dict[str, str] = {
36
+ METRIC_COSINE: "vec_distance_cosine",
37
+ METRIC_EUCLIDEAN: "vec_distance_l2",
38
+ METRIC_DOT: "vec_distance_ip",
39
+ }
40
+
41
+ STRATEGY_SQLITE_VEC = "sqlite-vec"
42
+ STRATEGY_UDF = "udf"
43
+ STRATEGY_SQL = "sql"
44
+ STRATEGY_MEMORY = "memory"
45
+
46
+ _BYTES_PER_FLOAT = 4
47
+ _FLOAT32 = struct.Struct("<f")
48
+
49
+
50
+ def normalize_metric(metric: Optional[str]) -> str:
51
+ return metric if metric in (METRIC_EUCLIDEAN, METRIC_DOT) else METRIC_COSINE
52
+
53
+
54
+ # ─── Codec ───────────────────────────────────────────────────────────────────────
55
+
56
+
57
+ def parse_vector(value: Any) -> Optional[List[float]]:
58
+ """Reads a numeric sequence, or JSON text holding one."""
59
+ if value is None or isinstance(value, (str, bytes, bytearray, memoryview)):
60
+ if isinstance(value, (bytes, bytearray, memoryview)):
61
+ return None
62
+ text = str(value).strip()
63
+ if not text.startswith("["):
64
+ return None
65
+ try:
66
+ return parse_vector(json.loads(text))
67
+ except Exception:
68
+ return None
69
+ if isinstance(value, dict):
70
+ return None
71
+ try:
72
+ values = [float(v) for v in value]
73
+ except (TypeError, ValueError):
74
+ return None
75
+ if not values or any(math.isnan(v) or math.isinf(v) for v in values):
76
+ return None
77
+ return values
78
+
79
+
80
+ def decode_vector(value: Any) -> Optional[List[float]]:
81
+ """Decodes a stored vector: float32 BLOB, legacy JSON text, or a sequence."""
82
+ direct = parse_vector(value)
83
+ if direct:
84
+ return direct
85
+ if isinstance(value, (bytes, bytearray, memoryview)):
86
+ raw = bytes(value)
87
+ if not raw or len(raw) % _BYTES_PER_FLOAT != 0:
88
+ return None
89
+ count = len(raw) // _BYTES_PER_FLOAT
90
+ return list(_FLOAT32.unpack_from(raw, i * _BYTES_PER_FLOAT)[0] for i in range(count))
91
+ return None
92
+
93
+
94
+ def encode_vector(values: Sequence[float]) -> bytes:
95
+ """Encodes a vector as the little-endian float32 BLOB the column stores."""
96
+ return b"".join(_FLOAT32.pack(float(v)) for v in values)
97
+
98
+
99
+ def needs_vector_encoding(value: Any) -> bool:
100
+ """True when the value still has to become bytes on its way into a column."""
101
+ if isinstance(value, (bytes, bytearray, memoryview, str)):
102
+ return False
103
+ if value is None:
104
+ return False
105
+ return isinstance(value, (list, tuple)) or (
106
+ hasattr(value, "__iter__") and not isinstance(value, (dict, int, float, bool))
107
+ )
108
+
109
+
110
+ def is_vector_field(definition: Any) -> bool:
111
+ """True when generated metadata describes a `VECTOR(n)` column."""
112
+ if not definition:
113
+ return False
114
+ if isinstance(definition, str):
115
+ return "[]" in definition or definition.strip().upper().startswith("VECTOR")
116
+ if not isinstance(definition, dict):
117
+ return False
118
+ if definition.get("kind") == "vector":
119
+ return True
120
+ sql = str(definition.get("sql") or definition.get("type") or "")
121
+ if sql.strip().upper().startswith("VECTOR") and sql.strip()[6:7] in ("", "("):
122
+ return True
123
+ return "[]" in str(definition.get("ts") or "")
124
+
125
+
126
+ # ─── Distance ─────────────────────────────────────────────────────────────────────
127
+
128
+
129
+ def vector_distance(a: Sequence[float], b: Sequence[float], metric: str) -> Optional[float]:
130
+ """Distance between two equal-length vectors, lower is closer.
131
+
132
+ Mirrors the SQL functions so a search ranks the same way whichever strategy
133
+ ran.
134
+ """
135
+ if a is None or b is None or len(a) != len(b) or len(a) == 0:
136
+ return None
137
+ resolved = normalize_metric(metric)
138
+ if resolved == METRIC_EUCLIDEAN:
139
+ return math.sqrt(sum((float(x) - float(y)) ** 2 for x, y in zip(a, b)))
140
+ dot = sum(float(x) * float(y) for x, y in zip(a, b))
141
+ if resolved == METRIC_DOT:
142
+ return -dot
143
+ m1 = math.sqrt(sum(float(x) ** 2 for x in a))
144
+ m2 = math.sqrt(sum(float(y) ** 2 for y in b))
145
+ if not m1 or not m2:
146
+ return 1.0
147
+ return 1.0 - dot / (m1 * m2)
148
+
149
+
150
+ def make_distance_function(metric: str) -> Callable[[Any, Any], Optional[float]]:
151
+ """The scalar function registered with the driver for one metric."""
152
+
153
+ def distance(left: Any, right: Any) -> Optional[float]:
154
+ a = decode_vector(left)
155
+ b = decode_vector(right)
156
+ if not a or not b:
157
+ return None
158
+ return vector_distance(a, b, metric)
159
+
160
+ return distance
161
+
162
+
163
+ # ─── Query building ───────────────────────────────────────────────────────────────
164
+
165
+
166
+ def _json_text(column: str) -> str:
167
+ """`json_valid` fails on a float32 BLOB, so the JSON path needs this guard."""
168
+ return f"CASE WHEN json_valid({column}) THEN {column} ELSE '[]' END"
169
+
170
+
171
+ def _json_distance_expr(metric: str, column: str) -> str:
172
+ row = _json_text(column)
173
+ same_length = f"(SELECT COUNT(*) FROM json_each({row})) = (SELECT COUNT(*) FROM q)"
174
+ terms = {
175
+ METRIC_COSINE: (
176
+ f"1.0 - (SELECT SUM(je.value * q.v) FROM json_each({row}) je JOIN q ON q.k = je.key)"
177
+ f" / NULLIF(sqrt((SELECT SUM(q.v * q.v) FROM q))"
178
+ f" * sqrt((SELECT SUM(je.value * je.value) FROM json_each({row}) je)), 0)"
179
+ ),
180
+ METRIC_EUCLIDEAN: (
181
+ f"sqrt((SELECT SUM((je.value - q.v) * (je.value - q.v))"
182
+ f" FROM json_each({row}) je JOIN q ON q.k = je.key))"
183
+ ),
184
+ METRIC_DOT: f"-(SELECT SUM(je.value * q.v) FROM json_each({row}) je JOIN q ON q.k = je.key)",
185
+ }
186
+ return f"CASE WHEN {same_length} THEN {terms[metric]} END"
187
+
188
+
189
+ def build_sqlite_vector_query(
190
+ strategy: str,
191
+ metric: str,
192
+ table: str,
193
+ column: str,
194
+ vector: Sequence[float],
195
+ take: int,
196
+ tail: str = "",
197
+ params: Optional[List] = None,
198
+ ) -> Tuple[str, List]:
199
+ """Ranking SQL for one strategy, with the query vector appended to `params`.
200
+
201
+ `sqlite-vec` and `udf` bind the query vector as a float32 BLOB; `sql` binds
202
+ it as JSON text because `json_each` reads text.
203
+ """
204
+ bound = []
205
+ metric = normalize_metric(metric)
206
+
207
+ def rank(inner: str) -> str:
208
+ # A row whose stored vector cannot be scored (a NULL, a text column,
209
+ # another dimension) yields a NULL distance; those rows are dropped
210
+ # rather than reported, so a caller never has to sort them out.
211
+ return (f"SELECT * FROM ({inner}) AS an5_ranked"
212
+ f" WHERE distance IS NOT NULL ORDER BY distance ASC LIMIT {int(take)}")
213
+
214
+
215
+ if strategy == STRATEGY_SQLITE_VEC:
216
+ bound.append(encode_vector(vector))
217
+ distance = f"CASE WHEN vec_length(vec_f32({column})) = {len(vector)} THEN {SQLITE_VEC_FUNCTIONS[metric]}(vec_f32({column}), ?) END"
218
+ # vec_f32 accepts both JSON text and float32 BLOBs.
219
+ query = rank(
220
+ f"SELECT *, {distance} AS distance FROM {table}"
221
+ + (tail + " AND " if tail else " WHERE ")
222
+ + f"{column} IS NOT NULL"
223
+ )
224
+ return query, bound + list(params or [])
225
+
226
+ if strategy == STRATEGY_UDF:
227
+ bound.append(encode_vector(vector))
228
+ distance = f"{AN5_VECTOR_FUNCTIONS[metric]}({column}, ?)"
229
+ return rank(f"SELECT *, {distance} AS distance FROM {table}{tail}"), bound + list(params or [])
230
+
231
+ bound.append(json.dumps(list(vector)))
232
+ query = (
233
+ "WITH q AS (SELECT je.key AS k, CAST(je.value AS REAL) AS v FROM json_each(?) je) "
234
+ + rank(f"SELECT *, {_json_distance_expr(metric, column)} AS distance FROM {table}{tail}")
235
+ )
236
+ return query, bound + list(params or [])
237
+
238
+
239
+ # ─── Capabilities ──────────────────────────────────────────────────────────────────
240
+
241
+
242
+ def plan_sqlite_vector_strategies(
243
+ capabilities: Dict[str, bool],
244
+ declared_type: Optional[str],
245
+ preference: Optional[str] = None,
246
+ ) -> List[str]:
247
+ """The ordered strategies to try for this connection.
248
+
249
+ `declared_type` is the column's DDL type: the JSON strategy only reaches rows
250
+ stored as text, so a `BLOB` column drops it from the order.
251
+ """
252
+ if preference and preference != "auto":
253
+ # `memory` is the caller's own path, not a query, so it yields no plan.
254
+ return [] if preference == STRATEGY_MEMORY else [preference]
255
+ available: List[str] = []
256
+ if capabilities.get("vec"):
257
+ available.append(STRATEGY_SQLITE_VEC)
258
+ if capabilities.get("udf"):
259
+ available.append(STRATEGY_UDF)
260
+ if capabilities.get("json1"):
261
+ available.append(STRATEGY_SQL)
262
+ declared = str(declared_type or "").upper()
263
+ if declared and not any(token in declared for token in ("TEXT", "CHAR", "CLOB", "STRING", "JSON")):
264
+ # A BLOB column has no JSON to read, so json_each can only produce NULLs.
265
+ return [s for s in available if s != STRATEGY_SQL]
266
+ return available
267
+
268
+
269
+ def read_sqlite_column_type(
270
+ exec_fn: Callable[[str, List], List[Dict]],
271
+ table: str,
272
+ column: str,
273
+ ) -> Optional[str]:
274
+ """Reads a vector column's declared DDL type, or None when it is not found."""
275
+ try:
276
+ rows = exec_fn("SELECT type FROM pragma_table_info(?) WHERE name = ?", [table, column])
277
+ except Exception:
278
+ return None
279
+ value = rows[0].get("type") if rows else None
280
+ return str(value) if value else None
281
+
282
+
283
+ class SqliteVectorSupport:
284
+ """Registers the distance functions and loads sqlite-vec for a connection."""
285
+
286
+ def __init__(
287
+ self,
288
+ native: Optional[Callable[[], Any]] = None,
289
+ register_function: Optional[Callable[[str, Callable], None]] = None,
290
+ sqlite_vec: Optional[str] = None,
291
+ ):
292
+ self._native = native
293
+ self._register_function = register_function
294
+ self._sqlite_vec = sqlite_vec
295
+ self._capabilities: Optional[Dict[str, bool]] = None
296
+ # Declared column types and the "any binary row" probe, keyed by column.
297
+ # Both cost a query and neither changes while a schema is in use; a stale
298
+ # answer is still safe, because a strategy that fails falls through to the
299
+ # next one rather than reporting a wrong result.
300
+ self._column_types: Dict[str, Optional[str]] = {}
301
+ self._has_binary_rows: Dict[str, bool] = {}
302
+
303
+ def column_type(self, exec_fn, table: str, column: str) -> Optional[str]:
304
+ """The column's declared DDL type, remembered per column."""
305
+ key = f"{table}.{column}"
306
+ if key not in self._column_types:
307
+ self._column_types[key] = read_sqlite_column_type(exec_fn, table, column)
308
+ return self._column_types[key]
309
+
310
+ def has_binary_rows(self, exec_fn, table: str, column: str, tail: str, params: List) -> bool:
311
+ """Whether any row stores raw bytes, which `json_each` cannot read.
312
+
313
+ Checked only for the `sql` strategy, where a legacy JSON column that has
314
+ since taken new float32 rows would otherwise rank just the old ones.
315
+ """
316
+ clause = (tail + " AND ") if tail else " WHERE "
317
+ try:
318
+ rows = exec_fn(
319
+ f"SELECT 1 FROM {table}{clause}typeof({column}) = 'blob' LIMIT 1",
320
+ list(params))
321
+ return bool(rows)
322
+ except Exception:
323
+ return False
324
+
325
+ def capabilities(self) -> Dict[str, bool]:
326
+ """Probes once and caches the answer."""
327
+ if self._capabilities is not None:
328
+ return self._capabilities
329
+
330
+ caps = {"vec": False, "udf": False, "json1": False}
331
+ native = self._native() if self._native else None
332
+ if native is not None:
333
+ if self._sqlite_vec and hasattr(native, "enable_load_extension"):
334
+ try:
335
+ native.enable_load_extension(True)
336
+ native.load_extension(self._sqlite_vec)
337
+ except Exception:
338
+ pass
339
+ finally:
340
+ try:
341
+ native.enable_load_extension(False)
342
+ except Exception:
343
+ pass
344
+ try:
345
+ native.execute("SELECT vec_version()").fetchone()
346
+ caps["vec"] = True
347
+ except Exception:
348
+ caps["vec"] = False
349
+ try:
350
+ native.execute("SELECT an5_vector_version()").fetchone()
351
+ caps["udf"] = True
352
+ caps["vec"] = False
353
+ except Exception:
354
+ pass
355
+
356
+ if self._register_function and not caps["udf"]:
357
+ try:
358
+ for metric, name in AN5_VECTOR_FUNCTIONS.items():
359
+ self._register_function(name, make_distance_function(metric))
360
+ caps["udf"] = True
361
+ except Exception:
362
+ caps["udf"] = False
363
+
364
+ self._capabilities = caps
365
+ return caps
366
+
367
+ def probe(self, exec_fn: Callable[[str, List], List[Dict]]) -> Dict[str, bool]:
368
+ """Confirms JSON1 before a query is built that needs it."""
369
+ caps = self.capabilities()
370
+ if not caps["json1"]:
371
+ try:
372
+ exec_fn("SELECT json_valid('[1]') AS an5_json_ok", [])
373
+ caps["json1"] = True
374
+ except Exception:
375
+ caps["json1"] = False
376
+ return caps
377
+
378
+
379
+ def run_sqlite_vector_search(
380
+ exec_fn: Callable[[str, List], List[Dict]],
381
+ table: str,
382
+ raw_table: str,
383
+ column: str,
384
+ raw_column: str,
385
+ vector: Sequence[float],
386
+ metric: str,
387
+ take: int,
388
+ tail: str = "",
389
+ params: Optional[List] = None,
390
+ support: Optional[SqliteVectorSupport] = None,
391
+ preference: Optional[str] = None,
392
+ ) -> Optional[List[Dict]]:
393
+ """Ranks a SQLite table inside the database.
394
+
395
+ Returns None when the connection offers no strategy, so the caller can fall
396
+ back to scoring in memory.
397
+ """
398
+ if vector is None or len(vector) == 0:
399
+ return None
400
+
401
+ if support is not None:
402
+ caps = support.probe(exec_fn)
403
+ else:
404
+ try:
405
+ exec_fn("SELECT json_valid('[1]') AS an5_json_ok", [])
406
+ caps = {"vec": False, "udf": False, "json1": True}
407
+ except Exception:
408
+ caps = {"vec": False, "udf": False, "json1": False}
409
+
410
+ declared = (support.column_type(exec_fn, raw_table, raw_column)
411
+ if support is not None
412
+ else read_sqlite_column_type(exec_fn, raw_table, raw_column))
413
+ for strategy in plan_sqlite_vector_strategies(caps, declared, preference):
414
+ if strategy == STRATEGY_SQL:
415
+ binary = (support.has_binary_rows(exec_fn, table, column, tail, list(params or []))
416
+ if support is not None
417
+ else bool(exec_fn("SELECT 1 FROM " + table + (tail + " AND " if tail else " WHERE ")
418
+ + f"typeof({column}) = 'blob' LIMIT 1", list(params or []))))
419
+ if binary:
420
+ continue
421
+ query, bound = build_sqlite_vector_query(
422
+ strategy, metric, table, column, vector, take, tail, list(params or [])
423
+ )
424
+ try:
425
+ return exec_fn(query, bound)
426
+ except Exception:
427
+ # The driver advertised the capability but the query failed, e.g. an
428
+ # extension that did not really load. The next strategy is cheaper
429
+ # than reporting the failure, and memory is the last resort.
430
+ continue
431
+ return None
@@ -1,3 +1,3 @@
1
- from .provider import connect, is_memory, parse_connection_string, placeholder
1
+ from .provider import connect, is_memory, parse_connection_string, placeholder, register_vector_functions
2
2
 
3
- __all__ = ["connect", "is_memory", "parse_connection_string", "placeholder"]
3
+ __all__ = ["connect", "is_memory", "parse_connection_string", "placeholder", "register_vector_functions"]
@@ -12,7 +12,12 @@ statement) would leave transactions dangling. `None` means autocommit, matching
12
12
  """
13
13
 
14
14
  import sqlite3
15
- from typing import Tuple
15
+ from typing import Optional, Tuple
16
+
17
+ try: # package import
18
+ from ..base.vectors import SqliteVectorSupport
19
+ except ImportError: # flat layout, when the folder itself is on sys.path
20
+ from base.vectors import SqliteVectorSupport
16
21
 
17
22
  # Strip the `sqlite:`/`sqlite://`/`sqlite:///` wrapper first, only then look for
18
23
  # `file:`. Two steps are required because `sqlite:file::memory:?cache=shared` only
@@ -79,5 +84,20 @@ def connect(connection_string: str):
79
84
  return conn
80
85
 
81
86
 
87
+ def register_vector_functions(conn, sqlite_vec: Optional[str] = None) -> SqliteVectorSupport:
88
+ """Prepares the connection for in-database vector search.
89
+
90
+ Registers `an5_vec_cosine` / `an5_vec_l2` / `an5_vec_ip` so SQLite can rank
91
+ a `VECTOR(n)` column without the client loading it, and loads the sqlite-vec
92
+ extension when a path was given. A driver that refuses either is not an
93
+ error: the search then falls back to `json_each` or to Python.
94
+ """
95
+ return SqliteVectorSupport(
96
+ native=lambda: conn,
97
+ register_function=lambda name, fn: conn.create_function(name, 2, fn),
98
+ sqlite_vec=sqlite_vec,
99
+ )
100
+
101
+
82
102
  def placeholder() -> str:
83
103
  return "?"