@an5/adapters 0.2.10 → 0.2.13

package/rust/src/base/vectors.rs

@@ -0,0 +1,293 @@
+//! SQLite vector codec, distance and strategy plan.
+//!
+//! SQLite has no vector type, so a `VECTOR(n)` column stores a BLOB of little-endian
+//! float32 values. The ranking itself is built in [`crate::adapter`] from these
+//! helpers.
+//!
+//! The strategies, in the order every runtime tries them:
+//!
+//! 1. `sqlite-vec` — the extension, when it loads.
+//! 2. `udf` — `an5_vec_cosine` / `an5_vec_l2` / `an5_vec_ip` registered with the driver.
+//! 3. `sql` — `json_each` brute force in plain SQL.
+//! 4. `memory` — the column is loaded and scored in Rust.
+//!
+//! `sqlx` connects through a pooled `AnyPool` and offers no way to register a SQL
+//! function or load an extension, so a stock Rust adapter reaches `sql` and
+//! `memory`. That is enough to rank in the database; the first two need a driver
+//! built for it.
+
+use std::collections::HashMap;
+
+use serde_json::Value;
+
+/// The size of one stored value; a column's byte length is its dimension times this.
+pub const BYTES_PER_VECTOR_FLOAT: usize = 4;
+
+pub const STRATEGY_SQLITE_VEC: &str = "sqlite-vec";
+pub const STRATEGY_UDF: &str = "udf";
+pub const STRATEGY_SQL: &str = "sql";
+pub const STRATEGY_MEMORY: &str = "memory";
+
+/// Scalar functions the adapter registers itself, when the driver allows it.
+pub const AN5_VECTOR_FUNCTIONS: [(&str, &str); 3] = [
+    ("cosine", "an5_vec_cosine"),
+    ("euclidean", "an5_vec_l2"),
+    ("dot", "an5_vec_ip"),
+];
+
+/// Scalar functions the sqlite-vec extension provides.
+pub const SQLITE_VEC_FUNCTIONS: [(&str, &str); 3] = [
+    ("cosine", "vec_distance_cosine"),
+    ("euclidean", "vec_distance_l2"),
+    ("dot", "vec_distance_ip"),
+];
+
+/// Returns a known metric name, defaulting to cosine.
+pub fn normalize_metric(metric: &str) -> &str {
+    match metric.to_lowercase().as_str() {
+        "euclidean" | "dot" => metric,
+        _ => "cosine",
+    }
+}
+
+/// Returns the function name a strategy uses for a metric.
+pub fn distance_function(strategy: &str, metric: &str) -> Option<&'static str> {
+    let table = match strategy {
+        STRATEGY_SQLITE_VEC => &SQLITE_VEC_FUNCTIONS,
+        STRATEGY_UDF => &AN5_VECTOR_FUNCTIONS,
+        _ => return None,
+    };
+    let metric = normalize_metric(metric);
+    table
+        .iter()
+        .find(|(name, _)| *name == metric)
+        .map(|(_, function)| *function)
+}
+
+/// Encodes a vector as the little-endian float32 BLOB the column stores.
+pub fn encode_vector(values: &[f64]) -> Vec<u8> {
+    let mut bytes = Vec::with_capacity(values.len() * BYTES_PER_VECTOR_FLOAT);
+    for value in values {
+        bytes.extend_from_slice(&(*value as f32).to_le_bytes());
+    }
+    bytes
+}
+
+/// Reads a numeric sequence, or JSON text holding one.
+pub fn parse_vector(value: &Value) -> Option<Vec<f64>> {
+    match value {
+        Value::Array(items) => {
+            let mut out = Vec::with_capacity(items.len());
+            for item in items {
+                out.push(item.as_f64()?);
+            }
+            if out.is_empty() {
+                None
+            } else {
+                Some(out)
+            }
+        }
+        Value::String(text) => parse_vector_text(text),
+        _ => None,
+    }
+}
+
+/// Parses the JSON text a legacy `VECTOR(n)` column holds.
+pub fn parse_vector_text(raw: &str) -> Option<Vec<f64>> {
+    let trimmed = raw.trim();
+    if !trimmed.starts_with('[') {
+        return None;
+    }
+    match serde_json::from_str::<Vec<f64>>(trimmed) {
+        Ok(values) if !values.is_empty() => Some(values),
+        _ => None,
+    }
+}
+
+/// Decodes a stored vector: a float32 BLOB, a legacy JSON text column, or an
+/// array the driver already decoded.
+pub fn decode_vector(value: &Value) -> Option<Vec<f64>> {
+    if let Some(bytes) = crate::adapter::blob_bytes(value) {
+        return decode_vector_bytes(&bytes);
+    }
+    if let Some(text) = value.as_str() {
+        return parse_vector_text(text);
+    }
+    parse_vector(value)
+}
+
+/// Decodes the float32 BLOB a `VECTOR(n)` column stores.
+///
+/// A column written by an older version holds JSON text instead, which is not a
+/// multiple of four bytes once encoded, so the length check tells them apart.
+pub fn decode_vector_bytes(bytes: &[u8]) -> Option<Vec<f64>> {
+    if let Some(values) = parse_vector_text(&String::from_utf8_lossy(bytes)) {
+        return Some(values);
+    }
+    if bytes.is_empty() || bytes.len() % BYTES_PER_VECTOR_FLOAT != 0 {
+        return None;
+    }
+    Some(
+        bytes
+            .chunks_exact(BYTES_PER_VECTOR_FLOAT)
+            .map(|chunk| f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]) as f64)
+            .collect(),
+    )
+}
+
+/// True when the value still has to become bytes on its way into a column.
+pub fn needs_vector_encoding(value: &Value) -> bool {
+    value.is_array()
+}
+
+/// True when generated metadata describes a `VECTOR(n)` column.
+///
+/// The metadata value is either the declared SQL type as a string or a record with
+/// `sql` / `ts` / `kind` members, so both shapes are accepted.
+pub fn is_vector_field(definition: &Value) -> bool {
+    match definition {
+        Value::String(text) => {
+            text.contains("[]") || text.trim().to_uppercase().starts_with("VECTOR")
+        }
+        Value::Object(map) => {
+            if map.get("kind").and_then(Value::as_str) == Some("vector") {
+                return true;
+            }
+            for key in ["sql", "type"] {
+                if let Some(sql) = map.get(key).and_then(Value::as_str) {
+                    if sql_looks_vector(sql) {
+                        return true;
+                    }
+                }
+            }
+            map.get("ts")
+                .and_then(Value::as_str)
+                .map(|ts| ts.contains("[]"))
+                .unwrap_or(false)
+        }
+        _ => false,
+    }
+}
+
+/// True when a declared SQL type names a vector column.
+fn sql_looks_vector(sql: &str) -> bool {
+    let trimmed = sql.trim().to_uppercase();
+    match trimmed.strip_prefix("VECTOR") {
+        None => false,
+        Some(rest) => rest.is_empty() || rest.starts_with('(') || rest.starts_with(' '),
+    }
+}
+
+/// The ordered strategies to try for this connection.
+///
+/// `declared_type` is the column's DDL type: the JSON strategy only reaches rows
+/// stored as text, so a `BLOB` column drops it from the order. `preference` pins a
+/// strategy instead of probing.
+pub fn plan_vector_strategies(
+    capabilities: &HashMap<&str, bool>,
+    declared_type: Option<&str>,
+    preference: Option<&str>,
+) -> Vec<&'static str> {
+    if let Some(preference) = preference.filter(|p| !p.is_empty() && *p != "auto") {
+        // `memory` is the caller's own path, not a query, so it yields no plan.
+        return match preference {
+            STRATEGY_MEMORY => Vec::new(),
+            STRATEGY_SQLITE_VEC => vec![STRATEGY_SQLITE_VEC],
+            STRATEGY_UDF => vec![STRATEGY_UDF],
+            _ => vec![STRATEGY_SQL],
+        };
+    }
+
+    let mut available: Vec<&'static str> = Vec::new();
+    if capabilities.get("vec").copied().unwrap_or(false) {
+        available.push(STRATEGY_SQLITE_VEC);
+    }
+    if capabilities.get("udf").copied().unwrap_or(false) {
+        available.push(STRATEGY_UDF);
+    }
+    if capabilities.get("json1").copied().unwrap_or(false) {
+        available.push(STRATEGY_SQL);
+    }
+
+    let declared = declared_type.unwrap_or_default().trim().to_uppercase();
+    if !declared.is_empty()
+        && !["TEXT", "CHAR", "CLOB", "STRING", "JSON"]
+            .iter()
+            .any(|token| declared.contains(token))
+    {
+        // A BLOB column has no JSON to read, so json_each could only produce NULLs.
+        available.retain(|strategy| *strategy != STRATEGY_SQL);
+    }
+    available
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn encodes_little_endian_float32() {
+        let bytes = encode_vector(&[1.0, -2.0, 0.5]);
+        assert_eq!(bytes.len(), 12);
+        assert_eq!(&bytes[0..4], &[0x00, 0x00, 0x80, 0x3f]);
+        assert_eq!(&bytes[4..8], &[0x00, 0x00, 0x00, 0xc0]);
+    }
+
+    #[test]
+    fn round_trips_a_blob() {
+        assert_eq!(decode_vector_bytes(&encode_vector(&[0.25, -0.5, 3.0])), Some(vec![0.25, -0.5, 3.0]));
+    }
+
+    #[test]
+    fn reads_the_legacy_json_text_column() {
+        assert_eq!(decode_vector(&Value::from("[1, 0, 0]")), Some(vec![1.0, 0.0, 0.0]));
+        assert_eq!(decode_vector_bytes(b"[1, 0, 0]"), Some(vec![1.0, 0.0, 0.0]));
+        assert_eq!(decode_vector(&Value::from("[1, 2]")), Some(vec![1.0, 2.0]));
+    }
+
+    #[test]
+    fn rejects_values_it_cannot_read() {
+        assert!(decode_vector(&Value::Null).is_none());
+        assert!(decode_vector(&Value::from("not json")).is_none());
+        assert!(decode_vector_bytes(&[1, 2, 3]).is_none());
+        assert!(decode_vector_bytes(&[]).is_none());
+    }
+
+    #[test]
+    fn recognises_vector_columns() {
+        assert!(is_vector_field(&serde_json::json!({ "ts": "number[] | string", "sql": "VECTOR(3)" })));
+        assert!(is_vector_field(&serde_json::json!({ "ts": "number[] | string" })));
+        assert!(is_vector_field(&serde_json::json!({ "kind": "vector" })));
+        assert!(is_vector_field(&Value::from("number[] | string")));
+        assert!(!is_vector_field(&serde_json::json!({ "ts": "string", "sql": "TEXT" })));
+        assert!(!is_vector_field(&Value::Null));
+    }
+
+    #[test]
+    fn plans_the_strategies_in_order() {
+        let all = HashMap::from([("vec", true), ("udf", true), ("json1", true)]);
+        assert_eq!(
+            plan_vector_strategies(&all, Some("BLOB"), None),
+            vec![STRATEGY_SQLITE_VEC, STRATEGY_UDF]
+        );
+        let json_only = HashMap::from([("json1", true)]);
+        assert_eq!(plan_vector_strategies(&json_only, Some("TEXT"), None), vec![STRATEGY_SQL]);
+        // A BLOB column has no JSON to read.
+        assert!(plan_vector_strategies(&json_only, Some("BLOB"), None).is_empty());
+        assert!(plan_vector_strategies(&HashMap::new(), Some("TEXT"), None).is_empty());
+        assert_eq!(plan_vector_strategies(&all, Some("BLOB"), Some("udf")), vec![STRATEGY_UDF]);
+        assert!(plan_vector_strategies(&all, Some("BLOB"), Some("memory")).is_empty());
+    }
+
+    #[test]
+    fn maps_each_metric_to_its_function() {
+        assert_eq!(distance_function(STRATEGY_UDF, "cosine"), Some("an5_vec_cosine"));
+        assert_eq!(distance_function(STRATEGY_UDF, "euclidean"), Some("an5_vec_l2"));
+        assert_eq!(distance_function(STRATEGY_UDF, "dot"), Some("an5_vec_ip"));
+        assert_eq!(
+            distance_function(STRATEGY_SQLITE_VEC, "cosine"),
+            Some("vec_distance_cosine")
+        );
+        assert_eq!(distance_function(STRATEGY_SQL, "cosine"), None);
+    }
+}

package/rust/src/lib.rs

@@ -31,9 +31,9 @@ pub mod adapter;
 pub mod base;
 
 pub use adapter::{
-    AggregateArgs, An5Adapter, CountArgs, CreateArgs, CreateManyArgs, DeleteManyArgs, FindManyArgs,
-    GroupByArgs, Result, RowMap, TableClient, UpdateArgs, UpdateManyArgs, UpsertArgs,
-    VectorSearchArgs, ViewClient,
+    blob_bytes, blob_value, AggregateArgs, An5Adapter, CountArgs, CreateArgs, CreateManyArgs,
+    DeleteManyArgs, FindManyArgs, GroupByArgs, Result, RowMap, TableClient, UpdateArgs,
+    UpdateManyArgs, UpsertArgs, VectorSearchArgs, ViewClient,
 };
 pub use base::{
     add_table_override, build_order_by, build_where, cosine_similarity, dot_product,

package/scripts/sqlite-native-build.js

@@ -0,0 +1,32 @@
+const fs = require('node:fs');
+const path = require('node:path');
+const { execFileSync } = require('node:child_process');
+
+function buildNative(directory) {
+  const source = path.resolve(__dirname, '../native/sqlite');
+  fs.mkdirSync(directory, { recursive: true });
+  if (process.platform === 'win32') {
+    execFileSync('cmake', ['-S', source, '-B', directory, '-DCMAKE_BUILD_TYPE=Release'], { stdio: 'inherit' });
+    execFileSync('cmake', ['--build', directory, '--config', 'Release'], { stdio: 'inherit' });
+    const dll = path.join(directory, 'Release', 'an5_vector.dll');
+    return fs.existsSync(dll) ? dll : path.join(directory, 'an5_vector.dll');
+  }
+  const binary = path.join(directory, process.platform === 'darwin' ? 'an5_vector.dylib' : 'an5_vector.so');
+  const flags = ['-std=c99', '-O3', '-Wall', '-Wextra', '-Werror', '-fPIC',
+    process.platform === 'darwin' ? '-dynamiclib' : '-shared'];
+  if (process.env.SQLITE_INCLUDE_DIR) flags.push('-I', process.env.SQLITE_INCLUDE_DIR);
+  if (process.env.AN5_NATIVE_SANITIZE) flags.push('-fsanitize=undefined', '-fno-sanitize-recover=all');
+  execFileSync(process.env.CC || 'cc', [...flags, path.join(source, 'an5_vector.c'), '-lm', '-o', binary], { stdio: 'inherit' });
+  return binary;
+}
+
+module.exports = { buildNative };
+if (require.main === module) {
+  try {
+    console.log(buildNative(path.resolve(process.argv[2] || path.join(__dirname, '../native/sqlite/build'))));
+  } catch (error) {
+    console.error('Native vector build requires a C compiler and sqlite3ext.h (Windows: CMake and a C compiler).');
+    console.error(error.message);
+    process.exitCode = 1;
+  }
+}

package/swift/Package.swift

@@ -0,0 +1,22 @@
+// swift-tools-version: 5.9
+import PackageDescription
+
+// The product is `An5Adapters`. A consumer refers to it as
+// `.product(name: "An5Adapters", package: "an5Adapters")` when it depends on this repository
+// by URL — SwiftPM derives the package identity from the URL's last path component. A
+// `.package(path:)` dependency takes the checkout's directory name instead, so from a local
+// clone the identity is `swift` and that argument must match.
+let package = Package(
+    name: "An5Adapters",
+    products: [
+        .library(name: "An5Adapters", targets: ["An5Adapters"])
+    ],
+    targets: [
+        // The system SQLite, reached through a shim header rather than a copy of the C API.
+        // SQLite is what the other mobile-facing clients read and write on device, and the
+        // runtime is always present on iOS, tvOS, watchOS, macOS and Android.
+        .systemLibrary(name: "CSQLite", path: "Sources/CSQLite"),
+        .target(name: "An5Adapters", dependencies: ["CSQLite"]),
+        .testTarget(name: "An5AdaptersTests", dependencies: ["An5Adapters"]),
+    ]
+)

package/swift/Sources/An5Adapters/An5Adapter.swift

@@ -0,0 +1,160 @@
+import Foundation
+
+/// The AN5 runtime: a driver, the dialect it speaks, and the metadata that gives models
+/// their tables.
+///
+/// ```swift
+/// let driver = try SQLiteDriver(path: ":memory:")
+/// let db = try An5Adapter(driver: driver, connectionString: "sqlite::memory:", metadata: An5Metadata.registry)
+/// try db.table("User").create(["name": "Ada", "score": 10])
+/// let adults = try db.table("User").findMany(where: ["age": ["gte": 18]])
+/// ```
+///
+/// One connection is held for its lifetime, so an in-memory database survives between
+/// statements — which is what makes it usable in a test.
+public final class An5Adapter {
+
+    public let driver: SQLDriver
+    public let dialect: Dialect
+    public let metadata: Metadata
+
+    /// The open transaction, when there is one. Every statement inside it has to use this
+    /// driver's single connection, which is why the flag exists at all: a rollback that ran
+    /// on a different connection would return to an already committed one.
+    ///
+    /// Shared with the scoped adapter `transaction` hands to the caller, so a nested
+    /// transaction is refused no matter which handle the inner call went through.
+    private var state = TransactionState()
+
+    /// Opens a runtime over an existing driver.
+    ///
+    /// - Parameters:
+    ///   - driver: the SQL surface to run statements through.
+    ///   - connectionString: the project's connection string; decides the dialect.
+    ///   - metadata: the generated models' tables and columns.
+    public convenience init(driver: SQLDriver, connectionString: String, metadata: Metadata = .empty) {
+        self.init(driver: driver, dialect: Dialect.detect(connectionString), metadata: metadata)
+    }
+
+    /// Opens a runtime when the dialect is already known.
+    ///
+    /// Separate from the connection-string form because re-deriving the dialect from a bare
+    /// name like `sqlite` does not work: `Dialect.detect` reads a scheme or a file suffix,
+    /// and neither is present, so it falls through to SQL Server and every statement in a
+    /// transaction comes out with `WITH (NOLOCK)`.
+    init(driver: SQLDriver, dialect: Dialect, metadata: Metadata = .empty) {
+        self.driver = driver
+        self.dialect = dialect
+        self.metadata = metadata
+    }
+
+    /// Opens a runtime over the system SQLite.
+    public convenience init(
+        path: String,
+        metadata: Metadata = .empty,
+        connectionString: String? = nil
+    ) throws {
+        try self.init(
+            driver: SQLiteDriver(path: path),
+            connectionString: connectionString ?? "sqlite::memory:",
+            metadata: metadata
+        )
+    }
+
+    /// A runtime over the same driver that shares this one's transaction state.
+    private func scoped() -> An5Adapter {
+        let scoped = An5Adapter(driver: driver, dialect: dialect, metadata: metadata)
+        scoped.state = state
+        return scoped
+    }
+
+    /// The table client for a model.
+    public func table(_ model: String) -> TableClient {
+        TableClient(adapter: self, model: model)
+    }
+
+    /// A read-only client for a database view.
+    public func view(_ name: String) -> ViewClient {
+        ViewClient(adapter: self, view: name)
+    }
+
+    /// Pins one SQLite vector strategy instead of probing for the fastest one available:
+    /// `sqlite-vec`, `udf`, `sql` or `memory`. `nil` probes.
+    ///
+    /// SQLite has no vector type, so a `VECTOR(n)` column is ranked by the sqlite-vec
+    /// extension, a distance function the driver registers, `json_each` in plain SQL, or in
+    /// the client. The bundled `SQLiteDriver` registers the distance functions and loads
+    /// `SQLiteDriver.sqliteVecPath` when one is set.
+    public var vectorStrategy: String?
+
+    /// What this SQLite connection can do for vector search, probed once and cached.
+    ///
+    /// `nil` on the other dialects. The bundled driver registers the `an5_vec_*` functions
+    /// itself, so `udf` is reported whenever it is in use; a custom `SQLDriver` normally
+    /// reaches only `json_each` and the in-memory path.
+    public private(set) lazy var sqliteCapabilities: SqliteVectors.Capabilities = {
+        SqliteVectors.Capabilities(
+            vec: self.probe("SELECT vec_version()"),
+            udf: self.probe("SELECT an5_vec_cosine(zeroblob(4), zeroblob(4))"),
+            json1: self.probe("SELECT json_valid('[1]')")
+        )
+    }()
+
+    /// True when a probe statement ran, whatever it returned.
+    private func probe(_ sql: String) -> Bool {
+        (try? driver.query(sql, [])) != nil
+    }
+
+    /// A vector column's declared DDL type, or `nil` when it cannot be read.
+    ///
+    /// It is what decides whether the `json_each` strategy can reach the rows in the column:
+    /// a `BLOB` column has no JSON to walk.
+    public func queryColumnType(table: String, column: String) throws -> String? {
+        let rows = try driver.query(
+            "SELECT type FROM pragma_table_info(?) WHERE name = ?", [.text(table), .text(column)]
+        )
+        return rows.first?["type"].flatMap { $0 as? String }
+    }
+
+    /// Runs a query and returns its rows keyed by column name.
+    public func query(_ sql: String, _ parameters: [Any?] = []) throws -> [Row] {
+        try driver.query(sql, parameters.compactMap { SQLValue.encode($0) })
+    }
+
+    /// Runs a statement that returns no rows and gives back the affected row count.
+    @discardableResult
+    public func execute(_ sql: String, _ parameters: [Any?] = []) throws -> Int {
+        try driver.execute(sql, parameters.compactMap { SQLValue.encode($0) })
+    }
+
+    /// Runs `body` inside a transaction, committing on return and rolling back on failure.
+    ///
+    /// Nesting is rejected rather than flattened: an inner commit would make the outer
+    /// rollback a partial save, which is the opposite of what the caller asked for.
+    public func transaction<T>(_ body: (An5Adapter) throws -> T) throws -> T {
+        guard !state.inTransaction else {
+            throw An5Error.invalidQuery("a nested transaction is not supported")
+        }
+        state.inTransaction = true
+        defer { state.inTransaction = false }
+        do {
+            try driver.beginTransaction()
+        } catch {
+            throw An5Error.driver("cannot begin a transaction: \(error)")
+        }
+        do {
+            let result = try body(scoped())
+            try driver.commit()
+            return result
+        } catch {
+            try? driver.rollback()
+            throw error
+        }
+    }
+}
+
+/// Whether a transaction is open, shared between an adapter and the scoped handle its
+/// transaction hands out.
+private final class TransactionState {
+    var inTransaction = false
+}