@an5/adapters 0.2.11 → 0.2.13

package/typescript/src/an5Adapter.ts

@@ -27,6 +27,15 @@ import {
   type NBaseVectorClient,
 } from './nbase';
 import { bindParam, buildOrderBy, parseWhere, quote } from './base/sql';
+import {
+  decodeVector,
+  isVectorField,
+  needsVectorEncoding,
+  runSqliteVectorSearch,
+  vectorDistance,
+  type SqliteVectorSupport,
+  type VectorStrategyPreference,
+} from './sqlite/vector';
 import {
   createAdapterProxy,
   getFieldsForModel,
@@ -72,7 +81,10 @@ function isDuplicateKeyError(error: unknown, dialect: Dialect): boolean {
 function appendUpdateSet(sets: string[], params: Record<string, any>, col: string, val: any, dialect: Dialect): void {
   const quoted = quote(col, dialect);
   const safeCol = sanitizeParamName(col);
-  if (val && typeof val === 'object' && !(val instanceof Date)) {
+  // `TypedArray.prototype.set` is the copy method, not the update operator, so a
+  // vector written as a float32 BLOB must not be read as `{ set: ... }`.
+  const isLiteral = Array.isArray(val) || ArrayBuffer.isView(val) || val instanceof ArrayBuffer;
+  if (val && typeof val === 'object' && !(val instanceof Date) && !isLiteral) {
     if (val.increment !== undefined) {
       const p = bindParam(params, `s_${safeCol}_inc`, val.increment);
       sets.push(`${quoted} = ${quoted} + @${p}`);
@@ -386,6 +398,22 @@ export class An5Adapter {
     return (await this.requireEngine()).exec<T>(query, params);
   }
 
+  /**
+   * The SQLite engine's vector hooks, or null when this is not a SQLite adapter.
+   * `vectorSearch` needs them to load sqlite-vec and register the distance
+   * functions; a connection with no hooks still searches in memory.
+   */
+  async sqliteVectorSupport(): Promise<SqliteVectorSupport | null> {
+    if (this.sheetsAdapter) return null;
+    const engine = await this.requireEngine();
+    return engine.dialect === 'sqlite' ? (engine.vectorSupport ?? null) : null;
+  }
+
+  /** `vectorStrategy` from the config, so a project can pin one strategy. */
+  get vectorStrategy(): VectorStrategyPreference | undefined {
+    return this._engineConfig?.vectorStrategy;
+  }
+
   /** INTERNAL: used by AdapterTableClient for DML statements needing row count */
   async _executeRaw(query: string, params?: Record<string, any>): Promise<number> {
     return (await this.requireEngine()).executeRaw(query, params);
@@ -732,6 +760,14 @@ export class AdapterTableClient<T = any> {
   public get dialect(): Dialect { return this.adapter.dialect; }
 
   public get tableName(): string {
+    const t = this.mappedTableName;
+    if (t.startsWith('[') || t.startsWith('"') || t.startsWith('`')) return t;
+    if (t.includes('.')) return t.split('.').map(p => quote(p, this.dialect)).join('.');
+    return quote(t, this.dialect);
+  }
+
+  /** The table name this model maps to, before any quoting is applied. */
+  private get mappedTableName(): string {
     const name = this.modelName;
     let t = name;
     const modelToTable = getModelToTable();
@@ -744,9 +780,17 @@ export class AdapterTableClient<T = any> {
         if (modelToTable[lower]) t = modelToTable[lower];
       }
     }
-    if (t.startsWith('[') || t.startsWith('"') || t.startsWith('`')) return t;
-    if (t.includes('.')) return t.split('.').map(p => quote(p, this.dialect)).join('.');
-    return quote(t, this.dialect);
+    return t;
+  }
+
+  /**
+   * The bare table name, for helpers that take a string literal. `@@map` values
+   * may be pre-quoted or schema-qualified, neither of which `pragma_table_info`
+   * accepts, so the last segment is unwrapped here.
+   */
+  public get rawTableName(): string {
+    const last = this.mappedTableName.split('.').pop() ?? this.mappedTableName;
+    return last.replace(/^[\s]*[\[\]"`]+|[\[\]"`]+[\s]*$/g, '');
   }
 
   private get nolock(): string {
@@ -770,6 +814,53 @@ export class AdapterTableClient<T = any> {
     return this.adapter._executeRaw(query, params);
   }
 
+  /**
+   * Encode every `VECTOR(n)` value on its way into SQLite.
+   *
+   * The column holds a float32 BLOB, which is a third of the size of the JSON
+   * text it replaces and is what sqlite-vec reads. Values that already arrived
+   * as text or bytes are left alone, so rows written by an older version stay
+   * readable.
+   */
+  private encodeVectorFields(data: Record<string, any>): Record<string, any> {
+    if (this.dialect !== 'sqlite') return data;
+    const fields = getFieldsForModel(this.modelName);
+    const vectorFields = Object.keys(fields).filter((name) => isVectorField(fields[name]));
+    if (vectorFields.length === 0) return data;
+    const { encodeVector } = require('./sqlite/vector');
+    const out = { ...data };
+    for (const name of vectorFields) {
+      const value = out[name];
+      if (needsVectorEncoding(value)) {
+        const parsed = Array.from(value as ArrayLike<number>);
+        if (parsed.length > 0) out[name] = encodeVector(parsed);
+      }
+    }
+    return out;
+  }
+
+  /** Decode `VECTOR(n)` columns back into `number[]` after a read. */
+  private decodeVectorFields<T>(rows: T[]): T[] {
+    if (this.dialect !== 'sqlite' || !Array.isArray(rows) || rows.length === 0) return rows;
+    const fields = getFieldsForModel(this.modelName);
+    const vectorFields = Object.keys(fields).filter((name) => isVectorField(fields[name]));
+    if (vectorFields.length === 0) return rows;
+    return rows.map((row: any) => {
+      if (!row || typeof row !== 'object') return row;
+      let changed = false;
+      const out: any = { ...row };
+      for (const name of vectorFields) {
+        if (!(name in out)) continue;
+        const decoded = decodeVector(out[name]);
+        if (decoded) {
+          out[name] = decoded;
+          changed = true;
+        }
+      }
+      return changed ? out : row;
+    });
+  }
+
   async findMany(args?: { where?: any; orderBy?: any; skip?: number; take?: number; select?: any; include?: any }): Promise<T[]> {
     const params: Record<string, any> = {};
     const whereCtx = this.whereContext();
@@ -825,7 +916,7 @@ export class AdapterTableClient<T = any> {
       if (orderSql) query += ` ${orderSql}`;
     }
 
-    const rows = await this.doExec<T>(query, params);
+    const rows = this.decodeVectorFields(await this.doExec<T>(query, params));
 
     if (args?.include) {
       await resolveIncludes(this.modelName, rows, args.include, this.adapter);
@@ -875,7 +966,7 @@ export class AdapterTableClient<T = any> {
     const fields = getFieldsForModel(this.modelName);
     const idFieldName = resolveIdField(fields);
 
-    const data: any = { ...rawData };
+    const data: any = this.encodeVectorFields(rawData);
     if (idFieldName) {
       const fieldDef: any = fields[idFieldName];
       const rawType = typeof fieldDef === 'string' ? fieldDef : (fieldDef?.ts || fieldDef?.sql || fieldDef?.type || '');
@@ -954,9 +1045,10 @@ export class AdapterTableClient<T = any> {
 
       for (let r = 0; r < args.data.length; r++) {
         const row = args.data[r] as any;
+        const encoded = this.encodeVectorFields(row);
         const vals = firstCols.map(col => {
           const p = `r${r}_${col}`;
-          params[p] = row[col] ?? null;
+          params[p] = encoded[col] ?? null;
           return `@${p}`;
         });
         rowPlaceholders.push(`(${vals.join(', ')})`);
@@ -988,13 +1080,14 @@ export class AdapterTableClient<T = any> {
       }
     }
 
+    const data = this.encodeVectorFields(rawData);
     const setCols = Object.keys(rawData).filter(k => rawData[k] !== undefined);
     if (setCols.length > 0) {
       const params: Record<string, any> = {};
       const whereSql = parseWhere(this.modelName, args.where, params, this.dialect, 'w_', this.whereContext());
       const sets: string[] = [];
       for (const col of setCols) {
-        appendUpdateSet(sets, params, col, rawData[col], this.dialect);
+        appendUpdateSet(sets, params, col, data[col], this.dialect);
       }
       if (sets.length > 0) {
         const query = `UPDATE ${this.tableName} SET ${sets.join(', ')}${whereSql ? ` WHERE ${whereSql}` : ''}`;
@@ -1050,11 +1143,12 @@ export class AdapterTableClient<T = any> {
   async updateMany(args: { where?: any; data: Partial<T> }): Promise<{ count: number }> {
     const params: Record<string, any> = {};
     const whereSql = parseWhere(this.modelName, args.where, params, this.dialect, 'w_', this.whereContext());
+    const data = this.encodeVectorFields(args.data as Record<string, any>);
     const setCols = Object.keys(args.data).filter(k => (args.data as any)[k] !== undefined);
     const sets: string[] = [];
 
     for (const col of setCols) {
-      appendUpdateSet(sets, params, col, (args.data as any)[col], this.dialect);
+      appendUpdateSet(sets, params, col, data[col], this.dialect);
     }
 
     if (sets.length === 0) return { count: 0 };
@@ -1410,6 +1504,28 @@ export class AdapterTableClient<T = any> {
         const msg = String(err?.message || '').toLowerCase();
         if (!msg.includes('vector') && !msg.includes('operator does not exist')) throw err;
       }
+    } else if (this.dialect === 'sqlite') {
+      // SQLite has no vector type, so the ranking runs through sqlite-vec, a
+      // driver function, or json_each — whichever this connection supports.
+      const whereSql = parseWhere(this.modelName, args.where, params, this.dialect, '', this.whereContext());
+      const support = await (this.adapter as any).sqliteVectorSupport?.() ?? null;
+      const ranked = await runSqliteVectorSearch({
+        exec: (sql, p) => this.doExec<any>(sql, p),
+        vectorSupport: support,
+        table: this.tableName,
+        rawTable: this.rawTableName,
+        column: col,
+        rawColumn: vectorField,
+        vector: args.vector,
+        metric,
+        take,
+        tail: whereSql ? ` WHERE ${whereSql}` : '',
+        params,
+        preference: (this.adapter as any).vectorStrategy,
+      });
+      if (ranked) return this.decodeVectorFields(ranked.rows) as (T & { distance: number })[];
+      // No in-database strategy: score the column in the client instead.
+      return this.scoreVectorsInMemory(args, metric, take, vectorField);
     } else if (this.dialect === 'mssql') {
       const distFn = metric === 'cosine' ? 'cosine' : metric === 'euclidean' ? 'euclidean' : 'dot';
       const whereSql = parseWhere(this.modelName, args.where, params, this.dialect, '', this.whereContext());
@@ -1436,30 +1552,31 @@ export class AdapterTableClient<T = any> {
     }
 
     // 2. Fallback: in-memory similarity computation when native vector support is unavailable
+    return this.scoreVectorsInMemory(args, metric, take, vectorField);
+  }
+
+  /**
+   * Rank the vector column in the client. This is the last resort for every
+   * provider, and the only option on SQLite connections that offer neither
+   * sqlite-vec, a driver function nor JSON1.
+   *
+   * The whole table is loaded, so it is only reasonable for a table small
+   * enough to fit in memory.
+   */
+  private async scoreVectorsInMemory(
+    args: { vector: number[]; where?: any },
+    metric: 'cosine' | 'euclidean' | 'dot',
+    take: number,
+    vectorField = 'embedding',
+  ): Promise<(T & { distance: number })[]> {
     const rows = await this.findMany({ where: args.where });
 
     const scored: { row: T; dist: number }[] = [];
     for (const row of rows) {
-      const raw = (row as any)[vectorField];
-      if (!raw) continue;
-      let vec: number[] = [];
-      try { vec = typeof raw === 'string' ? JSON.parse(raw) : raw; } catch { continue; }
-      if (!Array.isArray(vec) || vec.length !== args.vector.length) continue;
-
-      let dot = 0, m1 = 0, m2 = 0;
-      for (let i = 0; i < args.vector.length; i++) {
-        const av = args.vector[i] ?? 0;
-        const bv = vec[i] ?? 0;
-        dot += av * bv;
-        m1 += av ** 2;
-        m2 += bv ** 2;
-      }
-      const cosine = m1 && m2 ? dot / (Math.sqrt(m1) * Math.sqrt(m2)) : 0;
-      const dist = metric === 'cosine'
-        ? 1 - cosine
-        : metric === 'dot'
-          ? -dot
-          : Math.sqrt(args.vector.reduce((s, v, i) => s + (v - (vec[i] ?? 0)) ** 2, 0));
+      const vec = decodeVector((row as any)[vectorField]);
+      if (!vec) continue;
+      const dist = vectorDistance(args.vector, vec, metric);
+      if (dist === null) continue;
       scored.push({ row, dist });
     }
     scored.sort((a, b) => a.dist - b.dist);

package/typescript/src/base/types.ts

@@ -1,3 +1,5 @@
+import type { SqliteVectorSupport, VectorStrategyPreference } from '../sqlite/vector';
+
 export type Dialect = 'mssql' | 'postgres' | 'mysql' | 'sqlite' | 'googlesheets';
 
 export interface An5AdapterConfig {
@@ -8,6 +10,17 @@ export interface An5AdapterConfig {
    * run in NBase while the rows stay in `connectionString`'s database.
    */
   nbase?: string;
+  /**
+   * Path to sqlite-vec or the AN5 native vector extension. On SQLite it is loaded before the
+   * vector capabilities are probed, so `vectorSearch` ranks inside the database
+   * instead of loading the table into memory. Other providers ignore it.
+   */
+  sqliteVec?: string;
+  /**
+   * Pin the SQLite vector search strategy. `auto` (the default) probes for
+   * sqlite-vec, a driver function and JSON1, and uses the first that works.
+   */
+  vectorStrategy?: VectorStrategyPreference;
   engine?: QueryEngine;
   db?: any;
   driver?: any;
@@ -30,4 +43,6 @@ export interface QueryEngine {
   connect(): Promise<void>;
   disconnect(): Promise<void>;
   beginTransaction(): Promise<TransactionHandle>;
+  /** SQLite only: extension loading and distance-function registration. */
+  vectorSupport?: SqliteVectorSupport;
 }

package/typescript/src/index.ts

@@ -29,6 +29,19 @@ export type {
   AdapterMetadata,
   Dialect,
 } from './an5Adapter';
+export {
+  encodeVector,
+  decodeVector,
+  vectorDistance,
+  isVectorField,
+  AN5_VECTOR_FUNCTIONS,
+  SQLITE_VEC_FUNCTIONS,
+} from './sqlite/vector';
+export type {
+  SqliteVectorStrategy,
+  VectorMetric,
+  VectorStrategyPreference,
+} from './sqlite/vector';
 export type { TypedAn5Adapter, AdapterAPI } from './typed';
 export {
   getLlmConfig, setLlmConfig,

package/typescript/src/sqlite/browserEngine.ts

@@ -1,4 +1,5 @@
 import type { Dialect, QueryEngine, TransactionHandle } from '../base/types';
+import { SqliteVectorSupport } from './vector';
 
 // ─── Browser SQLite Driver & Config ──────────────────────────────────────────────────
 
@@ -13,18 +14,41 @@ export interface SqliteBrowserConfig {
   driver?: SqliteDriver;
   exec?: <T = any>(query: string, params?: Record<string, any>) => Promise<T[]> | T[];
   executeRaw?: (query: string, params?: Record<string, any>) => Promise<number> | number;
+  /** Path to the sqlite-vec extension, for WASM builds that allow loading one. */
+  sqliteVec?: string;
+  /** Pin the vector search strategy instead of probing for one. */
+  vectorStrategy?: 'auto' | 'sqlite-vec' | 'udf' | 'sql' | 'memory';
+  /** Register a scalar function with the driver, when its API allows it. */
+  registerFunction?: (name: string, fn: (...args: any[]) => any) => void;
 }
 
 // ─── Browser SQLite Engine ──────────────────────────────────────────────────────────
 
 export class SqliteBrowserEngine implements QueryEngine {
   dialect: Dialect = 'sqlite';
+  /**
+   * A WASM build cannot register a JS callback as an SQL function, so this only
+   * reaches the `sql` strategy (json_each in plain SQL) and the in-memory one.
+   * A custom `registerFunction` hook unlocks `udf` as well.
+   */
+  readonly vectorSupport: SqliteVectorSupport;
   private db: any = null;
   private driver: SqliteDriver | null = null;
   private customExec: ((query: string, params?: Record<string, any>) => Promise<any[]> | any[]) | null = null;
   private customExecuteRaw: ((query: string, params?: Record<string, any>) => Promise<number> | number) | null = null;
 
   constructor(config?: SqliteBrowserConfig | any) {
+    this.vectorSupport = new SqliteVectorSupport({
+      native: () => this.db,
+      registerFunction: (name, fn) => {
+        const register = config?.registerFunction;
+        if (typeof register === 'function') return register(name, fn);
+        const db = this.db;
+        if (db && typeof db.create_function === 'function') return db.create_function(name, fn);
+        throw new Error('this SQLite build cannot register functions');
+      },
+    }, { sqliteVec: config?.sqliteVec, vectorStrategy: config?.vectorStrategy });
+
     if (!config) return;
 
     if (config.exec && typeof config.exec === 'function') {

package/typescript/src/sqlite/engine.ts

@@ -1,4 +1,5 @@
 import type { An5AdapterConfig, Dialect, QueryEngine, TransactionHandle } from '../base/types';
+import { SqliteVectorSupport } from './vector';
 
 // ─── SQLite Engine ─────────────────────────────────────────────────────────────────
 
@@ -7,6 +8,8 @@ try { BetterSqlite3 = require('better-sqlite3'); } catch { }
 
 export class SqliteEngine implements QueryEngine {
   dialect: Dialect = 'sqlite';
+  /** Registers `an5_vec_*` and loads sqlite-vec for in-database vector search. */
+  readonly vectorSupport: SqliteVectorSupport;
   private db: any = null;
   private readonly filePath: string;
 
@@ -16,6 +19,15 @@ export class SqliteEngine implements QueryEngine {
       .replace(/^sqlite:\/\/\//i, '/')
       .replace(/^sqlite:\/\//i, '')
       .replace(/^sqlite:/i, '');
+    this.vectorSupport = new SqliteVectorSupport({
+      native: () => this.getDb(),
+      registerFunction: (name, fn) => this.getDb().function(name, { deterministic: true }, fn),
+    }, { sqliteVec: adapterConfig.sqliteVec, vectorStrategy: adapterConfig.vectorStrategy });
+  }
+
+  /** The better-sqlite3 handle, needed for extension loading. */
+  native(): any {
+    return this.getDb();
   }
 
   private getDb(): any {
@@ -42,6 +54,7 @@ export class SqliteEngine implements QueryEngine {
 
   async disconnect(): Promise<void> {
     if (this.db) { this.db.close(); this.db = null; }
+    this.vectorSupport.reset();
   }
 
   async beginTransaction(): Promise<TransactionHandle> {

package/typescript/src/sqlite/index.ts

@@ -1,3 +1,3 @@
 export * from './engine';
 export * from './browserEngine';
-
+export * from './vector';