@an5/adapters 0.2.6 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -2
- package/dist/an5Adapter.d.ts +57 -0
- package/dist/an5Adapter.js +262 -4
- package/dist/base/types.d.ts +6 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +9 -1
- package/dist/nbase/client.d.ts +92 -0
- package/dist/nbase/client.js +147 -0
- package/dist/nbase/index.d.ts +47 -0
- package/dist/nbase/index.js +80 -0
- package/dotnet/An5Adapter.cs +41 -18
- package/dotnet/Base/Core.cs +42 -2
- package/dotnet/Postgres/PostgresEngine.cs +58 -30
- package/dotnet/Sqlite/SqliteEngine.cs +214 -0
- package/package.json +15 -8
- package/python/an5_adapter.py +74 -9
- package/python/base/__init__.py +1 -1
- package/python/base/metadata.py +25 -1
- package/python/base/sql.py +44 -14
- package/python/sqlite/__init__.py +3 -0
- package/python/sqlite/provider.py +83 -0
- package/python/table_client.py +53 -36
- package/rust/Cargo.toml +31 -0
- package/rust/README.md +71 -0
- package/rust/src/adapter.rs +1312 -0
- package/rust/src/base/dialect.rs +172 -0
- package/rust/src/base/metadata.rs +134 -0
- package/rust/src/base/mod.rs +17 -0
- package/rust/src/base/where_builder.rs +405 -0
- package/rust/src/lib.rs +75 -0
- package/typescript/src/an5Adapter.ts +299 -4
- package/typescript/src/base/types.ts +6 -0
- package/typescript/src/index.ts +17 -0
- package/typescript/src/nbase/client.ts +214 -0
- package/typescript/src/nbase/index.ts +104 -0
|
@@ -7,6 +7,25 @@ import {
|
|
|
7
7
|
} from './googlesheets';
|
|
8
8
|
import { parseSheetsConnectionString } from './googlesheets/parseConnectionString';
|
|
9
9
|
import type { An5AdapterConfig, Dialect, QueryEngine, TransactionHandle } from './base/types';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* File extensions that mean SQLite when a connection string has no scheme.
|
|
13
|
+
*
|
|
14
|
+
* `.sqlite3` is here as well as in `@an5/orm`'s `detectProvider`, because the ORM
|
|
15
|
+
* validates field types and writes the DDL from the provider this file picks: if the
|
|
16
|
+
* two lists differed, a `.sqlite3` file would be pushed as SQL Server DDL.
|
|
17
|
+
*/
|
|
18
|
+
const SQLITE_FILE_SUFFIXES = ['.sqlite', '.sqlite3', '.db'];
|
|
19
|
+
import {
|
|
20
|
+
NBaseError,
|
|
21
|
+
buildVectorId,
|
|
22
|
+
parseVectorId,
|
|
23
|
+
isNBaseConnectionString,
|
|
24
|
+
parseNBaseConnectionString,
|
|
25
|
+
createNBaseVectorClient,
|
|
26
|
+
type NBaseAdapterConfig,
|
|
27
|
+
type NBaseVectorClient,
|
|
28
|
+
} from './nbase';
|
|
10
29
|
import { buildOrderBy, parseWhere, quote } from './base/sql';
|
|
11
30
|
import {
|
|
12
31
|
createAdapterProxy,
|
|
@@ -204,6 +223,34 @@ export class An5Adapter {
|
|
|
204
223
|
private _engineType: 'postgres' | 'mysql' | 'sqlite' | 'mssql' | null = null;
|
|
205
224
|
private _engineConfig: An5AdapterConfig | null = null;
|
|
206
225
|
sheetsAdapter: An5SheetsAdapter | null = null;
|
|
226
|
+
private _nbase: NBaseVectorClient | null = null;
|
|
227
|
+
private _nbaseConfig: NBaseAdapterConfig | null = null;
|
|
228
|
+
private _nbaseOnly = false;
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Returns the NBase client for this adapter, if the project configured one.
|
|
232
|
+
* The connection string may also point straight at NBase, which keeps
|
|
233
|
+
* `DATABASE_URL=nbase://host:port` working on its own.
|
|
234
|
+
*/
|
|
235
|
+
nbaseClient(): { client: NBaseVectorClient; config: NBaseAdapterConfig } | null {
|
|
236
|
+
if (this._nbase && this._nbaseConfig) return { client: this._nbase, config: this._nbaseConfig };
|
|
237
|
+
|
|
238
|
+
if (!this._nbaseConfig) return null;
|
|
239
|
+
const config = this._nbaseConfig;
|
|
240
|
+
|
|
241
|
+
const client: NBaseVectorClient = createNBaseVectorClient(config);
|
|
242
|
+
this._nbase = client;
|
|
243
|
+
this._nbaseConfig = config;
|
|
244
|
+
return { client, config };
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* True when the adapter only knows about NBase, so there is no relational
|
|
249
|
+
* database to read rows from.
|
|
250
|
+
*/
|
|
251
|
+
get nbaseOnly(): boolean {
|
|
252
|
+
return this._nbaseOnly === true;
|
|
253
|
+
}
|
|
207
254
|
|
|
208
255
|
get dialect(): Dialect {
|
|
209
256
|
if (this.sheetsAdapter) return 'googlesheets';
|
|
@@ -220,6 +267,25 @@ export class An5Adapter {
|
|
|
220
267
|
return createAdapterProxy(this, (name) => this.table(name));
|
|
221
268
|
}
|
|
222
269
|
|
|
270
|
+
// Keep the config even when the caller supplies its own engine: the NBase
|
|
271
|
+
// settings and anything else on the config are read from it later.
|
|
272
|
+
this._engineConfig = adapterConfig as An5AdapterConfig;
|
|
273
|
+
|
|
274
|
+
// `nbase://host:port` as the connection string configures the vector store
|
|
275
|
+
// on its own: there is no relational database, so the adapter is
|
|
276
|
+
// vector-only. The same string passed as `nbase` keeps the database from
|
|
277
|
+
// `connectionString` as the source of rows.
|
|
278
|
+
const rawConnectionString = String(this._engineConfig.connectionString ?? '').trim();
|
|
279
|
+
if (isNBaseConnectionString(rawConnectionString)) {
|
|
280
|
+
this._nbaseConfig = parseNBaseConnectionString(rawConnectionString);
|
|
281
|
+
this._nbaseOnly = true;
|
|
282
|
+
return createAdapterProxy(this, (name) => this.table(name));
|
|
283
|
+
}
|
|
284
|
+
const attached = this._engineConfig.nbase;
|
|
285
|
+
if (attached && isNBaseConnectionString(attached)) {
|
|
286
|
+
this._nbaseConfig = parseNBaseConnectionString(attached);
|
|
287
|
+
}
|
|
288
|
+
|
|
223
289
|
if ((adapterConfig as any).engine) {
|
|
224
290
|
this._engine = (adapterConfig as any).engine;
|
|
225
291
|
this._engineType = (this._engine?.dialect as any) || 'sqlite';
|
|
@@ -233,18 +299,27 @@ export class An5Adapter {
|
|
|
233
299
|
return createAdapterProxy(this, (name) => this.table(name));
|
|
234
300
|
}
|
|
235
301
|
|
|
302
|
+
// Lower-cased for the dialect test only; the connection string itself keeps its
|
|
303
|
+
// case for the driver. A URI scheme is case-insensitive, and `@an5/orm` reads the
|
|
304
|
+
// provider the same way to validate field types and write DDL — if the two
|
|
305
|
+
// disagreed, a schema would be checked against one database and run on another.
|
|
236
306
|
const cs = (adapterConfig.connectionString || '').trim();
|
|
237
|
-
|
|
307
|
+
const forDialect = cs.toLowerCase();
|
|
308
|
+
|
|
309
|
+
if (forDialect.startsWith('googlesheets://')) {
|
|
238
310
|
this.sheetsAdapter = new An5SheetsAdapter(parseSheetsConnectionString(cs));
|
|
239
311
|
return createAdapterProxy(this, (name) => this.table(name));
|
|
240
312
|
}
|
|
241
313
|
|
|
242
314
|
this._engineConfig = adapterConfig;
|
|
243
|
-
if (
|
|
315
|
+
if (forDialect.startsWith('postgres://') || forDialect.startsWith('postgresql://')) {
|
|
244
316
|
this._engineType = 'postgres';
|
|
245
|
-
} else if (
|
|
317
|
+
} else if (forDialect.startsWith('mysql://') || forDialect.startsWith('mariadb://')) {
|
|
246
318
|
this._engineType = 'mysql';
|
|
247
|
-
} else if (
|
|
319
|
+
} else if (
|
|
320
|
+
forDialect.startsWith('sqlite://')
|
|
321
|
+
|| SQLITE_FILE_SUFFIXES.some((suffix) => forDialect.endsWith(suffix))
|
|
322
|
+
) {
|
|
248
323
|
this._engineType = 'sqlite';
|
|
249
324
|
} else {
|
|
250
325
|
this._engineType = 'mssql';
|
|
@@ -336,9 +411,35 @@ export class An5Adapter {
|
|
|
336
411
|
|
|
337
412
|
async $connect(): Promise<void> {
|
|
338
413
|
if (this.sheetsAdapter) return this.sheetsAdapter.$connect();
|
|
414
|
+
if (this._nbaseOnly) {
|
|
415
|
+
// NBase is stateless over HTTP, so there is nothing to open. Report the
|
|
416
|
+
// endpoint so a misconfiguration is obvious.
|
|
417
|
+
const config = this.nbaseClient()?.config;
|
|
418
|
+
if (config) await this.nbaseClient()!.client.health();
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
339
421
|
await (await this.requireEngine()).connect();
|
|
340
422
|
}
|
|
341
423
|
|
|
424
|
+
/**
|
|
425
|
+
* Health check for the configured NBase instance, or the database when the
|
|
426
|
+
* adapter has none.
|
|
427
|
+
*/
|
|
428
|
+
async $nbaseHealth(): Promise<{ connected: boolean; endpoint?: string; error?: string }> {
|
|
429
|
+
const nbase = this.nbaseClient();
|
|
430
|
+
if (!nbase) return { connected: false, error: 'No NBase instance configured.' };
|
|
431
|
+
try {
|
|
432
|
+
const health = await nbase.client.health();
|
|
433
|
+
return { connected: health.status === 'ok', endpoint: nbase.config.url };
|
|
434
|
+
} catch (err) {
|
|
435
|
+
return {
|
|
436
|
+
connected: false,
|
|
437
|
+
endpoint: nbase.config.url,
|
|
438
|
+
error: err instanceof Error ? err.message : String(err),
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
342
443
|
async $disconnect(): Promise<void> {
|
|
343
444
|
if (this.sheetsAdapter) return this.sheetsAdapter.$disconnect();
|
|
344
445
|
await (await this.requireEngine()).disconnect();
|
|
@@ -1059,6 +1160,181 @@ export class AdapterTableClient<T = any> {
|
|
|
1059
1160
|
return this.doExec(query, params);
|
|
1060
1161
|
}
|
|
1061
1162
|
|
|
1163
|
+
/**
|
|
1164
|
+
* Runs similarity search in NBase and hydrates the matching rows from the
|
|
1165
|
+
* relational table, keeping the `{ ...row, distance }` shape the other
|
|
1166
|
+
* engines return.
|
|
1167
|
+
*/
|
|
1168
|
+
private async searchViaNBase(
|
|
1169
|
+
nbase: { client: NBaseVectorClient; config: NBaseAdapterConfig },
|
|
1170
|
+
args: { vector: number[]; take?: number; where?: any },
|
|
1171
|
+
metric: 'cosine' | 'euclidean' | 'dot',
|
|
1172
|
+
take: number,
|
|
1173
|
+
): Promise<(T & { distance: number })[]> {
|
|
1174
|
+
const { client, config } = nbase;
|
|
1175
|
+
const idField = config.idField ?? 'an5Id';
|
|
1176
|
+
const modelField = config.modelField ?? 'an5Model';
|
|
1177
|
+
|
|
1178
|
+
const results = await client.search(args.vector, {
|
|
1179
|
+
k: take,
|
|
1180
|
+
// NBase implements cosine and euclidean; a dot-product request falls back
|
|
1181
|
+
// to cosine, which still orders the results sensibly.
|
|
1182
|
+
distanceMetric: metric === 'euclidean' ? 'euclidean' : 'cosine',
|
|
1183
|
+
includeMetadata: true,
|
|
1184
|
+
...(config.method ? { method: config.method } : {}),
|
|
1185
|
+
});
|
|
1186
|
+
|
|
1187
|
+
// Map each hit back to its row id. Ids are written as `Model:id`, and the
|
|
1188
|
+
// metadata key is honoured for projects that index vectors themselves.
|
|
1189
|
+
const ids: string[] = [];
|
|
1190
|
+
const distances = new Map<string, number>();
|
|
1191
|
+
for (const hit of results) {
|
|
1192
|
+
const fromMetadata = hit.metadata?.[idField];
|
|
1193
|
+
const rawId = fromMetadata !== undefined && fromMetadata !== null
|
|
1194
|
+
? String(fromMetadata)
|
|
1195
|
+
: parseVectorId(hit.id).id;
|
|
1196
|
+
if (!rawId) continue;
|
|
1197
|
+
const model = hit.metadata?.[modelField] ?? parseVectorId(hit.id).model;
|
|
1198
|
+
ids.push(model === this.modelName ? rawId : `${model}:${rawId}`);
|
|
1199
|
+
distances.set(ids[ids.length - 1] as string, hit.dist);
|
|
1200
|
+
}
|
|
1201
|
+
if (ids.length === 0) return [];
|
|
1202
|
+
|
|
1203
|
+
const pk = this.primaryKeyColumn();
|
|
1204
|
+
const inList = ids.map((_, i) => `@nbase_id_${i}`).join(', ');
|
|
1205
|
+
const idParams: Record<string, any> = {};
|
|
1206
|
+
ids.forEach((id, i) => { idParams[`nbase_id_${i}`] = id; });
|
|
1207
|
+
idParams.take = take;
|
|
1208
|
+
|
|
1209
|
+
const selectColumns = this.modelFields().map((f) => quote(f, this.dialect)).join(', ');
|
|
1210
|
+
const columns = selectColumns ? `${selectColumns}, ${quote(pk, this.dialect)} AS __an5_nbase_key` : `${quote(pk, this.dialect)} AS __an5_nbase_key`;
|
|
1211
|
+
const query = `SELECT ${columns} FROM ${this.tableName} WHERE ${quote(pk, this.dialect)} IN (${inList})`;
|
|
1212
|
+
|
|
1213
|
+
const rows = await this.adapter.exec<Record<string, any>>(query, idParams);
|
|
1214
|
+
const byKey = new Map<string, Record<string, any>>();
|
|
1215
|
+
for (const row of rows ?? []) {
|
|
1216
|
+
byKey.set(String(row.__an5_nbase_key), row);
|
|
1217
|
+
}
|
|
1218
|
+
|
|
1219
|
+
const hydrated: (T & { distance: number })[] = [];
|
|
1220
|
+
for (const id of ids) {
|
|
1221
|
+
const row = byKey.get(id);
|
|
1222
|
+
if (!row) continue;
|
|
1223
|
+
const { __an5_nbase_key, ...rest } = row;
|
|
1224
|
+
hydrated.push({ ...(rest as T), distance: distances.get(id) ?? 0 } as T & { distance: number });
|
|
1225
|
+
if (hydrated.length >= take) break;
|
|
1226
|
+
}
|
|
1227
|
+
return hydrated;
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
/**
|
|
1231
|
+
* Vector search for an adapter configured with only an NBase connection
|
|
1232
|
+
* string. The hit id and metadata are the result, since there is no
|
|
1233
|
+
* relational row to hydrate.
|
|
1234
|
+
*/
|
|
1235
|
+
private async searchViaNBaseOnly(
|
|
1236
|
+
nbase: { client: NBaseVectorClient; config: NBaseAdapterConfig },
|
|
1237
|
+
args: { vector: number[]; take?: number },
|
|
1238
|
+
metric: 'cosine' | 'euclidean' | 'dot',
|
|
1239
|
+
take: number,
|
|
1240
|
+
): Promise<Array<Record<string, unknown> & { distance: number }>> {
|
|
1241
|
+
const { client, config } = nbase;
|
|
1242
|
+
const idField = config.idField ?? 'an5Id';
|
|
1243
|
+
const results = await client.search(args.vector, {
|
|
1244
|
+
k: take,
|
|
1245
|
+
distanceMetric: metric === 'euclidean' ? 'euclidean' : 'cosine',
|
|
1246
|
+
includeMetadata: true,
|
|
1247
|
+
...(config.method ? { method: config.method } : {}),
|
|
1248
|
+
});
|
|
1249
|
+
|
|
1250
|
+
return results.map((hit) => {
|
|
1251
|
+
const metadata = { ...(hit.metadata ?? {}) };
|
|
1252
|
+
if (metadata[idField] === undefined) {
|
|
1253
|
+
const parsed = parseVectorId(hit.id);
|
|
1254
|
+
if (parsed.id) metadata[idField] = parsed.id;
|
|
1255
|
+
if (parsed.model) metadata[config.modelField ?? 'an5Model'] = parsed.model;
|
|
1256
|
+
}
|
|
1257
|
+
return { id: hit.id, ...metadata, distance: hit.dist };
|
|
1258
|
+
});
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
/** The primary key column registered for this model, falling back to `id`. */
|
|
1262
|
+
private primaryKeyColumn(): string {
|
|
1263
|
+
const fields = this.modelFields();
|
|
1264
|
+
const marked = fields.find((f) => f === 'id');
|
|
1265
|
+
return marked ?? fields[0] ?? 'id';
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1268
|
+
/** Scalar column names known for this model from the generated metadata. */
|
|
1269
|
+
private modelFields(): string[] {
|
|
1270
|
+
const fields = getFieldsForModel(this.modelName);
|
|
1271
|
+
if (fields && typeof fields === 'object' && !Array.isArray(fields)) {
|
|
1272
|
+
return Object.keys(fields as Record<string, unknown>);
|
|
1273
|
+
}
|
|
1274
|
+
return [];
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
/**
|
|
1278
|
+
* Pushes a table's embedding column into NBase so it can be searched there.
|
|
1279
|
+
*
|
|
1280
|
+
* Each vector keeps the row id in its metadata, which is what lets
|
|
1281
|
+
* `vectorSearch` return real rows instead of bare vector ids.
|
|
1282
|
+
*/
|
|
1283
|
+
async indexVectorsInNBase(args: {
|
|
1284
|
+
vectorField?: string;
|
|
1285
|
+
where?: any;
|
|
1286
|
+
take?: number;
|
|
1287
|
+
batchSize?: number;
|
|
1288
|
+
} = {}): Promise<{ indexed: number }> {
|
|
1289
|
+
const nbase = this.adapter.nbaseClient?.() ?? null;
|
|
1290
|
+
if (!nbase) {
|
|
1291
|
+
throw new Error(
|
|
1292
|
+
'No NBase instance configured. Use a nbase:// connection string, e.g. createAn5Adapter({ connectionString: "nbase://localhost:1307" }).',
|
|
1293
|
+
);
|
|
1294
|
+
}
|
|
1295
|
+
if (this.adapter.nbaseOnly) {
|
|
1296
|
+
throw new Error(
|
|
1297
|
+
'indexVectorsInNBase reads the vectors from a table, but this adapter was created with an nbase:// connection string and has no database. ' +
|
|
1298
|
+
'Create the adapter with your database connection string to index, or add the vectors with the NBase client directly.',
|
|
1299
|
+
);
|
|
1300
|
+
}
|
|
1301
|
+
const vectorField = args.vectorField || 'embedding';
|
|
1302
|
+
const batchSize = args.batchSize ?? 200;
|
|
1303
|
+
const pk = this.primaryKeyColumn();
|
|
1304
|
+
const rows = await this.findMany({ ...(args.where ? { where: args.where } : {}), ...(args.take ? { take: args.take } : {}) });
|
|
1305
|
+
|
|
1306
|
+
let indexed = 0;
|
|
1307
|
+
for (let i = 0; i < rows.length; i += batchSize) {
|
|
1308
|
+
const batch = rows.slice(i, i + batchSize);
|
|
1309
|
+
const vectors = batch
|
|
1310
|
+
.map((row) => {
|
|
1311
|
+
const raw = (row as Record<string, unknown>)[vectorField];
|
|
1312
|
+
if (raw === undefined || raw === null) return null;
|
|
1313
|
+
let vector: number[];
|
|
1314
|
+
try {
|
|
1315
|
+
vector = typeof raw === 'string' ? JSON.parse(raw) : (raw as number[]);
|
|
1316
|
+
} catch {
|
|
1317
|
+
return null;
|
|
1318
|
+
}
|
|
1319
|
+
if (!Array.isArray(vector) || vector.length === 0) return null;
|
|
1320
|
+
return {
|
|
1321
|
+
id: buildVectorId(this.modelName, String((row as Record<string, unknown>)[pk])),
|
|
1322
|
+
vector,
|
|
1323
|
+
metadata: {
|
|
1324
|
+
[nbase.config.idField ?? 'an5Id']: String((row as Record<string, unknown>)[pk]),
|
|
1325
|
+
[nbase.config.modelField ?? 'an5Model']: this.modelName,
|
|
1326
|
+
},
|
|
1327
|
+
};
|
|
1328
|
+
})
|
|
1329
|
+
.filter((v): v is NonNullable<typeof v> => v !== null);
|
|
1330
|
+
|
|
1331
|
+
if (vectors.length === 0) continue;
|
|
1332
|
+
const result = await nbase.client.addVectors(vectors);
|
|
1333
|
+
indexed += result.count;
|
|
1334
|
+
}
|
|
1335
|
+
return { indexed };
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1062
1338
|
async vectorSearch(args: {
|
|
1063
1339
|
vector: number[];
|
|
1064
1340
|
take?: number;
|
|
@@ -1078,6 +1354,25 @@ export class AdapterTableClient<T = any> {
|
|
|
1078
1354
|
const col = quote(vectorField, this.dialect);
|
|
1079
1355
|
const params: Record<string, any> = { query_vector: vectorJson };
|
|
1080
1356
|
|
|
1357
|
+
// 0. NBase, when the project configured one: the vectors live in the vector
|
|
1358
|
+
// database, so the search never loads the table into memory.
|
|
1359
|
+
const nbase = this.adapter.nbaseClient?.() ?? null;
|
|
1360
|
+
if (nbase) {
|
|
1361
|
+
try {
|
|
1362
|
+
// With an NBase-only connection string there is no table to read rows
|
|
1363
|
+
// from, so the hits are returned as the vector metadata itself.
|
|
1364
|
+
if (this.adapter.nbaseOnly) {
|
|
1365
|
+
return (await this.searchViaNBaseOnly(nbase, args, metric, take)) as (T & { distance: number })[];
|
|
1366
|
+
}
|
|
1367
|
+
return await this.searchViaNBase(nbase, args, metric, take);
|
|
1368
|
+
} catch (err) {
|
|
1369
|
+
if (!(err instanceof NBaseError)) throw err;
|
|
1370
|
+
// A NBase outage must not take vector search down with it; fall through
|
|
1371
|
+
// to the database instead of failing the query.
|
|
1372
|
+
console.warn(`[an5] NBase vector search unavailable (${err.message}); falling back to ${this.dialect}.`);
|
|
1373
|
+
}
|
|
1374
|
+
}
|
|
1375
|
+
|
|
1081
1376
|
// 1. Try native dialect execution
|
|
1082
1377
|
if (this.dialect === 'postgres') {
|
|
1083
1378
|
const op = metric === 'cosine' ? '<=>' : metric === 'dot' ? '<#>' : '<->';
|
|
@@ -2,6 +2,12 @@ export type Dialect = 'mssql' | 'postgres' | 'mysql' | 'sqlite' | 'googlesheets'
|
|
|
2
2
|
|
|
3
3
|
export interface An5AdapterConfig {
|
|
4
4
|
connectionString?: string;
|
|
5
|
+
/**
|
|
6
|
+
* NBase (Neural Vector Database) endpoint, written as a connection string,
|
|
7
|
+
* e.g. `nbase://localhost:1307`. Attach it here when vector search should
|
|
8
|
+
* run in NBase while the rows stay in `connectionString`'s database.
|
|
9
|
+
*/
|
|
10
|
+
nbase?: string;
|
|
5
11
|
engine?: QueryEngine;
|
|
6
12
|
db?: any;
|
|
7
13
|
driver?: any;
|
package/typescript/src/index.ts
CHANGED
|
@@ -7,6 +7,23 @@ export {
|
|
|
7
7
|
setAdapterMetadata,
|
|
8
8
|
} from './an5Adapter';
|
|
9
9
|
export { parseWhere, buildOrderBy, quote } from './base/sql';
|
|
10
|
+
export {
|
|
11
|
+
NBaseVectorClient,
|
|
12
|
+
NBaseError,
|
|
13
|
+
createNBaseVectorClient,
|
|
14
|
+
createNBaseClient,
|
|
15
|
+
parseNBaseConnectionString,
|
|
16
|
+
buildVectorId,
|
|
17
|
+
parseVectorId,
|
|
18
|
+
} from './nbase';
|
|
19
|
+
export type {
|
|
20
|
+
NBaseAdapterConfig,
|
|
21
|
+
NBaseClientConfig,
|
|
22
|
+
NBaseVector,
|
|
23
|
+
NBaseSearchOptions,
|
|
24
|
+
NBaseSearchResult,
|
|
25
|
+
NBaseStats,
|
|
26
|
+
} from './nbase';
|
|
10
27
|
export type {
|
|
11
28
|
An5AdapterConfig,
|
|
12
29
|
AdapterMetadata,
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NBase — Neural Vector Database client.
|
|
3
|
+
*
|
|
4
|
+
* NBase is a vector database with HNSW/LSH/KNN indexing exposed over a REST
|
|
5
|
+
* API. an5 uses it to move similarity search out of the database process: the
|
|
6
|
+
* vectors live in NBase, the rows stay in the relational table, and the search
|
|
7
|
+
* results are hydrated from the table by the id kept in vector metadata.
|
|
8
|
+
*
|
|
9
|
+
* The client only uses the global `fetch`, so it works in Node 18+ and in the
|
|
10
|
+
* browser without pulling in a dependency.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
export interface NBaseVector {
|
|
14
|
+
id: string | number;
|
|
15
|
+
vector: number[];
|
|
16
|
+
metadata?: Record<string, unknown>;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface NBaseSearchOptions {
|
|
20
|
+
/** Number of neighbours to return. */
|
|
21
|
+
k?: number;
|
|
22
|
+
/** Distance metric; NBase accepts `cosine` and `euclidean`. */
|
|
23
|
+
distanceMetric?: 'cosine' | 'euclidean';
|
|
24
|
+
/** `hnsw` uses the graph index, anything else uses the clustered path. */
|
|
25
|
+
method?: 'hnsw' | 'clustered';
|
|
26
|
+
efSearch?: number;
|
|
27
|
+
filters?: Record<string, unknown> | Array<Record<string, unknown>>;
|
|
28
|
+
includeMetadata?: boolean;
|
|
29
|
+
includeVectors?: boolean;
|
|
30
|
+
skipCache?: boolean;
|
|
31
|
+
searchTimeoutMs?: number;
|
|
32
|
+
rerank?: boolean;
|
|
33
|
+
rerankingMethod?: string;
|
|
34
|
+
rerankLambda?: number;
|
|
35
|
+
partitionIds?: string[];
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export interface NBaseSearchResult {
|
|
39
|
+
id: string | number;
|
|
40
|
+
/** Distance from the query vector. Lower is more similar. */
|
|
41
|
+
dist: number;
|
|
42
|
+
metadata?: Record<string, unknown>;
|
|
43
|
+
vector?: number[];
|
|
44
|
+
dimension?: number;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface NBaseClientConfig {
|
|
48
|
+
/** Base URL, e.g. `http://localhost:1307`. */
|
|
49
|
+
url: string;
|
|
50
|
+
/** Optional bearer token, for deployments behind an auth proxy. */
|
|
51
|
+
token?: string;
|
|
52
|
+
/** Per-request timeout in milliseconds. */
|
|
53
|
+
timeoutMs?: number;
|
|
54
|
+
/** Injectable for tests and for runtimes without a global fetch. */
|
|
55
|
+
fetchImpl?: typeof fetch;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface NBaseStats {
|
|
59
|
+
vectorCount?: number;
|
|
60
|
+
metadataCount?: number;
|
|
61
|
+
memoryUsage?: number;
|
|
62
|
+
[key: string]: unknown;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const DEFAULT_TIMEOUT_MS = 30_000;
|
|
66
|
+
|
|
67
|
+
export class NBaseError extends Error {
|
|
68
|
+
constructor(
|
|
69
|
+
message: string,
|
|
70
|
+
readonly status?: number,
|
|
71
|
+
readonly body?: unknown,
|
|
72
|
+
) {
|
|
73
|
+
super(message);
|
|
74
|
+
this.name = 'NBaseError';
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export class NBaseVectorClient {
|
|
79
|
+
private readonly baseUrl: string;
|
|
80
|
+
private readonly fetchImpl: typeof fetch;
|
|
81
|
+
private readonly timeoutMs: number;
|
|
82
|
+
|
|
83
|
+
constructor(private readonly config: NBaseClientConfig) {
|
|
84
|
+
this.baseUrl = String(config.url).replace(/\/+$/, '');
|
|
85
|
+
const impl = config.fetchImpl ?? (globalThis as { fetch?: typeof fetch }).fetch;
|
|
86
|
+
if (!impl) {
|
|
87
|
+
throw new NBaseError('No fetch implementation available. Node 18+ or a fetch polyfill is required.');
|
|
88
|
+
}
|
|
89
|
+
this.fetchImpl = impl;
|
|
90
|
+
this.timeoutMs = config.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
private async request<T>(path: string, init?: RequestInit): Promise<T> {
|
|
94
|
+
const controller = new AbortController();
|
|
95
|
+
const timer = setTimeout(() => controller.abort(), this.timeoutMs);
|
|
96
|
+
try {
|
|
97
|
+
const response = await this.fetchImpl(`${this.baseUrl}${path}`, {
|
|
98
|
+
...init,
|
|
99
|
+
signal: controller.signal,
|
|
100
|
+
headers: {
|
|
101
|
+
'content-type': 'application/json',
|
|
102
|
+
...(this.config.token ? { authorization: `Bearer ${this.config.token}` } : {}),
|
|
103
|
+
...(init?.headers as Record<string, string> | undefined),
|
|
104
|
+
},
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
const text = await response.text();
|
|
108
|
+
let body: unknown;
|
|
109
|
+
try {
|
|
110
|
+
body = text ? JSON.parse(text) : undefined;
|
|
111
|
+
} catch {
|
|
112
|
+
body = text;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
if (!response.ok) {
|
|
116
|
+
const message =
|
|
117
|
+
(body && typeof body === 'object' && 'error' in (body as Record<string, unknown>)
|
|
118
|
+
? String((body as Record<string, unknown>).error)
|
|
119
|
+
: undefined) ?? `HTTP ${response.status}`;
|
|
120
|
+
throw new NBaseError(`NBase request ${path} failed: ${message}`, response.status, body);
|
|
121
|
+
}
|
|
122
|
+
return body as T;
|
|
123
|
+
} catch (err) {
|
|
124
|
+
if (err instanceof NBaseError) throw err;
|
|
125
|
+
if ((err as { name?: string })?.name === 'AbortError') {
|
|
126
|
+
throw new NBaseError(`NBase request ${path} timed out after ${this.timeoutMs}ms`);
|
|
127
|
+
}
|
|
128
|
+
throw new NBaseError(
|
|
129
|
+
`Could not reach NBase at ${this.baseUrl}: ${err instanceof Error ? err.message : String(err)}`,
|
|
130
|
+
);
|
|
131
|
+
} finally {
|
|
132
|
+
clearTimeout(timer);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Liveness probe. NBase mounts its status routes at the server root. */
|
|
137
|
+
async health(): Promise<{ status: string; version?: string }> {
|
|
138
|
+
return this.request<{ status: string; version?: string }>('/health');
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** Vector count, memory use and other database statistics. */
|
|
142
|
+
async stats(): Promise<NBaseStats> {
|
|
143
|
+
return this.request<NBaseStats>('/stats');
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** Adds or replaces vectors. NBase accepts a single vector or a bulk array. */
|
|
147
|
+
async addVectors(vectors: NBaseVector[]): Promise<{ count: number; ids?: Array<string | number> }> {
|
|
148
|
+
if (vectors.length === 0) return { count: 0 };
|
|
149
|
+
const result = await this.request<{ count: number; addedIds?: Array<string | number> }>(
|
|
150
|
+
'/api/vectors',
|
|
151
|
+
{ method: 'POST', body: JSON.stringify({ vectors: vectors.length === 1 ? vectors[0] : vectors }) },
|
|
152
|
+
);
|
|
153
|
+
return { count: result.count ?? vectors.length, ...(result.addedIds ? { ids: result.addedIds } : {}) };
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Nearest-neighbour search. Results are ordered by ascending `dist`. */
|
|
157
|
+
async search(vector: number[], options: NBaseSearchOptions = {}): Promise<NBaseSearchResult[]> {
|
|
158
|
+
const body = await this.request<{ results?: NBaseSearchResult[] }>('/api/search', {
|
|
159
|
+
method: 'POST',
|
|
160
|
+
body: JSON.stringify({
|
|
161
|
+
query: vector,
|
|
162
|
+
k: options.k ?? 10,
|
|
163
|
+
...(options.distanceMetric ? { distanceMetric: options.distanceMetric } : {}),
|
|
164
|
+
...(options.method ? { method: options.method } : {}),
|
|
165
|
+
...(options.efSearch !== undefined ? { efSearch: options.efSearch } : {}),
|
|
166
|
+
...(options.filters ? { filters: options.filters } : {}),
|
|
167
|
+
...(options.includeMetadata !== undefined ? { includeMetadata: options.includeMetadata } : {}),
|
|
168
|
+
...(options.includeVectors !== undefined ? { includeVectors: options.includeVectors } : {}),
|
|
169
|
+
...(options.skipCache !== undefined ? { skipCache: options.skipCache } : {}),
|
|
170
|
+
...(options.searchTimeoutMs !== undefined ? { searchTimeoutMs: options.searchTimeoutMs } : {}),
|
|
171
|
+
...(options.rerank !== undefined ? { rerank: options.rerank } : {}),
|
|
172
|
+
...(options.rerankingMethod ? { rerankingMethod: options.rerankingMethod } : {}),
|
|
173
|
+
...(options.rerankLambda !== undefined ? { rerankLambda: options.rerankLambda } : {}),
|
|
174
|
+
...(options.partitionIds ? { partitionIds: options.partitionIds } : {}),
|
|
175
|
+
}),
|
|
176
|
+
});
|
|
177
|
+
return body.results ?? [];
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/** Reads a single vector. Returns undefined when the id is unknown. */
|
|
181
|
+
async get(id: string | number): Promise<NBaseSearchResult | undefined> {
|
|
182
|
+
try {
|
|
183
|
+
return await this.request<NBaseSearchResult>(`/api/vectors/${encodeURIComponent(String(id))}`);
|
|
184
|
+
} catch (err) {
|
|
185
|
+
if (err instanceof NBaseError && err.status === 404) return undefined;
|
|
186
|
+
throw err;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
async exists(id: string | number): Promise<boolean> {
|
|
191
|
+
const result = await this.request<{ exists?: boolean }>(
|
|
192
|
+
`/api/vectors/${encodeURIComponent(String(id))}/exists`,
|
|
193
|
+
);
|
|
194
|
+
return result.exists === true;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
async delete(id: string | number): Promise<boolean> {
|
|
198
|
+
await this.request(`/api/vectors/${encodeURIComponent(String(id))}`, { method: 'DELETE' });
|
|
199
|
+
return true;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
async updateMetadata(id: string | number, metadata: Record<string, unknown>): Promise<boolean> {
|
|
203
|
+
await this.request(`/api/vectors/${encodeURIComponent(String(id))}/metadata`, {
|
|
204
|
+
method: 'PATCH',
|
|
205
|
+
body: JSON.stringify(metadata),
|
|
206
|
+
});
|
|
207
|
+
return true;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/** Creates an NBase vector client. */
|
|
212
|
+
export function createNBaseVectorClient(config: NBaseClientConfig): NBaseVectorClient {
|
|
213
|
+
return new NBaseVectorClient(config);
|
|
214
|
+
}
|