@an5/adapters 0.2.6 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -2
- package/dist/an5Adapter.d.ts +57 -0
- package/dist/an5Adapter.js +262 -4
- package/dist/base/types.d.ts +6 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +9 -1
- package/dist/nbase/client.d.ts +92 -0
- package/dist/nbase/client.js +147 -0
- package/dist/nbase/index.d.ts +47 -0
- package/dist/nbase/index.js +80 -0
- package/dotnet/An5Adapter.cs +41 -18
- package/dotnet/Base/Core.cs +42 -2
- package/dotnet/Postgres/PostgresEngine.cs +58 -30
- package/dotnet/Sqlite/SqliteEngine.cs +214 -0
- package/package.json +15 -8
- package/python/an5_adapter.py +74 -9
- package/python/base/__init__.py +1 -1
- package/python/base/metadata.py +25 -1
- package/python/base/sql.py +44 -14
- package/python/sqlite/__init__.py +3 -0
- package/python/sqlite/provider.py +83 -0
- package/python/table_client.py +53 -36
- package/rust/Cargo.toml +31 -0
- package/rust/README.md +71 -0
- package/rust/src/adapter.rs +1312 -0
- package/rust/src/base/dialect.rs +172 -0
- package/rust/src/base/metadata.rs +134 -0
- package/rust/src/base/mod.rs +17 -0
- package/rust/src/base/where_builder.rs +405 -0
- package/rust/src/lib.rs +75 -0
- package/typescript/src/an5Adapter.ts +299 -4
- package/typescript/src/base/types.ts +6 -0
- package/typescript/src/index.ts +17 -0
- package/typescript/src/nbase/client.ts +214 -0
- package/typescript/src/nbase/index.ts +104 -0
package/README.md
CHANGED
|
@@ -11,9 +11,9 @@ Standalone runtime database adapter and query engine for AN5 ORM. Provides conne
|
|
|
11
11
|
- **Query Builder & Dialects** — Dialect-aware SQL formatting for MSSQL, PostgreSQL, MySQL, and SQLite
|
|
12
12
|
- **Aggregations & GroupBy** — Standard ORM aggregates (`_count`, `_sum`, `_avg`, `_min`, `_max`) and `groupBy`
|
|
13
13
|
- **Field Math Operators** — Atomic updates with `increment`, `decrement`, `multiply`, `divide`, `set`
|
|
14
|
-
- **Vector Search** — Similarity search using pgvector (PostgreSQL), `VECTOR_DISTANCE` (MSSQL), or in-memory cosine/euclidean/dot similarity
|
|
14
|
+
- **Vector Search** — Similarity search using NBase, pgvector (PostgreSQL), `VECTOR_DISTANCE` (MSSQL), or in-memory cosine/euclidean/dot similarity
|
|
15
15
|
- **Real Transactions** — Interactive and callback transactions (`$transaction(async tx => ...)` and `$begin()/$commit()/$rollback()`)
|
|
16
|
-
- **Cross-Language** — Unified API in TypeScript, Python, .NET (C#), and
|
|
16
|
+
- **Cross-Language** — Unified API in TypeScript, Python, .NET (C#), Golang, and Rust
|
|
17
17
|
- **Google Sheets Database** — Use spreadsheets as a live database with full CRUD and SQL syntax support
|
|
18
18
|
|
|
19
19
|
---
|
|
@@ -44,8 +44,102 @@ dotnet add package An5Adapters
|
|
|
44
44
|
# Included in the npm package under golang/ or via go module
|
|
45
45
|
```
|
|
46
46
|
|
|
47
|
+
### Rust
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
cargo add an5-adapters
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The crate is driver-agnostic — it depends on `sqlx` with `any`, so the app enables the driver it needs. See [crates.io](https://crates.io/crates/an5-adapters) and [docs.rs](https://docs.rs/an5-adapters).
|
|
54
|
+
|
|
47
55
|
---
|
|
48
56
|
|
|
57
|
+
## NBase — Neural Vector Database
|
|
58
|
+
|
|
59
|
+
[NBase](https://github.com/N2FlowJS/nbase) is a partitioned vector database with
|
|
60
|
+
HNSW, LSH and KNN indexing behind a REST API. an5 can run vector search there
|
|
61
|
+
instead of loading every row into memory, which matters once a table holds
|
|
62
|
+
millions of embeddings.
|
|
63
|
+
|
|
64
|
+
The rows stay in your relational table: only the vectors move to NBase, each one
|
|
65
|
+
tagged with its row id, and search results are hydrated from the table.
|
|
66
|
+
|
|
67
|
+
### Configure
|
|
68
|
+
|
|
69
|
+
NBase is configured through the connection string, like every other backend:
|
|
70
|
+
|
|
71
|
+
```ts
|
|
72
|
+
import { createAn5Adapter } from '@an5/adapters';
|
|
73
|
+
|
|
74
|
+
const db = createAn5Adapter({ connectionString: 'nbase://localhost:1307' });
|
|
75
|
+
await db.$connect(); // probes NBase
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Accepted forms:
|
|
79
|
+
|
|
80
|
+
| Form | Result |
|
|
81
|
+
|------|--------|
|
|
82
|
+
| `nbase://localhost:1307` | `http://localhost:1307` |
|
|
83
|
+
| `nbase:localhost:1307` | `http://localhost:1307` |
|
|
84
|
+
| `nbase:http://localhost:1307` | `http://localhost:1307` |
|
|
85
|
+
| `nbase:https://vectors.example.com` | `https://vectors.example.com` |
|
|
86
|
+
| `nbase://localhost:1307?token=t&timeoutMs=500&method=hnsw` | options applied |
|
|
87
|
+
|
|
88
|
+
`?token`, `?timeoutMs`, `?method`, `?idField` and `?modelField` are read from the
|
|
89
|
+
query string, and a path is kept as a base path for deployments behind a
|
|
90
|
+
reverse proxy.
|
|
91
|
+
|
|
92
|
+
This adapter is **vector-only**: it has no relational database, so a search
|
|
93
|
+
returns the NBase hits — id, metadata and `distance` — instead of table rows.
|
|
94
|
+
|
|
95
|
+
### Search
|
|
96
|
+
|
|
97
|
+
With an `nbase://` connection string, search returns the hits themselves:
|
|
98
|
+
|
|
99
|
+
```ts
|
|
100
|
+
const hits = await db.table('Article').vectorSearch({ vector: queryEmbedding, take: 5 });
|
|
101
|
+
// hits[0] = { id: 'Article:42', an5Id: '42', an5Model: 'Article', distance: 0.11 }
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Search rows from your own table
|
|
105
|
+
|
|
106
|
+
To get real rows back, keep your database connection string and point it at the
|
|
107
|
+
same NBase instance. NBase is still configured with a connection string, never a
|
|
108
|
+
params object:
|
|
109
|
+
|
|
110
|
+
```ts
|
|
111
|
+
const db = createAn5Adapter({
|
|
112
|
+
connectionString: 'sqlserver://localhost:1433;database=mydb',
|
|
113
|
+
nbase: 'nbase://localhost:1307',
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
await db.table('Article').indexVectorsInNBase({ vectorField: 'embedding' });
|
|
117
|
+
// → { indexed: 1200 }
|
|
118
|
+
|
|
119
|
+
const hits = await db.table('Article').vectorSearch({ vector: queryEmbedding, take: 5 });
|
|
120
|
+
// hits[0] = { id, title, ..., distance }
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Vectors are tagged with their row id, so a hit is hydrated from the table by
|
|
124
|
+
that id and the result keeps the `{ ...row, distance }` shape the other engines
|
|
125
|
+
return.
|
|
126
|
+
|
|
127
|
+
### Standalone client
|
|
128
|
+
|
|
129
|
+
```ts
|
|
130
|
+
import { createNBaseVectorClient } from '@an5/adapters/nbase';
|
|
131
|
+
|
|
132
|
+
const nbase = createNBaseVectorClient({ url: 'http://localhost:1307' });
|
|
133
|
+
await nbase.health();
|
|
134
|
+
await nbase.addVectors([{ id: 'a:1', vector: [0.1, 0.2], metadata: { an5Id: '1' } }]);
|
|
135
|
+
const { results } = await nbase.search([0.1, 0.2], { k: 5, distanceMetric: 'cosine' });
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
If NBase is unreachable, `vectorSearch` logs a warning and falls back to the
|
|
139
|
+
database's native vector support, then to in-memory similarity, so an outage
|
|
140
|
+
degrades performance rather than breaking queries.
|
|
141
|
+
|
|
142
|
+
|
|
49
143
|
## Usage
|
|
50
144
|
|
|
51
145
|
### TypeScript
|
package/dist/an5Adapter.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { An5SheetsAdapter, An5SheetsAdapterConfig, SheetsTableClient } from './googlesheets';
|
|
2
2
|
import type { An5AdapterConfig, Dialect, TransactionHandle } from './base/types';
|
|
3
|
+
import { type NBaseAdapterConfig, type NBaseVectorClient } from './nbase';
|
|
3
4
|
export type { An5AdapterConfig, Dialect, QueryEngine, TransactionHandle } from './base/types';
|
|
4
5
|
export { setAdapterMetadata, createAdapterProxy, type AdapterMetadata } from './base/metadata';
|
|
5
6
|
export type { TypedAn5Adapter, AdapterAPI } from './typed';
|
|
@@ -13,6 +14,23 @@ export declare class An5Adapter {
|
|
|
13
14
|
private _engineType;
|
|
14
15
|
private _engineConfig;
|
|
15
16
|
sheetsAdapter: An5SheetsAdapter | null;
|
|
17
|
+
private _nbase;
|
|
18
|
+
private _nbaseConfig;
|
|
19
|
+
private _nbaseOnly;
|
|
20
|
+
/**
|
|
21
|
+
* Returns the NBase client for this adapter, if the project configured one.
|
|
22
|
+
* The connection string may also point straight at NBase, which keeps
|
|
23
|
+
* `DATABASE_URL=nbase://host:port` working on its own.
|
|
24
|
+
*/
|
|
25
|
+
nbaseClient(): {
|
|
26
|
+
client: NBaseVectorClient;
|
|
27
|
+
config: NBaseAdapterConfig;
|
|
28
|
+
} | null;
|
|
29
|
+
/**
|
|
30
|
+
* True when the adapter only knows about NBase, so there is no relational
|
|
31
|
+
* database to read rows from.
|
|
32
|
+
*/
|
|
33
|
+
get nbaseOnly(): boolean;
|
|
16
34
|
get dialect(): Dialect;
|
|
17
35
|
constructor(adapterConfig: An5AdapterConfig | An5SheetsAdapterConfig);
|
|
18
36
|
private requireEngine;
|
|
@@ -25,6 +43,15 @@ export declare class An5Adapter {
|
|
|
25
43
|
$executeRaw(query: string, ...values: any[]): Promise<number>;
|
|
26
44
|
$executeRawUnsafe(query: string, ...values: any[]): Promise<number>;
|
|
27
45
|
$connect(): Promise<void>;
|
|
46
|
+
/**
|
|
47
|
+
* Health check for the configured NBase instance, or the database when the
|
|
48
|
+
* adapter has none.
|
|
49
|
+
*/
|
|
50
|
+
$nbaseHealth(): Promise<{
|
|
51
|
+
connected: boolean;
|
|
52
|
+
endpoint?: string;
|
|
53
|
+
error?: string;
|
|
54
|
+
}>;
|
|
28
55
|
$disconnect(): Promise<void>;
|
|
29
56
|
/** Real transaction with BEGIN / COMMIT / ROLLBACK */
|
|
30
57
|
$transaction<R>(fn: (tx: An5AdapterTx) => Promise<R>, options?: {
|
|
@@ -179,6 +206,36 @@ export declare class AdapterTableClient<T = any> {
|
|
|
179
206
|
}): Promise<T>;
|
|
180
207
|
aggregate(args: any): Promise<any>;
|
|
181
208
|
groupBy(args: any): Promise<any[]>;
|
|
209
|
+
/**
|
|
210
|
+
* Runs similarity search in NBase and hydrates the matching rows from the
|
|
211
|
+
* relational table, keeping the `{ ...row, distance }` shape the other
|
|
212
|
+
* engines return.
|
|
213
|
+
*/
|
|
214
|
+
private searchViaNBase;
|
|
215
|
+
/**
|
|
216
|
+
* Vector search for an adapter configured with only an NBase connection
|
|
217
|
+
* string. The hit id and metadata are the result, since there is no
|
|
218
|
+
* relational row to hydrate.
|
|
219
|
+
*/
|
|
220
|
+
private searchViaNBaseOnly;
|
|
221
|
+
/** The primary key column registered for this model, falling back to `id`. */
|
|
222
|
+
private primaryKeyColumn;
|
|
223
|
+
/** Scalar column names known for this model from the generated metadata. */
|
|
224
|
+
private modelFields;
|
|
225
|
+
/**
|
|
226
|
+
* Pushes a table's embedding column into NBase so it can be searched there.
|
|
227
|
+
*
|
|
228
|
+
* Each vector keeps the row id in its metadata, which is what lets
|
|
229
|
+
* `vectorSearch` return real rows instead of bare vector ids.
|
|
230
|
+
*/
|
|
231
|
+
indexVectorsInNBase(args?: {
|
|
232
|
+
vectorField?: string;
|
|
233
|
+
where?: any;
|
|
234
|
+
take?: number;
|
|
235
|
+
batchSize?: number;
|
|
236
|
+
}): Promise<{
|
|
237
|
+
indexed: number;
|
|
238
|
+
}>;
|
|
182
239
|
vectorSearch(args: {
|
|
183
240
|
vector: number[];
|
|
184
241
|
take?: number;
|
package/dist/an5Adapter.js
CHANGED
|
@@ -7,6 +7,15 @@ exports.executorFromAdapter = executorFromAdapter;
|
|
|
7
7
|
const uuid_1 = require("./base/uuid");
|
|
8
8
|
const googlesheets_1 = require("./googlesheets");
|
|
9
9
|
const parseConnectionString_1 = require("./googlesheets/parseConnectionString");
|
|
10
|
+
/**
|
|
11
|
+
* File extensions that mean SQLite when a connection string has no scheme.
|
|
12
|
+
*
|
|
13
|
+
* `.sqlite3` is here as well as in `@an5/orm`'s `detectProvider`, because the ORM
|
|
14
|
+
* validates field types and writes the DDL from the provider this file picks: if the
|
|
15
|
+
* two lists differed, a `.sqlite3` file would be pushed as SQL Server DDL.
|
|
16
|
+
*/
|
|
17
|
+
const SQLITE_FILE_SUFFIXES = ['.sqlite', '.sqlite3', '.db'];
|
|
18
|
+
const nbase_1 = require("./nbase");
|
|
10
19
|
const sql_1 = require("./base/sql");
|
|
11
20
|
const metadata_1 = require("./base/metadata");
|
|
12
21
|
var metadata_2 = require("./base/metadata");
|
|
@@ -171,6 +180,29 @@ async function resolveIncludes(modelName, rows, include, adapter) {
|
|
|
171
180
|
}
|
|
172
181
|
// ─── An5Adapter ────────────────────────────────────────────────────────────────────
|
|
173
182
|
class An5Adapter {
|
|
183
|
+
/**
|
|
184
|
+
* Returns the NBase client for this adapter, if the project configured one.
|
|
185
|
+
* The connection string may also point straight at NBase, which keeps
|
|
186
|
+
* `DATABASE_URL=nbase://host:port` working on its own.
|
|
187
|
+
*/
|
|
188
|
+
nbaseClient() {
|
|
189
|
+
if (this._nbase && this._nbaseConfig)
|
|
190
|
+
return { client: this._nbase, config: this._nbaseConfig };
|
|
191
|
+
if (!this._nbaseConfig)
|
|
192
|
+
return null;
|
|
193
|
+
const config = this._nbaseConfig;
|
|
194
|
+
const client = (0, nbase_1.createNBaseVectorClient)(config);
|
|
195
|
+
this._nbase = client;
|
|
196
|
+
this._nbaseConfig = config;
|
|
197
|
+
return { client, config };
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* True when the adapter only knows about NBase, so there is no relational
|
|
201
|
+
* database to read rows from.
|
|
202
|
+
*/
|
|
203
|
+
get nbaseOnly() {
|
|
204
|
+
return this._nbaseOnly === true;
|
|
205
|
+
}
|
|
174
206
|
get dialect() {
|
|
175
207
|
if (this.sheetsAdapter)
|
|
176
208
|
return 'googlesheets';
|
|
@@ -189,12 +221,32 @@ class An5Adapter {
|
|
|
189
221
|
this._engineType = null;
|
|
190
222
|
this._engineConfig = null;
|
|
191
223
|
this.sheetsAdapter = null;
|
|
224
|
+
this._nbase = null;
|
|
225
|
+
this._nbaseConfig = null;
|
|
226
|
+
this._nbaseOnly = false;
|
|
192
227
|
this._listeners = {};
|
|
193
228
|
this._middlewares = [];
|
|
194
229
|
if (isSheetsConfig(adapterConfig)) {
|
|
195
230
|
this.sheetsAdapter = new googlesheets_1.An5SheetsAdapter(adapterConfig);
|
|
196
231
|
return (0, metadata_1.createAdapterProxy)(this, (name) => this.table(name));
|
|
197
232
|
}
|
|
233
|
+
// Keep the config even when the caller supplies its own engine: the NBase
|
|
234
|
+
// settings and anything else on the config are read from it later.
|
|
235
|
+
this._engineConfig = adapterConfig;
|
|
236
|
+
// `nbase://host:port` as the connection string configures the vector store
|
|
237
|
+
// on its own: there is no relational database, so the adapter is
|
|
238
|
+
// vector-only. The same string passed as `nbase` keeps the database from
|
|
239
|
+
// `connectionString` as the source of rows.
|
|
240
|
+
const rawConnectionString = String(this._engineConfig.connectionString ?? '').trim();
|
|
241
|
+
if ((0, nbase_1.isNBaseConnectionString)(rawConnectionString)) {
|
|
242
|
+
this._nbaseConfig = (0, nbase_1.parseNBaseConnectionString)(rawConnectionString);
|
|
243
|
+
this._nbaseOnly = true;
|
|
244
|
+
return (0, metadata_1.createAdapterProxy)(this, (name) => this.table(name));
|
|
245
|
+
}
|
|
246
|
+
const attached = this._engineConfig.nbase;
|
|
247
|
+
if (attached && (0, nbase_1.isNBaseConnectionString)(attached)) {
|
|
248
|
+
this._nbaseConfig = (0, nbase_1.parseNBaseConnectionString)(attached);
|
|
249
|
+
}
|
|
198
250
|
if (adapterConfig.engine) {
|
|
199
251
|
this._engine = adapterConfig.engine;
|
|
200
252
|
this._engineType = this._engine?.dialect || 'sqlite';
|
|
@@ -206,19 +258,25 @@ class An5Adapter {
|
|
|
206
258
|
this._engineType = 'sqlite';
|
|
207
259
|
return (0, metadata_1.createAdapterProxy)(this, (name) => this.table(name));
|
|
208
260
|
}
|
|
261
|
+
// Lower-cased for the dialect test only; the connection string itself keeps its
|
|
262
|
+
// case for the driver. A URI scheme is case-insensitive, and `@an5/orm` reads the
|
|
263
|
+
// provider the same way to validate field types and write DDL — if the two
|
|
264
|
+
// disagreed, a schema would be checked against one database and run on another.
|
|
209
265
|
const cs = (adapterConfig.connectionString || '').trim();
|
|
210
|
-
|
|
266
|
+
const forDialect = cs.toLowerCase();
|
|
267
|
+
if (forDialect.startsWith('googlesheets://')) {
|
|
211
268
|
this.sheetsAdapter = new googlesheets_1.An5SheetsAdapter((0, parseConnectionString_1.parseSheetsConnectionString)(cs));
|
|
212
269
|
return (0, metadata_1.createAdapterProxy)(this, (name) => this.table(name));
|
|
213
270
|
}
|
|
214
271
|
this._engineConfig = adapterConfig;
|
|
215
|
-
if (
|
|
272
|
+
if (forDialect.startsWith('postgres://') || forDialect.startsWith('postgresql://')) {
|
|
216
273
|
this._engineType = 'postgres';
|
|
217
274
|
}
|
|
218
|
-
else if (
|
|
275
|
+
else if (forDialect.startsWith('mysql://') || forDialect.startsWith('mariadb://')) {
|
|
219
276
|
this._engineType = 'mysql';
|
|
220
277
|
}
|
|
221
|
-
else if (
|
|
278
|
+
else if (forDialect.startsWith('sqlite://')
|
|
279
|
+
|| SQLITE_FILE_SUFFIXES.some((suffix) => forDialect.endsWith(suffix))) {
|
|
222
280
|
this._engineType = 'sqlite';
|
|
223
281
|
}
|
|
224
282
|
else {
|
|
@@ -310,8 +368,36 @@ class An5Adapter {
|
|
|
310
368
|
async $connect() {
|
|
311
369
|
if (this.sheetsAdapter)
|
|
312
370
|
return this.sheetsAdapter.$connect();
|
|
371
|
+
if (this._nbaseOnly) {
|
|
372
|
+
// NBase is stateless over HTTP, so there is nothing to open. Report the
|
|
373
|
+
// endpoint so a misconfiguration is obvious.
|
|
374
|
+
const config = this.nbaseClient()?.config;
|
|
375
|
+
if (config)
|
|
376
|
+
await this.nbaseClient().client.health();
|
|
377
|
+
return;
|
|
378
|
+
}
|
|
313
379
|
await (await this.requireEngine()).connect();
|
|
314
380
|
}
|
|
381
|
+
/**
|
|
382
|
+
* Health check for the configured NBase instance, or the database when the
|
|
383
|
+
* adapter has none.
|
|
384
|
+
*/
|
|
385
|
+
async $nbaseHealth() {
|
|
386
|
+
const nbase = this.nbaseClient();
|
|
387
|
+
if (!nbase)
|
|
388
|
+
return { connected: false, error: 'No NBase instance configured.' };
|
|
389
|
+
try {
|
|
390
|
+
const health = await nbase.client.health();
|
|
391
|
+
return { connected: health.status === 'ok', endpoint: nbase.config.url };
|
|
392
|
+
}
|
|
393
|
+
catch (err) {
|
|
394
|
+
return {
|
|
395
|
+
connected: false,
|
|
396
|
+
endpoint: nbase.config.url,
|
|
397
|
+
error: err instanceof Error ? err.message : String(err),
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
}
|
|
315
401
|
async $disconnect() {
|
|
316
402
|
if (this.sheetsAdapter)
|
|
317
403
|
return this.sheetsAdapter.$disconnect();
|
|
@@ -1001,6 +1087,158 @@ class AdapterTableClient {
|
|
|
1001
1087
|
}
|
|
1002
1088
|
return this.doExec(query, params);
|
|
1003
1089
|
}
|
|
1090
|
+
/**
|
|
1091
|
+
* Runs similarity search in NBase and hydrates the matching rows from the
|
|
1092
|
+
* relational table, keeping the `{ ...row, distance }` shape the other
|
|
1093
|
+
* engines return.
|
|
1094
|
+
*/
|
|
1095
|
+
async searchViaNBase(nbase, args, metric, take) {
|
|
1096
|
+
const { client, config } = nbase;
|
|
1097
|
+
const idField = config.idField ?? 'an5Id';
|
|
1098
|
+
const modelField = config.modelField ?? 'an5Model';
|
|
1099
|
+
const results = await client.search(args.vector, {
|
|
1100
|
+
k: take,
|
|
1101
|
+
// NBase implements cosine and euclidean; a dot-product request falls back
|
|
1102
|
+
// to cosine, which still orders the results sensibly.
|
|
1103
|
+
distanceMetric: metric === 'euclidean' ? 'euclidean' : 'cosine',
|
|
1104
|
+
includeMetadata: true,
|
|
1105
|
+
...(config.method ? { method: config.method } : {}),
|
|
1106
|
+
});
|
|
1107
|
+
// Map each hit back to its row id. Ids are written as `Model:id`, and the
|
|
1108
|
+
// metadata key is honoured for projects that index vectors themselves.
|
|
1109
|
+
const ids = [];
|
|
1110
|
+
const distances = new Map();
|
|
1111
|
+
for (const hit of results) {
|
|
1112
|
+
const fromMetadata = hit.metadata?.[idField];
|
|
1113
|
+
const rawId = fromMetadata !== undefined && fromMetadata !== null
|
|
1114
|
+
? String(fromMetadata)
|
|
1115
|
+
: (0, nbase_1.parseVectorId)(hit.id).id;
|
|
1116
|
+
if (!rawId)
|
|
1117
|
+
continue;
|
|
1118
|
+
const model = hit.metadata?.[modelField] ?? (0, nbase_1.parseVectorId)(hit.id).model;
|
|
1119
|
+
ids.push(model === this.modelName ? rawId : `${model}:${rawId}`);
|
|
1120
|
+
distances.set(ids[ids.length - 1], hit.dist);
|
|
1121
|
+
}
|
|
1122
|
+
if (ids.length === 0)
|
|
1123
|
+
return [];
|
|
1124
|
+
const pk = this.primaryKeyColumn();
|
|
1125
|
+
const inList = ids.map((_, i) => `@nbase_id_${i}`).join(', ');
|
|
1126
|
+
const idParams = {};
|
|
1127
|
+
ids.forEach((id, i) => { idParams[`nbase_id_${i}`] = id; });
|
|
1128
|
+
idParams.take = take;
|
|
1129
|
+
const selectColumns = this.modelFields().map((f) => (0, sql_1.quote)(f, this.dialect)).join(', ');
|
|
1130
|
+
const columns = selectColumns ? `${selectColumns}, ${(0, sql_1.quote)(pk, this.dialect)} AS __an5_nbase_key` : `${(0, sql_1.quote)(pk, this.dialect)} AS __an5_nbase_key`;
|
|
1131
|
+
const query = `SELECT ${columns} FROM ${this.tableName} WHERE ${(0, sql_1.quote)(pk, this.dialect)} IN (${inList})`;
|
|
1132
|
+
const rows = await this.adapter.exec(query, idParams);
|
|
1133
|
+
const byKey = new Map();
|
|
1134
|
+
for (const row of rows ?? []) {
|
|
1135
|
+
byKey.set(String(row.__an5_nbase_key), row);
|
|
1136
|
+
}
|
|
1137
|
+
const hydrated = [];
|
|
1138
|
+
for (const id of ids) {
|
|
1139
|
+
const row = byKey.get(id);
|
|
1140
|
+
if (!row)
|
|
1141
|
+
continue;
|
|
1142
|
+
const { __an5_nbase_key, ...rest } = row;
|
|
1143
|
+
hydrated.push({ ...rest, distance: distances.get(id) ?? 0 });
|
|
1144
|
+
if (hydrated.length >= take)
|
|
1145
|
+
break;
|
|
1146
|
+
}
|
|
1147
|
+
return hydrated;
|
|
1148
|
+
}
|
|
1149
|
+
/**
|
|
1150
|
+
* Vector search for an adapter configured with only an NBase connection
|
|
1151
|
+
* string. The hit id and metadata are the result, since there is no
|
|
1152
|
+
* relational row to hydrate.
|
|
1153
|
+
*/
|
|
1154
|
+
async searchViaNBaseOnly(nbase, args, metric, take) {
|
|
1155
|
+
const { client, config } = nbase;
|
|
1156
|
+
const idField = config.idField ?? 'an5Id';
|
|
1157
|
+
const results = await client.search(args.vector, {
|
|
1158
|
+
k: take,
|
|
1159
|
+
distanceMetric: metric === 'euclidean' ? 'euclidean' : 'cosine',
|
|
1160
|
+
includeMetadata: true,
|
|
1161
|
+
...(config.method ? { method: config.method } : {}),
|
|
1162
|
+
});
|
|
1163
|
+
return results.map((hit) => {
|
|
1164
|
+
const metadata = { ...(hit.metadata ?? {}) };
|
|
1165
|
+
if (metadata[idField] === undefined) {
|
|
1166
|
+
const parsed = (0, nbase_1.parseVectorId)(hit.id);
|
|
1167
|
+
if (parsed.id)
|
|
1168
|
+
metadata[idField] = parsed.id;
|
|
1169
|
+
if (parsed.model)
|
|
1170
|
+
metadata[config.modelField ?? 'an5Model'] = parsed.model;
|
|
1171
|
+
}
|
|
1172
|
+
return { id: hit.id, ...metadata, distance: hit.dist };
|
|
1173
|
+
});
|
|
1174
|
+
}
|
|
1175
|
+
/** The primary key column registered for this model, falling back to `id`. */
|
|
1176
|
+
primaryKeyColumn() {
|
|
1177
|
+
const fields = this.modelFields();
|
|
1178
|
+
const marked = fields.find((f) => f === 'id');
|
|
1179
|
+
return marked ?? fields[0] ?? 'id';
|
|
1180
|
+
}
|
|
1181
|
+
/** Scalar column names known for this model from the generated metadata. */
|
|
1182
|
+
modelFields() {
|
|
1183
|
+
const fields = (0, metadata_1.getFieldsForModel)(this.modelName);
|
|
1184
|
+
if (fields && typeof fields === 'object' && !Array.isArray(fields)) {
|
|
1185
|
+
return Object.keys(fields);
|
|
1186
|
+
}
|
|
1187
|
+
return [];
|
|
1188
|
+
}
|
|
1189
|
+
/**
|
|
1190
|
+
* Pushes a table's embedding column into NBase so it can be searched there.
|
|
1191
|
+
*
|
|
1192
|
+
* Each vector keeps the row id in its metadata, which is what lets
|
|
1193
|
+
* `vectorSearch` return real rows instead of bare vector ids.
|
|
1194
|
+
*/
|
|
1195
|
+
async indexVectorsInNBase(args = {}) {
|
|
1196
|
+
const nbase = this.adapter.nbaseClient?.() ?? null;
|
|
1197
|
+
if (!nbase) {
|
|
1198
|
+
throw new Error('No NBase instance configured. Use a nbase:// connection string, e.g. createAn5Adapter({ connectionString: "nbase://localhost:1307" }).');
|
|
1199
|
+
}
|
|
1200
|
+
if (this.adapter.nbaseOnly) {
|
|
1201
|
+
throw new Error('indexVectorsInNBase reads the vectors from a table, but this adapter was created with an nbase:// connection string and has no database. ' +
|
|
1202
|
+
'Create the adapter with your database connection string to index, or add the vectors with the NBase client directly.');
|
|
1203
|
+
}
|
|
1204
|
+
const vectorField = args.vectorField || 'embedding';
|
|
1205
|
+
const batchSize = args.batchSize ?? 200;
|
|
1206
|
+
const pk = this.primaryKeyColumn();
|
|
1207
|
+
const rows = await this.findMany({ ...(args.where ? { where: args.where } : {}), ...(args.take ? { take: args.take } : {}) });
|
|
1208
|
+
let indexed = 0;
|
|
1209
|
+
for (let i = 0; i < rows.length; i += batchSize) {
|
|
1210
|
+
const batch = rows.slice(i, i + batchSize);
|
|
1211
|
+
const vectors = batch
|
|
1212
|
+
.map((row) => {
|
|
1213
|
+
const raw = row[vectorField];
|
|
1214
|
+
if (raw === undefined || raw === null)
|
|
1215
|
+
return null;
|
|
1216
|
+
let vector;
|
|
1217
|
+
try {
|
|
1218
|
+
vector = typeof raw === 'string' ? JSON.parse(raw) : raw;
|
|
1219
|
+
}
|
|
1220
|
+
catch {
|
|
1221
|
+
return null;
|
|
1222
|
+
}
|
|
1223
|
+
if (!Array.isArray(vector) || vector.length === 0)
|
|
1224
|
+
return null;
|
|
1225
|
+
return {
|
|
1226
|
+
id: (0, nbase_1.buildVectorId)(this.modelName, String(row[pk])),
|
|
1227
|
+
vector,
|
|
1228
|
+
metadata: {
|
|
1229
|
+
[nbase.config.idField ?? 'an5Id']: String(row[pk]),
|
|
1230
|
+
[nbase.config.modelField ?? 'an5Model']: this.modelName,
|
|
1231
|
+
},
|
|
1232
|
+
};
|
|
1233
|
+
})
|
|
1234
|
+
.filter((v) => v !== null);
|
|
1235
|
+
if (vectors.length === 0)
|
|
1236
|
+
continue;
|
|
1237
|
+
const result = await nbase.client.addVectors(vectors);
|
|
1238
|
+
indexed += result.count;
|
|
1239
|
+
}
|
|
1240
|
+
return { indexed };
|
|
1241
|
+
}
|
|
1004
1242
|
async vectorSearch(args) {
|
|
1005
1243
|
const vectorField = args.vectorField || 'embedding';
|
|
1006
1244
|
const METRICS = ['cosine', 'euclidean', 'dot'];
|
|
@@ -1012,6 +1250,26 @@ class AdapterTableClient {
|
|
|
1012
1250
|
const vectorJson = JSON.stringify(args.vector);
|
|
1013
1251
|
const col = (0, sql_1.quote)(vectorField, this.dialect);
|
|
1014
1252
|
const params = { query_vector: vectorJson };
|
|
1253
|
+
// 0. NBase, when the project configured one: the vectors live in the vector
|
|
1254
|
+
// database, so the search never loads the table into memory.
|
|
1255
|
+
const nbase = this.adapter.nbaseClient?.() ?? null;
|
|
1256
|
+
if (nbase) {
|
|
1257
|
+
try {
|
|
1258
|
+
// With an NBase-only connection string there is no table to read rows
|
|
1259
|
+
// from, so the hits are returned as the vector metadata itself.
|
|
1260
|
+
if (this.adapter.nbaseOnly) {
|
|
1261
|
+
return (await this.searchViaNBaseOnly(nbase, args, metric, take));
|
|
1262
|
+
}
|
|
1263
|
+
return await this.searchViaNBase(nbase, args, metric, take);
|
|
1264
|
+
}
|
|
1265
|
+
catch (err) {
|
|
1266
|
+
if (!(err instanceof nbase_1.NBaseError))
|
|
1267
|
+
throw err;
|
|
1268
|
+
// A NBase outage must not take vector search down with it; fall through
|
|
1269
|
+
// to the database instead of failing the query.
|
|
1270
|
+
console.warn(`[an5] NBase vector search unavailable (${err.message}); falling back to ${this.dialect}.`);
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1015
1273
|
// 1. Try native dialect execution
|
|
1016
1274
|
if (this.dialect === 'postgres') {
|
|
1017
1275
|
const op = metric === 'cosine' ? '<=>' : metric === 'dot' ? '<#>' : '<->';
|
package/dist/base/types.d.ts
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
export type Dialect = 'mssql' | 'postgres' | 'mysql' | 'sqlite' | 'googlesheets';
|
|
2
2
|
export interface An5AdapterConfig {
|
|
3
3
|
connectionString?: string;
|
|
4
|
+
/**
|
|
5
|
+
* NBase (Neural Vector Database) endpoint, written as a connection string,
|
|
6
|
+
* e.g. `nbase://localhost:1307`. Attach it here when vector search should
|
|
7
|
+
* run in NBase while the rows stay in `connectionString`'s database.
|
|
8
|
+
*/
|
|
9
|
+
nbase?: string;
|
|
4
10
|
engine?: QueryEngine;
|
|
5
11
|
db?: any;
|
|
6
12
|
driver?: any;
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
export { An5Adapter, AdapterTableClient, createAn5Adapter, createAdapterProxy, executorFromAdapter, setAdapterMetadata, } from './an5Adapter';
|
|
2
2
|
export { parseWhere, buildOrderBy, quote } from './base/sql';
|
|
3
|
+
export { NBaseVectorClient, NBaseError, createNBaseVectorClient, createNBaseClient, parseNBaseConnectionString, buildVectorId, parseVectorId, } from './nbase';
|
|
4
|
+
export type { NBaseAdapterConfig, NBaseClientConfig, NBaseVector, NBaseSearchOptions, NBaseSearchResult, NBaseStats, } from './nbase';
|
|
3
5
|
export type { An5AdapterConfig, AdapterMetadata, Dialect, } from './an5Adapter';
|
|
4
6
|
export type { TypedAn5Adapter, AdapterAPI } from './typed';
|
|
5
7
|
export { getLlmConfig, setLlmConfig, getEmbeddingConfig, setEmbeddingConfig, resetAdapter, } from './config';
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.parseSheetsConnectionString = exports.createAn5SheetsAdapter = exports.SheetsTableClient = exports.An5SheetsAdapter = exports.resetAdapter = exports.setEmbeddingConfig = exports.getEmbeddingConfig = exports.setLlmConfig = exports.getLlmConfig = exports.quote = exports.buildOrderBy = exports.parseWhere = exports.setAdapterMetadata = exports.executorFromAdapter = exports.createAdapterProxy = exports.createAn5Adapter = exports.AdapterTableClient = exports.An5Adapter = void 0;
|
|
3
|
+
exports.parseSheetsConnectionString = exports.createAn5SheetsAdapter = exports.SheetsTableClient = exports.An5SheetsAdapter = exports.resetAdapter = exports.setEmbeddingConfig = exports.getEmbeddingConfig = exports.setLlmConfig = exports.getLlmConfig = exports.parseVectorId = exports.buildVectorId = exports.parseNBaseConnectionString = exports.createNBaseClient = exports.createNBaseVectorClient = exports.NBaseError = exports.NBaseVectorClient = exports.quote = exports.buildOrderBy = exports.parseWhere = exports.setAdapterMetadata = exports.executorFromAdapter = exports.createAdapterProxy = exports.createAn5Adapter = exports.AdapterTableClient = exports.An5Adapter = void 0;
|
|
4
4
|
var an5Adapter_1 = require("./an5Adapter");
|
|
5
5
|
Object.defineProperty(exports, "An5Adapter", { enumerable: true, get: function () { return an5Adapter_1.An5Adapter; } });
|
|
6
6
|
Object.defineProperty(exports, "AdapterTableClient", { enumerable: true, get: function () { return an5Adapter_1.AdapterTableClient; } });
|
|
@@ -12,6 +12,14 @@ var sql_1 = require("./base/sql");
|
|
|
12
12
|
Object.defineProperty(exports, "parseWhere", { enumerable: true, get: function () { return sql_1.parseWhere; } });
|
|
13
13
|
Object.defineProperty(exports, "buildOrderBy", { enumerable: true, get: function () { return sql_1.buildOrderBy; } });
|
|
14
14
|
Object.defineProperty(exports, "quote", { enumerable: true, get: function () { return sql_1.quote; } });
|
|
15
|
+
var nbase_1 = require("./nbase");
|
|
16
|
+
Object.defineProperty(exports, "NBaseVectorClient", { enumerable: true, get: function () { return nbase_1.NBaseVectorClient; } });
|
|
17
|
+
Object.defineProperty(exports, "NBaseError", { enumerable: true, get: function () { return nbase_1.NBaseError; } });
|
|
18
|
+
Object.defineProperty(exports, "createNBaseVectorClient", { enumerable: true, get: function () { return nbase_1.createNBaseVectorClient; } });
|
|
19
|
+
Object.defineProperty(exports, "createNBaseClient", { enumerable: true, get: function () { return nbase_1.createNBaseClient; } });
|
|
20
|
+
Object.defineProperty(exports, "parseNBaseConnectionString", { enumerable: true, get: function () { return nbase_1.parseNBaseConnectionString; } });
|
|
21
|
+
Object.defineProperty(exports, "buildVectorId", { enumerable: true, get: function () { return nbase_1.buildVectorId; } });
|
|
22
|
+
Object.defineProperty(exports, "parseVectorId", { enumerable: true, get: function () { return nbase_1.parseVectorId; } });
|
|
15
23
|
var config_1 = require("./config");
|
|
16
24
|
Object.defineProperty(exports, "getLlmConfig", { enumerable: true, get: function () { return config_1.getLlmConfig; } });
|
|
17
25
|
Object.defineProperty(exports, "setLlmConfig", { enumerable: true, get: function () { return config_1.setLlmConfig; } });
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NBase — Neural Vector Database client.
|
|
3
|
+
*
|
|
4
|
+
* NBase is a vector database with HNSW/LSH/KNN indexing exposed over a REST
|
|
5
|
+
* API. an5 uses it to move similarity search out of the database process: the
|
|
6
|
+
* vectors live in NBase, the rows stay in the relational table, and the search
|
|
7
|
+
* results are hydrated from the table by the id kept in vector metadata.
|
|
8
|
+
*
|
|
9
|
+
* The client only uses the global `fetch`, so it works in Node 18+ and in the
|
|
10
|
+
* browser without pulling in a dependency.
|
|
11
|
+
*/
|
|
12
|
+
export interface NBaseVector {
|
|
13
|
+
id: string | number;
|
|
14
|
+
vector: number[];
|
|
15
|
+
metadata?: Record<string, unknown>;
|
|
16
|
+
}
|
|
17
|
+
export interface NBaseSearchOptions {
|
|
18
|
+
/** Number of neighbours to return. */
|
|
19
|
+
k?: number;
|
|
20
|
+
/** Distance metric; NBase accepts `cosine` and `euclidean`. */
|
|
21
|
+
distanceMetric?: 'cosine' | 'euclidean';
|
|
22
|
+
/** `hnsw` uses the graph index, anything else uses the clustered path. */
|
|
23
|
+
method?: 'hnsw' | 'clustered';
|
|
24
|
+
efSearch?: number;
|
|
25
|
+
filters?: Record<string, unknown> | Array<Record<string, unknown>>;
|
|
26
|
+
includeMetadata?: boolean;
|
|
27
|
+
includeVectors?: boolean;
|
|
28
|
+
skipCache?: boolean;
|
|
29
|
+
searchTimeoutMs?: number;
|
|
30
|
+
rerank?: boolean;
|
|
31
|
+
rerankingMethod?: string;
|
|
32
|
+
rerankLambda?: number;
|
|
33
|
+
partitionIds?: string[];
|
|
34
|
+
}
|
|
35
|
+
export interface NBaseSearchResult {
|
|
36
|
+
id: string | number;
|
|
37
|
+
/** Distance from the query vector. Lower is more similar. */
|
|
38
|
+
dist: number;
|
|
39
|
+
metadata?: Record<string, unknown>;
|
|
40
|
+
vector?: number[];
|
|
41
|
+
dimension?: number;
|
|
42
|
+
}
|
|
43
|
+
export interface NBaseClientConfig {
|
|
44
|
+
/** Base URL, e.g. `http://localhost:1307`. */
|
|
45
|
+
url: string;
|
|
46
|
+
/** Optional bearer token, for deployments behind an auth proxy. */
|
|
47
|
+
token?: string;
|
|
48
|
+
/** Per-request timeout in milliseconds. */
|
|
49
|
+
timeoutMs?: number;
|
|
50
|
+
/** Injectable for tests and for runtimes without a global fetch. */
|
|
51
|
+
fetchImpl?: typeof fetch;
|
|
52
|
+
}
|
|
53
|
+
export interface NBaseStats {
|
|
54
|
+
vectorCount?: number;
|
|
55
|
+
metadataCount?: number;
|
|
56
|
+
memoryUsage?: number;
|
|
57
|
+
[key: string]: unknown;
|
|
58
|
+
}
|
|
59
|
+
export declare class NBaseError extends Error {
|
|
60
|
+
readonly status?: number | undefined;
|
|
61
|
+
readonly body?: unknown;
|
|
62
|
+
constructor(message: string, status?: number | undefined, body?: unknown);
|
|
63
|
+
}
|
|
64
|
+
export declare class NBaseVectorClient {
|
|
65
|
+
private readonly config;
|
|
66
|
+
private readonly baseUrl;
|
|
67
|
+
private readonly fetchImpl;
|
|
68
|
+
private readonly timeoutMs;
|
|
69
|
+
constructor(config: NBaseClientConfig);
|
|
70
|
+
private request;
|
|
71
|
+
/** Liveness probe. NBase mounts its status routes at the server root. */
|
|
72
|
+
health(): Promise<{
|
|
73
|
+
status: string;
|
|
74
|
+
version?: string;
|
|
75
|
+
}>;
|
|
76
|
+
/** Vector count, memory use and other database statistics. */
|
|
77
|
+
stats(): Promise<NBaseStats>;
|
|
78
|
+
/** Adds or replaces vectors. NBase accepts a single vector or a bulk array. */
|
|
79
|
+
addVectors(vectors: NBaseVector[]): Promise<{
|
|
80
|
+
count: number;
|
|
81
|
+
ids?: Array<string | number>;
|
|
82
|
+
}>;
|
|
83
|
+
/** Nearest-neighbour search. Results are ordered by ascending `dist`. */
|
|
84
|
+
search(vector: number[], options?: NBaseSearchOptions): Promise<NBaseSearchResult[]>;
|
|
85
|
+
/** Reads a single vector. Returns undefined when the id is unknown. */
|
|
86
|
+
get(id: string | number): Promise<NBaseSearchResult | undefined>;
|
|
87
|
+
exists(id: string | number): Promise<boolean>;
|
|
88
|
+
delete(id: string | number): Promise<boolean>;
|
|
89
|
+
updateMetadata(id: string | number, metadata: Record<string, unknown>): Promise<boolean>;
|
|
90
|
+
}
|
|
91
|
+
/** Creates an NBase vector client. */
|
|
92
|
+
export declare function createNBaseVectorClient(config: NBaseClientConfig): NBaseVectorClient;
|