@an5/adapters 0.2.6 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +96 -2
- package/dist/an5Adapter.d.ts +57 -0
- package/dist/an5Adapter.js +263 -4
- package/dist/base/sql.js +16 -15
- package/dist/base/types.d.ts +6 -0
- package/dist/googlesheets/helpers.js +13 -8
- package/dist/index.d.ts +2 -0
- package/dist/index.js +9 -1
- package/dist/nbase/client.d.ts +92 -0
- package/dist/nbase/client.js +147 -0
- package/dist/nbase/index.d.ts +47 -0
- package/dist/nbase/index.js +80 -0
- package/dist/sqlite/engine.js +2 -1
- package/dotnet/An5Adapter.cs +41 -18
- package/dotnet/Base/Core.cs +56 -26
- package/dotnet/Postgres/PostgresEngine.cs +58 -30
- package/dotnet/Sqlite/SqliteEngine.cs +214 -0
- package/golang/base/where.go +32 -49
- package/golang/base/where_test.go +57 -0
- package/package.json +26 -11
- package/python/an5_adapter.py +74 -9
- package/python/base/__init__.py +1 -1
- package/python/base/metadata.py +25 -1
- package/python/base/sql.py +50 -21
- package/python/sqlite/__init__.py +3 -0
- package/python/sqlite/provider.py +83 -0
- package/python/table_client.py +53 -36
- package/rust/Cargo.toml +31 -0
- package/rust/README.md +71 -0
- package/rust/src/adapter.rs +1314 -0
- package/rust/src/base/dialect.rs +201 -0
- package/rust/src/base/metadata.rs +138 -0
- package/rust/src/base/mod.rs +17 -0
- package/rust/src/base/where_builder.rs +386 -0
- package/rust/src/lib.rs +75 -0
- package/typescript/src/an5Adapter.ts +300 -4
- package/typescript/src/base/sql.ts +15 -14
- package/typescript/src/base/types.ts +6 -0
- package/typescript/src/googlesheets/helpers.ts +11 -8
- package/typescript/src/index.ts +17 -0
- package/typescript/src/nbase/client.ts +214 -0
- package/typescript/src/nbase/index.ts +104 -0
- package/typescript/src/sqlite/engine.ts +2 -1
package/README.md
CHANGED
|
@@ -11,9 +11,9 @@ Standalone runtime database adapter and query engine for AN5 ORM. Provides conne
|
|
|
11
11
|
- **Query Builder & Dialects** — Dialect-aware SQL formatting for MSSQL, PostgreSQL, MySQL, and SQLite
|
|
12
12
|
- **Aggregations & GroupBy** — Standard ORM aggregates (`_count`, `_sum`, `_avg`, `_min`, `_max`) and `groupBy`
|
|
13
13
|
- **Field Math Operators** — Atomic updates with `increment`, `decrement`, `multiply`, `divide`, `set`
|
|
14
|
-
- **Vector Search** — Similarity search using pgvector (PostgreSQL), `VECTOR_DISTANCE` (MSSQL), or in-memory cosine/euclidean/dot similarity
|
|
14
|
+
- **Vector Search** — Similarity search using NBase, pgvector (PostgreSQL), `VECTOR_DISTANCE` (MSSQL), or in-memory cosine/euclidean/dot similarity
|
|
15
15
|
- **Real Transactions** — Interactive and callback transactions (`$transaction(async tx => ...)` and `$begin()/$commit()/$rollback()`)
|
|
16
|
-
- **Cross-Language** — Unified API in TypeScript, Python, .NET (C#), and
|
|
16
|
+
- **Cross-Language** — Unified API in TypeScript, Python, .NET (C#), Golang, and Rust
|
|
17
17
|
- **Google Sheets Database** — Use spreadsheets as a live database with full CRUD and SQL syntax support
|
|
18
18
|
|
|
19
19
|
---
|
|
@@ -44,8 +44,102 @@ dotnet add package An5Adapters
|
|
|
44
44
|
# Included in the npm package under golang/ or via go module
|
|
45
45
|
```
|
|
46
46
|
|
|
47
|
+
### Rust
|
|
48
|
+
|
|
49
|
+
```bash
|
|
50
|
+
cargo add an5-adapters
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
The crate is driver-agnostic — it depends on `sqlx` with `any`, so the app enables the driver it needs. See [crates.io](https://crates.io/crates/an5-adapters) and [docs.rs](https://docs.rs/an5-adapters).
|
|
54
|
+
|
|
47
55
|
---
|
|
48
56
|
|
|
57
|
+
## NBase — Neural Vector Database
|
|
58
|
+
|
|
59
|
+
[NBase](https://github.com/N2FlowJS/nbase) is a partitioned vector database with
|
|
60
|
+
HNSW, LSH and KNN indexing behind a REST API. an5 can run vector search there
|
|
61
|
+
instead of loading every row into memory, which matters once a table holds
|
|
62
|
+
millions of embeddings.
|
|
63
|
+
|
|
64
|
+
The rows stay in your relational table: only the vectors move to NBase, each one
|
|
65
|
+
tagged with its row id, and search results are hydrated from the table.
|
|
66
|
+
|
|
67
|
+
### Configure
|
|
68
|
+
|
|
69
|
+
NBase is configured through the connection string, like every other backend:
|
|
70
|
+
|
|
71
|
+
```ts
|
|
72
|
+
import { createAn5Adapter } from '@an5/adapters';
|
|
73
|
+
|
|
74
|
+
const db = createAn5Adapter({ connectionString: 'nbase://localhost:1307' });
|
|
75
|
+
await db.$connect(); // probes NBase
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Accepted forms:
|
|
79
|
+
|
|
80
|
+
| Form | Result |
|
|
81
|
+
|------|--------|
|
|
82
|
+
| `nbase://localhost:1307` | `http://localhost:1307` |
|
|
83
|
+
| `nbase:localhost:1307` | `http://localhost:1307` |
|
|
84
|
+
| `nbase:http://localhost:1307` | `http://localhost:1307` |
|
|
85
|
+
| `nbase:https://vectors.example.com` | `https://vectors.example.com` |
|
|
86
|
+
| `nbase://localhost:1307?token=t&timeoutMs=500&method=hnsw` | options applied |
|
|
87
|
+
|
|
88
|
+
`?token`, `?timeoutMs`, `?method`, `?idField` and `?modelField` are read from the
|
|
89
|
+
query string, and a path is kept as a base path for deployments behind a
|
|
90
|
+
reverse proxy.
|
|
91
|
+
|
|
92
|
+
This adapter is **vector-only**: it has no relational database, so a search
|
|
93
|
+
returns the NBase hits — id, metadata and `distance` — instead of table rows.
|
|
94
|
+
|
|
95
|
+
### Search
|
|
96
|
+
|
|
97
|
+
With an `nbase://` connection string, search returns the hits themselves:
|
|
98
|
+
|
|
99
|
+
```ts
|
|
100
|
+
const hits = await db.table('Article').vectorSearch({ vector: queryEmbedding, take: 5 });
|
|
101
|
+
// hits[0] = { id: 'Article:42', an5Id: '42', an5Model: 'Article', distance: 0.11 }
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
### Search rows from your own table
|
|
105
|
+
|
|
106
|
+
To get real rows back, keep your database connection string and point it at the
|
|
107
|
+
same NBase instance. NBase is still configured with a connection string, never a
|
|
108
|
+
params object:
|
|
109
|
+
|
|
110
|
+
```ts
|
|
111
|
+
const db = createAn5Adapter({
|
|
112
|
+
connectionString: 'sqlserver://localhost:1433;database=mydb',
|
|
113
|
+
nbase: 'nbase://localhost:1307',
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
await db.table('Article').indexVectorsInNBase({ vectorField: 'embedding' });
|
|
117
|
+
// → { indexed: 1200 }
|
|
118
|
+
|
|
119
|
+
const hits = await db.table('Article').vectorSearch({ vector: queryEmbedding, take: 5 });
|
|
120
|
+
// hits[0] = { id, title, ..., distance }
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
Vectors are tagged with their row id, so a hit is hydrated from the table by
|
|
124
|
+
that id and the result keeps the `{ ...row, distance }` shape the other engines
|
|
125
|
+
return.
|
|
126
|
+
|
|
127
|
+
### Standalone client
|
|
128
|
+
|
|
129
|
+
```ts
|
|
130
|
+
import { createNBaseVectorClient } from '@an5/adapters/nbase';
|
|
131
|
+
|
|
132
|
+
const nbase = createNBaseVectorClient({ url: 'http://localhost:1307' });
|
|
133
|
+
await nbase.health();
|
|
134
|
+
await nbase.addVectors([{ id: 'a:1', vector: [0.1, 0.2], metadata: { an5Id: '1' } }]);
|
|
135
|
+
const { results } = await nbase.search([0.1, 0.2], { k: 5, distanceMetric: 'cosine' });
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
If NBase is unreachable, `vectorSearch` logs a warning and falls back to the
|
|
139
|
+
database's native vector support, then to in-memory similarity, so an outage
|
|
140
|
+
degrades performance rather than breaking queries.
|
|
141
|
+
|
|
142
|
+
|
|
49
143
|
## Usage
|
|
50
144
|
|
|
51
145
|
### TypeScript
|
package/dist/an5Adapter.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { An5SheetsAdapter, An5SheetsAdapterConfig, SheetsTableClient } from './googlesheets';
|
|
2
2
|
import type { An5AdapterConfig, Dialect, TransactionHandle } from './base/types';
|
|
3
|
+
import { type NBaseAdapterConfig, type NBaseVectorClient } from './nbase';
|
|
3
4
|
export type { An5AdapterConfig, Dialect, QueryEngine, TransactionHandle } from './base/types';
|
|
4
5
|
export { setAdapterMetadata, createAdapterProxy, type AdapterMetadata } from './base/metadata';
|
|
5
6
|
export type { TypedAn5Adapter, AdapterAPI } from './typed';
|
|
@@ -13,6 +14,23 @@ export declare class An5Adapter {
|
|
|
13
14
|
private _engineType;
|
|
14
15
|
private _engineConfig;
|
|
15
16
|
sheetsAdapter: An5SheetsAdapter | null;
|
|
17
|
+
private _nbase;
|
|
18
|
+
private _nbaseConfig;
|
|
19
|
+
private _nbaseOnly;
|
|
20
|
+
/**
|
|
21
|
+
* Returns the NBase client for this adapter, if the project configured one.
|
|
22
|
+
* The connection string may also point straight at NBase, which keeps
|
|
23
|
+
* `DATABASE_URL=nbase://host:port` working on its own.
|
|
24
|
+
*/
|
|
25
|
+
nbaseClient(): {
|
|
26
|
+
client: NBaseVectorClient;
|
|
27
|
+
config: NBaseAdapterConfig;
|
|
28
|
+
} | null;
|
|
29
|
+
/**
|
|
30
|
+
* True when the adapter only knows about NBase, so there is no relational
|
|
31
|
+
* database to read rows from.
|
|
32
|
+
*/
|
|
33
|
+
get nbaseOnly(): boolean;
|
|
16
34
|
get dialect(): Dialect;
|
|
17
35
|
constructor(adapterConfig: An5AdapterConfig | An5SheetsAdapterConfig);
|
|
18
36
|
private requireEngine;
|
|
@@ -25,6 +43,15 @@ export declare class An5Adapter {
|
|
|
25
43
|
$executeRaw(query: string, ...values: any[]): Promise<number>;
|
|
26
44
|
$executeRawUnsafe(query: string, ...values: any[]): Promise<number>;
|
|
27
45
|
$connect(): Promise<void>;
|
|
46
|
+
/**
|
|
47
|
+
* Health check for the configured NBase instance, or the database when the
|
|
48
|
+
* adapter has none.
|
|
49
|
+
*/
|
|
50
|
+
$nbaseHealth(): Promise<{
|
|
51
|
+
connected: boolean;
|
|
52
|
+
endpoint?: string;
|
|
53
|
+
error?: string;
|
|
54
|
+
}>;
|
|
28
55
|
$disconnect(): Promise<void>;
|
|
29
56
|
/** Real transaction with BEGIN / COMMIT / ROLLBACK */
|
|
30
57
|
$transaction<R>(fn: (tx: An5AdapterTx) => Promise<R>, options?: {
|
|
@@ -179,6 +206,36 @@ export declare class AdapterTableClient<T = any> {
|
|
|
179
206
|
}): Promise<T>;
|
|
180
207
|
aggregate(args: any): Promise<any>;
|
|
181
208
|
groupBy(args: any): Promise<any[]>;
|
|
209
|
+
/**
|
|
210
|
+
* Runs similarity search in NBase and hydrates the matching rows from the
|
|
211
|
+
* relational table, keeping the `{ ...row, distance }` shape the other
|
|
212
|
+
* engines return.
|
|
213
|
+
*/
|
|
214
|
+
private searchViaNBase;
|
|
215
|
+
/**
|
|
216
|
+
* Vector search for an adapter configured with only an NBase connection
|
|
217
|
+
* string. The hit id and metadata are the result, since there is no
|
|
218
|
+
* relational row to hydrate.
|
|
219
|
+
*/
|
|
220
|
+
private searchViaNBaseOnly;
|
|
221
|
+
/** The primary key column registered for this model, falling back to `id`. */
|
|
222
|
+
private primaryKeyColumn;
|
|
223
|
+
/** Scalar column names known for this model from the generated metadata. */
|
|
224
|
+
private modelFields;
|
|
225
|
+
/**
|
|
226
|
+
* Pushes a table's embedding column into NBase so it can be searched there.
|
|
227
|
+
*
|
|
228
|
+
* Each vector keeps the row id in its metadata, which is what lets
|
|
229
|
+
* `vectorSearch` return real rows instead of bare vector ids.
|
|
230
|
+
*/
|
|
231
|
+
indexVectorsInNBase(args?: {
|
|
232
|
+
vectorField?: string;
|
|
233
|
+
where?: any;
|
|
234
|
+
take?: number;
|
|
235
|
+
batchSize?: number;
|
|
236
|
+
}): Promise<{
|
|
237
|
+
indexed: number;
|
|
238
|
+
}>;
|
|
182
239
|
vectorSearch(args: {
|
|
183
240
|
vector: number[];
|
|
184
241
|
take?: number;
|
package/dist/an5Adapter.js
CHANGED
|
@@ -7,6 +7,15 @@ exports.executorFromAdapter = executorFromAdapter;
|
|
|
7
7
|
const uuid_1 = require("./base/uuid");
|
|
8
8
|
const googlesheets_1 = require("./googlesheets");
|
|
9
9
|
const parseConnectionString_1 = require("./googlesheets/parseConnectionString");
|
|
10
|
+
/**
|
|
11
|
+
* File extensions that mean SQLite when a connection string has no scheme.
|
|
12
|
+
*
|
|
13
|
+
* `.sqlite3` is here as well as in `@an5/orm`'s `detectProvider`, because the ORM
|
|
14
|
+
* validates field types and writes the DDL from the provider this file picks: if the
|
|
15
|
+
* two lists differed, a `.sqlite3` file would be pushed as SQL Server DDL.
|
|
16
|
+
*/
|
|
17
|
+
const SQLITE_FILE_SUFFIXES = ['.sqlite', '.sqlite3', '.db'];
|
|
18
|
+
const nbase_1 = require("./nbase");
|
|
10
19
|
const sql_1 = require("./base/sql");
|
|
11
20
|
const metadata_1 = require("./base/metadata");
|
|
12
21
|
var metadata_2 = require("./base/metadata");
|
|
@@ -171,6 +180,29 @@ async function resolveIncludes(modelName, rows, include, adapter) {
|
|
|
171
180
|
}
|
|
172
181
|
// ─── An5Adapter ────────────────────────────────────────────────────────────────────
|
|
173
182
|
class An5Adapter {
|
|
183
|
+
/**
|
|
184
|
+
* Returns the NBase client for this adapter, if the project configured one.
|
|
185
|
+
* The connection string may also point straight at NBase, which keeps
|
|
186
|
+
* `DATABASE_URL=nbase://host:port` working on its own.
|
|
187
|
+
*/
|
|
188
|
+
nbaseClient() {
|
|
189
|
+
if (this._nbase && this._nbaseConfig)
|
|
190
|
+
return { client: this._nbase, config: this._nbaseConfig };
|
|
191
|
+
if (!this._nbaseConfig)
|
|
192
|
+
return null;
|
|
193
|
+
const config = this._nbaseConfig;
|
|
194
|
+
const client = (0, nbase_1.createNBaseVectorClient)(config);
|
|
195
|
+
this._nbase = client;
|
|
196
|
+
this._nbaseConfig = config;
|
|
197
|
+
return { client, config };
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* True when the adapter only knows about NBase, so there is no relational
|
|
201
|
+
* database to read rows from.
|
|
202
|
+
*/
|
|
203
|
+
get nbaseOnly() {
|
|
204
|
+
return this._nbaseOnly === true;
|
|
205
|
+
}
|
|
174
206
|
get dialect() {
|
|
175
207
|
if (this.sheetsAdapter)
|
|
176
208
|
return 'googlesheets';
|
|
@@ -189,12 +221,32 @@ class An5Adapter {
|
|
|
189
221
|
this._engineType = null;
|
|
190
222
|
this._engineConfig = null;
|
|
191
223
|
this.sheetsAdapter = null;
|
|
224
|
+
this._nbase = null;
|
|
225
|
+
this._nbaseConfig = null;
|
|
226
|
+
this._nbaseOnly = false;
|
|
192
227
|
this._listeners = {};
|
|
193
228
|
this._middlewares = [];
|
|
194
229
|
if (isSheetsConfig(adapterConfig)) {
|
|
195
230
|
this.sheetsAdapter = new googlesheets_1.An5SheetsAdapter(adapterConfig);
|
|
196
231
|
return (0, metadata_1.createAdapterProxy)(this, (name) => this.table(name));
|
|
197
232
|
}
|
|
233
|
+
// Keep the config even when the caller supplies its own engine: the NBase
|
|
234
|
+
// settings and anything else on the config are read from it later.
|
|
235
|
+
this._engineConfig = adapterConfig;
|
|
236
|
+
// `nbase://host:port` as the connection string configures the vector store
|
|
237
|
+
// on its own: there is no relational database, so the adapter is
|
|
238
|
+
// vector-only. The same string passed as `nbase` keeps the database from
|
|
239
|
+
// `connectionString` as the source of rows.
|
|
240
|
+
const rawConnectionString = String(this._engineConfig.connectionString ?? '').trim();
|
|
241
|
+
if ((0, nbase_1.isNBaseConnectionString)(rawConnectionString)) {
|
|
242
|
+
this._nbaseConfig = (0, nbase_1.parseNBaseConnectionString)(rawConnectionString);
|
|
243
|
+
this._nbaseOnly = true;
|
|
244
|
+
return (0, metadata_1.createAdapterProxy)(this, (name) => this.table(name));
|
|
245
|
+
}
|
|
246
|
+
const attached = this._engineConfig.nbase;
|
|
247
|
+
if (attached && (0, nbase_1.isNBaseConnectionString)(attached)) {
|
|
248
|
+
this._nbaseConfig = (0, nbase_1.parseNBaseConnectionString)(attached);
|
|
249
|
+
}
|
|
198
250
|
if (adapterConfig.engine) {
|
|
199
251
|
this._engine = adapterConfig.engine;
|
|
200
252
|
this._engineType = this._engine?.dialect || 'sqlite';
|
|
@@ -206,19 +258,26 @@ class An5Adapter {
|
|
|
206
258
|
this._engineType = 'sqlite';
|
|
207
259
|
return (0, metadata_1.createAdapterProxy)(this, (name) => this.table(name));
|
|
208
260
|
}
|
|
261
|
+
// Lower-cased for the dialect test only; the connection string itself keeps its
|
|
262
|
+
// case for the driver. A URI scheme is case-insensitive, and `@an5/orm` reads the
|
|
263
|
+
// provider the same way to validate field types and write DDL — if the two
|
|
264
|
+
// disagreed, a schema would be checked against one database and run on another.
|
|
209
265
|
const cs = (adapterConfig.connectionString || '').trim();
|
|
210
|
-
|
|
266
|
+
const forDialect = cs.toLowerCase();
|
|
267
|
+
if (forDialect.startsWith('googlesheets://')) {
|
|
211
268
|
this.sheetsAdapter = new googlesheets_1.An5SheetsAdapter((0, parseConnectionString_1.parseSheetsConnectionString)(cs));
|
|
212
269
|
return (0, metadata_1.createAdapterProxy)(this, (name) => this.table(name));
|
|
213
270
|
}
|
|
214
271
|
this._engineConfig = adapterConfig;
|
|
215
|
-
if (
|
|
272
|
+
if (forDialect.startsWith('postgres://') || forDialect.startsWith('postgresql://')) {
|
|
216
273
|
this._engineType = 'postgres';
|
|
217
274
|
}
|
|
218
|
-
else if (
|
|
275
|
+
else if (forDialect.startsWith('mysql://') || forDialect.startsWith('mariadb://')) {
|
|
219
276
|
this._engineType = 'mysql';
|
|
220
277
|
}
|
|
221
|
-
else if (
|
|
278
|
+
else if (forDialect === ':memory:'
|
|
279
|
+
|| forDialect.startsWith('sqlite:')
|
|
280
|
+
|| SQLITE_FILE_SUFFIXES.some((suffix) => forDialect.endsWith(suffix))) {
|
|
222
281
|
this._engineType = 'sqlite';
|
|
223
282
|
}
|
|
224
283
|
else {
|
|
@@ -310,8 +369,36 @@ class An5Adapter {
|
|
|
310
369
|
async $connect() {
|
|
311
370
|
if (this.sheetsAdapter)
|
|
312
371
|
return this.sheetsAdapter.$connect();
|
|
372
|
+
if (this._nbaseOnly) {
|
|
373
|
+
// NBase is stateless over HTTP, so there is nothing to open. Report the
|
|
374
|
+
// endpoint so a misconfiguration is obvious.
|
|
375
|
+
const config = this.nbaseClient()?.config;
|
|
376
|
+
if (config)
|
|
377
|
+
await this.nbaseClient().client.health();
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
313
380
|
await (await this.requireEngine()).connect();
|
|
314
381
|
}
|
|
382
|
+
/**
|
|
383
|
+
* Health check for the configured NBase instance, or the database when the
|
|
384
|
+
* adapter has none.
|
|
385
|
+
*/
|
|
386
|
+
async $nbaseHealth() {
|
|
387
|
+
const nbase = this.nbaseClient();
|
|
388
|
+
if (!nbase)
|
|
389
|
+
return { connected: false, error: 'No NBase instance configured.' };
|
|
390
|
+
try {
|
|
391
|
+
const health = await nbase.client.health();
|
|
392
|
+
return { connected: health.status === 'ok', endpoint: nbase.config.url };
|
|
393
|
+
}
|
|
394
|
+
catch (err) {
|
|
395
|
+
return {
|
|
396
|
+
connected: false,
|
|
397
|
+
endpoint: nbase.config.url,
|
|
398
|
+
error: err instanceof Error ? err.message : String(err),
|
|
399
|
+
};
|
|
400
|
+
}
|
|
401
|
+
}
|
|
315
402
|
async $disconnect() {
|
|
316
403
|
if (this.sheetsAdapter)
|
|
317
404
|
return this.sheetsAdapter.$disconnect();
|
|
@@ -1001,6 +1088,158 @@ class AdapterTableClient {
|
|
|
1001
1088
|
}
|
|
1002
1089
|
return this.doExec(query, params);
|
|
1003
1090
|
}
|
|
1091
|
+
/**
|
|
1092
|
+
* Runs similarity search in NBase and hydrates the matching rows from the
|
|
1093
|
+
* relational table, keeping the `{ ...row, distance }` shape the other
|
|
1094
|
+
* engines return.
|
|
1095
|
+
*/
|
|
1096
|
+
async searchViaNBase(nbase, args, metric, take) {
|
|
1097
|
+
const { client, config } = nbase;
|
|
1098
|
+
const idField = config.idField ?? 'an5Id';
|
|
1099
|
+
const modelField = config.modelField ?? 'an5Model';
|
|
1100
|
+
const results = await client.search(args.vector, {
|
|
1101
|
+
k: take,
|
|
1102
|
+
// NBase implements cosine and euclidean; a dot-product request falls back
|
|
1103
|
+
// to cosine, which still orders the results sensibly.
|
|
1104
|
+
distanceMetric: metric === 'euclidean' ? 'euclidean' : 'cosine',
|
|
1105
|
+
includeMetadata: true,
|
|
1106
|
+
...(config.method ? { method: config.method } : {}),
|
|
1107
|
+
});
|
|
1108
|
+
// Map each hit back to its row id. Ids are written as `Model:id`, and the
|
|
1109
|
+
// metadata key is honoured for projects that index vectors themselves.
|
|
1110
|
+
const ids = [];
|
|
1111
|
+
const distances = new Map();
|
|
1112
|
+
for (const hit of results) {
|
|
1113
|
+
const fromMetadata = hit.metadata?.[idField];
|
|
1114
|
+
const rawId = fromMetadata !== undefined && fromMetadata !== null
|
|
1115
|
+
? String(fromMetadata)
|
|
1116
|
+
: (0, nbase_1.parseVectorId)(hit.id).id;
|
|
1117
|
+
if (!rawId)
|
|
1118
|
+
continue;
|
|
1119
|
+
const model = hit.metadata?.[modelField] ?? (0, nbase_1.parseVectorId)(hit.id).model;
|
|
1120
|
+
ids.push(model === this.modelName ? rawId : `${model}:${rawId}`);
|
|
1121
|
+
distances.set(ids[ids.length - 1], hit.dist);
|
|
1122
|
+
}
|
|
1123
|
+
if (ids.length === 0)
|
|
1124
|
+
return [];
|
|
1125
|
+
const pk = this.primaryKeyColumn();
|
|
1126
|
+
const inList = ids.map((_, i) => `@nbase_id_${i}`).join(', ');
|
|
1127
|
+
const idParams = {};
|
|
1128
|
+
ids.forEach((id, i) => { idParams[`nbase_id_${i}`] = id; });
|
|
1129
|
+
idParams.take = take;
|
|
1130
|
+
const selectColumns = this.modelFields().map((f) => (0, sql_1.quote)(f, this.dialect)).join(', ');
|
|
1131
|
+
const columns = selectColumns ? `${selectColumns}, ${(0, sql_1.quote)(pk, this.dialect)} AS __an5_nbase_key` : `${(0, sql_1.quote)(pk, this.dialect)} AS __an5_nbase_key`;
|
|
1132
|
+
const query = `SELECT ${columns} FROM ${this.tableName} WHERE ${(0, sql_1.quote)(pk, this.dialect)} IN (${inList})`;
|
|
1133
|
+
const rows = await this.adapter.exec(query, idParams);
|
|
1134
|
+
const byKey = new Map();
|
|
1135
|
+
for (const row of rows ?? []) {
|
|
1136
|
+
byKey.set(String(row.__an5_nbase_key), row);
|
|
1137
|
+
}
|
|
1138
|
+
const hydrated = [];
|
|
1139
|
+
for (const id of ids) {
|
|
1140
|
+
const row = byKey.get(id);
|
|
1141
|
+
if (!row)
|
|
1142
|
+
continue;
|
|
1143
|
+
const { __an5_nbase_key, ...rest } = row;
|
|
1144
|
+
hydrated.push({ ...rest, distance: distances.get(id) ?? 0 });
|
|
1145
|
+
if (hydrated.length >= take)
|
|
1146
|
+
break;
|
|
1147
|
+
}
|
|
1148
|
+
return hydrated;
|
|
1149
|
+
}
|
|
1150
|
+
/**
|
|
1151
|
+
* Vector search for an adapter configured with only an NBase connection
|
|
1152
|
+
* string. The hit id and metadata are the result, since there is no
|
|
1153
|
+
* relational row to hydrate.
|
|
1154
|
+
*/
|
|
1155
|
+
async searchViaNBaseOnly(nbase, args, metric, take) {
|
|
1156
|
+
const { client, config } = nbase;
|
|
1157
|
+
const idField = config.idField ?? 'an5Id';
|
|
1158
|
+
const results = await client.search(args.vector, {
|
|
1159
|
+
k: take,
|
|
1160
|
+
distanceMetric: metric === 'euclidean' ? 'euclidean' : 'cosine',
|
|
1161
|
+
includeMetadata: true,
|
|
1162
|
+
...(config.method ? { method: config.method } : {}),
|
|
1163
|
+
});
|
|
1164
|
+
return results.map((hit) => {
|
|
1165
|
+
const metadata = { ...(hit.metadata ?? {}) };
|
|
1166
|
+
if (metadata[idField] === undefined) {
|
|
1167
|
+
const parsed = (0, nbase_1.parseVectorId)(hit.id);
|
|
1168
|
+
if (parsed.id)
|
|
1169
|
+
metadata[idField] = parsed.id;
|
|
1170
|
+
if (parsed.model)
|
|
1171
|
+
metadata[config.modelField ?? 'an5Model'] = parsed.model;
|
|
1172
|
+
}
|
|
1173
|
+
return { id: hit.id, ...metadata, distance: hit.dist };
|
|
1174
|
+
});
|
|
1175
|
+
}
|
|
1176
|
+
/** The primary key column registered for this model, falling back to `id`. */
|
|
1177
|
+
primaryKeyColumn() {
|
|
1178
|
+
const fields = this.modelFields();
|
|
1179
|
+
const marked = fields.find((f) => f === 'id');
|
|
1180
|
+
return marked ?? fields[0] ?? 'id';
|
|
1181
|
+
}
|
|
1182
|
+
/** Scalar column names known for this model from the generated metadata. */
|
|
1183
|
+
modelFields() {
|
|
1184
|
+
const fields = (0, metadata_1.getFieldsForModel)(this.modelName);
|
|
1185
|
+
if (fields && typeof fields === 'object' && !Array.isArray(fields)) {
|
|
1186
|
+
return Object.keys(fields);
|
|
1187
|
+
}
|
|
1188
|
+
return [];
|
|
1189
|
+
}
|
|
1190
|
+
/**
|
|
1191
|
+
* Pushes a table's embedding column into NBase so it can be searched there.
|
|
1192
|
+
*
|
|
1193
|
+
* Each vector keeps the row id in its metadata, which is what lets
|
|
1194
|
+
* `vectorSearch` return real rows instead of bare vector ids.
|
|
1195
|
+
*/
|
|
1196
|
+
async indexVectorsInNBase(args = {}) {
|
|
1197
|
+
const nbase = this.adapter.nbaseClient?.() ?? null;
|
|
1198
|
+
if (!nbase) {
|
|
1199
|
+
throw new Error('No NBase instance configured. Use a nbase:// connection string, e.g. createAn5Adapter({ connectionString: "nbase://localhost:1307" }).');
|
|
1200
|
+
}
|
|
1201
|
+
if (this.adapter.nbaseOnly) {
|
|
1202
|
+
throw new Error('indexVectorsInNBase reads the vectors from a table, but this adapter was created with an nbase:// connection string and has no database. ' +
|
|
1203
|
+
'Create the adapter with your database connection string to index, or add the vectors with the NBase client directly.');
|
|
1204
|
+
}
|
|
1205
|
+
const vectorField = args.vectorField || 'embedding';
|
|
1206
|
+
const batchSize = args.batchSize ?? 200;
|
|
1207
|
+
const pk = this.primaryKeyColumn();
|
|
1208
|
+
const rows = await this.findMany({ ...(args.where ? { where: args.where } : {}), ...(args.take ? { take: args.take } : {}) });
|
|
1209
|
+
let indexed = 0;
|
|
1210
|
+
for (let i = 0; i < rows.length; i += batchSize) {
|
|
1211
|
+
const batch = rows.slice(i, i + batchSize);
|
|
1212
|
+
const vectors = batch
|
|
1213
|
+
.map((row) => {
|
|
1214
|
+
const raw = row[vectorField];
|
|
1215
|
+
if (raw === undefined || raw === null)
|
|
1216
|
+
return null;
|
|
1217
|
+
let vector;
|
|
1218
|
+
try {
|
|
1219
|
+
vector = typeof raw === 'string' ? JSON.parse(raw) : raw;
|
|
1220
|
+
}
|
|
1221
|
+
catch {
|
|
1222
|
+
return null;
|
|
1223
|
+
}
|
|
1224
|
+
if (!Array.isArray(vector) || vector.length === 0)
|
|
1225
|
+
return null;
|
|
1226
|
+
return {
|
|
1227
|
+
id: (0, nbase_1.buildVectorId)(this.modelName, String(row[pk])),
|
|
1228
|
+
vector,
|
|
1229
|
+
metadata: {
|
|
1230
|
+
[nbase.config.idField ?? 'an5Id']: String(row[pk]),
|
|
1231
|
+
[nbase.config.modelField ?? 'an5Model']: this.modelName,
|
|
1232
|
+
},
|
|
1233
|
+
};
|
|
1234
|
+
})
|
|
1235
|
+
.filter((v) => v !== null);
|
|
1236
|
+
if (vectors.length === 0)
|
|
1237
|
+
continue;
|
|
1238
|
+
const result = await nbase.client.addVectors(vectors);
|
|
1239
|
+
indexed += result.count;
|
|
1240
|
+
}
|
|
1241
|
+
return { indexed };
|
|
1242
|
+
}
|
|
1004
1243
|
async vectorSearch(args) {
|
|
1005
1244
|
const vectorField = args.vectorField || 'embedding';
|
|
1006
1245
|
const METRICS = ['cosine', 'euclidean', 'dot'];
|
|
@@ -1012,6 +1251,26 @@ class AdapterTableClient {
|
|
|
1012
1251
|
const vectorJson = JSON.stringify(args.vector);
|
|
1013
1252
|
const col = (0, sql_1.quote)(vectorField, this.dialect);
|
|
1014
1253
|
const params = { query_vector: vectorJson };
|
|
1254
|
+
// 0. NBase, when the project configured one: the vectors live in the vector
|
|
1255
|
+
// database, so the search never loads the table into memory.
|
|
1256
|
+
const nbase = this.adapter.nbaseClient?.() ?? null;
|
|
1257
|
+
if (nbase) {
|
|
1258
|
+
try {
|
|
1259
|
+
// With an NBase-only connection string there is no table to read rows
|
|
1260
|
+
// from, so the hits are returned as the vector metadata itself.
|
|
1261
|
+
if (this.adapter.nbaseOnly) {
|
|
1262
|
+
return (await this.searchViaNBaseOnly(nbase, args, metric, take));
|
|
1263
|
+
}
|
|
1264
|
+
return await this.searchViaNBase(nbase, args, metric, take);
|
|
1265
|
+
}
|
|
1266
|
+
catch (err) {
|
|
1267
|
+
if (!(err instanceof nbase_1.NBaseError))
|
|
1268
|
+
throw err;
|
|
1269
|
+
// A NBase outage must not take vector search down with it; fall through
|
|
1270
|
+
// to the database instead of failing the query.
|
|
1271
|
+
console.warn(`[an5] NBase vector search unavailable (${err.message}); falling back to ${this.dialect}.`);
|
|
1272
|
+
}
|
|
1273
|
+
}
|
|
1015
1274
|
// 1. Try native dialect execution
|
|
1016
1275
|
if (this.dialect === 'postgres') {
|
|
1017
1276
|
const op = metric === 'cosine' ? '<=>' : metric === 'dot' ? '<#>' : '<->';
|
package/dist/base/sql.js
CHANGED
|
@@ -68,9 +68,10 @@ function parseWhere(modelName, where, params, dialect, prefix = '', ctx) {
|
|
|
68
68
|
if (!where)
|
|
69
69
|
return '';
|
|
70
70
|
const conditions = [];
|
|
71
|
+
const relations = ctx?.relationMap?.[modelName] || {};
|
|
71
72
|
const cleanWhere = {};
|
|
72
73
|
for (const [key, value] of Object.entries(where)) {
|
|
73
|
-
if (key.includes('_') &&
|
|
74
|
+
if (key.includes('_') && !relations[key] &&
|
|
74
75
|
value && typeof value === 'object' &&
|
|
75
76
|
!(value instanceof Date) &&
|
|
76
77
|
!isOperatorValue(value)) {
|
|
@@ -80,13 +81,13 @@ function parseWhere(modelName, where, params, dialect, prefix = '', ctx) {
|
|
|
80
81
|
cleanWhere[key] = value;
|
|
81
82
|
}
|
|
82
83
|
}
|
|
83
|
-
const relations = ctx?.relationMap?.[modelName] || {};
|
|
84
84
|
const colPrefix = ctx?.colPrefix || '';
|
|
85
85
|
for (const [key, value] of Object.entries(cleanWhere)) {
|
|
86
|
+
if (value === undefined)
|
|
87
|
+
continue;
|
|
86
88
|
if (key === 'OR' && Array.isArray(value)) {
|
|
87
|
-
const sub = value.map((v, i) => parseWhere(modelName, v, params, dialect, `${prefix}or_${i}_`, ctx))
|
|
88
|
-
|
|
89
|
-
conditions.push(`(${sub.join(' OR ')})`);
|
|
89
|
+
const sub = value.map((v, i) => parseWhere(modelName, v, params, dialect, `${prefix}or_${i}_`, ctx) || '1=1');
|
|
90
|
+
conditions.push(sub.length > 0 ? `(${sub.join(' OR ')})` : '1=0');
|
|
90
91
|
}
|
|
91
92
|
else if (key === 'AND') {
|
|
92
93
|
const items = Array.isArray(value) ? value : [value];
|
|
@@ -96,12 +97,12 @@ function parseWhere(modelName, where, params, dialect, prefix = '', ctx) {
|
|
|
96
97
|
}
|
|
97
98
|
else if (key === 'NOT') {
|
|
98
99
|
const items = Array.isArray(value) ? value : [value];
|
|
99
|
-
const sub = items.map((v, i) => parseWhere(modelName, v, params, dialect, `${prefix}not_${i}_`, ctx))
|
|
100
|
+
const sub = items.map((v, i) => parseWhere(modelName, v, params, dialect, `${prefix}not_${i}_`, ctx) || '1=1');
|
|
100
101
|
if (sub.length > 0)
|
|
101
|
-
conditions.push(`NOT (${sub.join('
|
|
102
|
+
conditions.push(`NOT (${sub.join(' OR ')})`);
|
|
102
103
|
}
|
|
103
104
|
else if (relations[key] && value && typeof value === 'object') {
|
|
104
|
-
const clause = buildRelationClause(relations[key], value, modelName, params, dialect, prefix
|
|
105
|
+
const clause = buildRelationClause(relations[key], value, modelName, params, dialect, `${prefix}${key}_`, ctx);
|
|
105
106
|
if (clause)
|
|
106
107
|
conditions.push(clause);
|
|
107
108
|
}
|
|
@@ -220,16 +221,16 @@ function buildRelationClause(relation, value, modelName, params, dialect, prefix
|
|
|
220
221
|
const parts = [];
|
|
221
222
|
if (isMany) {
|
|
222
223
|
if (value.some !== undefined) {
|
|
223
|
-
const inner = buildInnerWhere(relation.modelName, value.some === null ? {} : value.some, params, dialect, prefix
|
|
224
|
+
const inner = buildInnerWhere(relation.modelName, value.some === null ? {} : value.some, params, dialect, `${prefix}some_`, ctx, childAlias);
|
|
224
225
|
parts.push(buildExists(inner));
|
|
225
226
|
}
|
|
226
227
|
if (value.none !== undefined) {
|
|
227
|
-
const inner = buildInnerWhere(relation.modelName, value.none === null ? {} : value.none, params, dialect, prefix
|
|
228
|
+
const inner = buildInnerWhere(relation.modelName, value.none === null ? {} : value.none, params, dialect, `${prefix}none_`, ctx, childAlias);
|
|
228
229
|
parts.push(`NOT ${buildExists(inner)}`);
|
|
229
230
|
}
|
|
230
231
|
if (value.every !== undefined) {
|
|
231
232
|
// "every" = no related row violates the condition.
|
|
232
|
-
const inner = buildInnerWhere(relation.modelName, value.every === null ? {} : value.every, params, dialect, prefix
|
|
233
|
+
const inner = buildInnerWhere(relation.modelName, value.every === null ? {} : value.every, params, dialect, `${prefix}every_`, ctx, childAlias);
|
|
233
234
|
const negated = inner ? `NOT (${inner})` : '';
|
|
234
235
|
const violated = `EXISTS (SELECT 1 FROM ${childTable} AS ${childAlias} WHERE ${correlation}${negated ? ` AND ${negated}` : ' AND 1=0'})`;
|
|
235
236
|
parts.push(`NOT ${violated}`);
|
|
@@ -237,12 +238,12 @@ function buildRelationClause(relation, value, modelName, params, dialect, prefix
|
|
|
237
238
|
}
|
|
238
239
|
else {
|
|
239
240
|
if (value.is !== undefined) {
|
|
240
|
-
const inner = buildInnerWhere(relation.modelName, value.is === null ? {} : value.is, params, dialect, prefix
|
|
241
|
-
parts.push(buildExists(inner));
|
|
241
|
+
const inner = buildInnerWhere(relation.modelName, value.is === null ? {} : value.is, params, dialect, `${prefix}is_`, ctx, childAlias);
|
|
242
|
+
parts.push(value.is === null ? `NOT ${buildExists(inner)}` : buildExists(inner));
|
|
242
243
|
}
|
|
243
244
|
if (value.isNot !== undefined) {
|
|
244
|
-
const inner = buildInnerWhere(relation.modelName, value.isNot === null ? {} : value.isNot, params, dialect, prefix
|
|
245
|
-
parts.push(`NOT ${buildExists(inner)}`);
|
|
245
|
+
const inner = buildInnerWhere(relation.modelName, value.isNot === null ? {} : value.isNot, params, dialect, `${prefix}isNot_`, ctx, childAlias);
|
|
246
|
+
parts.push(value.isNot === null ? buildExists(inner) : `NOT ${buildExists(inner)}`);
|
|
246
247
|
}
|
|
247
248
|
}
|
|
248
249
|
return parts.length > 0 ? parts.join(' AND ') : null;
|
package/dist/base/types.d.ts
CHANGED
|
@@ -1,6 +1,12 @@
|
|
|
1
1
|
export type Dialect = 'mssql' | 'postgres' | 'mysql' | 'sqlite' | 'googlesheets';
|
|
2
2
|
export interface An5AdapterConfig {
|
|
3
3
|
connectionString?: string;
|
|
4
|
+
/**
|
|
5
|
+
* NBase (Neural Vector Database) endpoint, written as a connection string,
|
|
6
|
+
* e.g. `nbase://localhost:1307`. Attach it here when vector search should
|
|
7
|
+
* run in NBase while the rows stay in `connectionString`'s database.
|
|
8
|
+
*/
|
|
9
|
+
nbase?: string;
|
|
4
10
|
engine?: QueryEngine;
|
|
5
11
|
db?: any;
|
|
6
12
|
driver?: any;
|