@fgv/ts-agent-memory-sqlite-vec 5.1.0-50 → 5.1.0-52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -1
- package/dist/packlets/sqlite-vec-index/connection.js +54 -0
- package/dist/packlets/sqlite-vec-index/connection.js.map +1 -0
- package/dist/packlets/sqlite-vec-index/model.js.map +1 -1
- package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +162 -8
- package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -1
- package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +115 -4
- package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -1
- package/dist/ts-agent-memory-sqlite-vec.d.ts +258 -10
- package/lib/packlets/sqlite-vec-index/connection.d.ts +42 -0
- package/lib/packlets/sqlite-vec-index/connection.d.ts.map +1 -0
- package/lib/packlets/sqlite-vec-index/connection.js +91 -0
- package/lib/packlets/sqlite-vec-index/connection.js.map +1 -0
- package/lib/packlets/sqlite-vec-index/model.d.ts +92 -0
- package/lib/packlets/sqlite-vec-index/model.d.ts.map +1 -1
- package/lib/packlets/sqlite-vec-index/model.js.map +1 -1
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts +89 -8
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts.map +1 -1
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +162 -8
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -1
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts +78 -5
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts.map +1 -1
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +115 -4
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -1
- package/package.json +7 -7
package/README.md
CHANGED
|
@@ -25,6 +25,31 @@ rush add -p @fgv/ts-agent-memory-sqlite-vec # or npm/pnpm add
|
|
|
25
25
|
|
|
26
26
|
## Quick start
|
|
27
27
|
|
|
28
|
+
Two factories, differing only in **who owns the connection**. If this index is the only
|
|
29
|
+
thing on the file, `open` is the shorter path and needs no `better-sqlite3` import of
|
|
30
|
+
your own:
|
|
31
|
+
|
|
32
|
+
```ts
|
|
33
|
+
import { SqliteVecVectorIndex } from '@fgv/ts-agent-memory-sqlite-vec';
|
|
34
|
+
import { FileTreeMemoryStore } from '@fgv/ts-agent-memory';
|
|
35
|
+
|
|
36
|
+
// We open the file and hand back a disposer for the connection we created.
|
|
37
|
+
const handle = (await SqliteVecVectorIndex.open({ path: '/path/to/vault/vectors.db' })).orThrow();
|
|
38
|
+
|
|
39
|
+
const store = (
|
|
40
|
+
await FileTreeMemoryStore.create({ root, registry, vectorIndex: handle.index, embed })
|
|
41
|
+
).orThrow();
|
|
42
|
+
|
|
43
|
+
// ...use the store; embeddings are written to vectors.db on every put.
|
|
44
|
+
handle.close(); // closes the connection THIS open() created; idempotent.
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
`SqliteVecFragmentIndex.open({ path })` is the identical shape for the fragment lane.
|
|
48
|
+
|
|
49
|
+
**Two `open` calls on one path give two independent connections, not a shared one.** To
|
|
50
|
+
back a record index *and* a fragment index with a single connection, own it yourself and
|
|
51
|
+
pass it to both `create` methods:
|
|
52
|
+
|
|
28
53
|
```ts
|
|
29
54
|
import Database from 'better-sqlite3';
|
|
30
55
|
import { SqliteVecVectorIndex } from '@fgv/ts-agent-memory-sqlite-vec';
|
|
@@ -42,9 +67,32 @@ const store = (
|
|
|
42
67
|
).orThrow();
|
|
43
68
|
|
|
44
69
|
// ...use the store; embeddings are written to vectors.db on every put.
|
|
70
|
+
vectorIndex.release(); // drop this index's prepared statements — see below
|
|
45
71
|
db.close(); // you own the lifecycle — this index never closes your connection.
|
|
46
72
|
```
|
|
47
73
|
|
|
74
|
+
### Releasing an index over a connection you own
|
|
75
|
+
|
|
76
|
+
An index caches prepared `Statement` objects. Those hold a reference to the
|
|
77
|
+
connection, so if you close a connection you own while an index over it is still
|
|
78
|
+
reachable, its statements outlive the connection and their native destructors run
|
|
79
|
+
whenever GC reaches them — potentially during process teardown.
|
|
80
|
+
|
|
81
|
+
**`release()` drops those statements and marks the index unusable. It never touches
|
|
82
|
+
the connection**, which is why it is safe to expose on a `create()`-made index that
|
|
83
|
+
does not own one. `open()`'s handle calls it for you before closing; with `create()`
|
|
84
|
+
you own the ordering, and it is `release()` then `close()`.
|
|
85
|
+
|
|
86
|
+
A released index **fails** (or, for the synchronous counts `size` / `recordCount` /
|
|
87
|
+
`fragmentCount`, **throws**) rather than answering. That is deliberate: an index that
|
|
88
|
+
has simply never had an `add` also holds no statements, and answering `0` from a
|
|
89
|
+
released one would be indistinguishable from an empty one.
|
|
90
|
+
|
|
91
|
+
Note the limit honestly: `better-sqlite3` exposes no public `finalize()`, so dropping
|
|
92
|
+
the last reference does not finalize a statement — it makes it collectable *earlier*,
|
|
93
|
+
while the environment is alive, instead of surviving to teardown. That narrows the
|
|
94
|
+
window; it is not a proof against it.
|
|
95
|
+
|
|
48
96
|
`SqliteVecVectorIndex` implements the full `IVectorIndex` contract — `add(target, vector)`, `remove(target)`, `query(vector, topK)` — with the **same semantics as `InMemoryCosineIndex`**:
|
|
49
97
|
|
|
50
98
|
- Keyed by the canonical `edgeTargetKey` (`(scope, id)`), so records that share a filename stem across scopes never collide.
|
|
@@ -108,7 +156,7 @@ Known instance: the release that added `IEmbeddedFragment.fragmentId` added a `+
|
|
|
108
156
|
Deliberately excluded — reach for the upstream libraries (or a different backend) directly if you need these:
|
|
109
157
|
|
|
110
158
|
- **ANN / large-N indexing.** Query is a brute-force `vec0` KNN scan — correct and durable for the same "thousands of records" regime the in-memory index targets. An approximate-nearest-neighbor structure for very large N is a different backend behind the same `IVectorIndex` / `IFragmentVectorIndex` seam. (This applies to both indexes, including `SqliteVecFragmentIndex`, whose capped query fetches the full ranked set.)
|
|
111
|
-
- **Connection lifecycle.**
|
|
159
|
+
- **Connection lifecycle beyond plain open/close.** With `create({ database })` you open and close the `better-sqlite3` `Database` and this index never does; with `open({ path })` this package opens the file and the returned handle's `close()` disposes of exactly what it opened. Either way, pooling, WAL/pragma tuning, backups, and multi-process coordination are yours.
|
|
112
160
|
- **Embedding.** This is a vector *index*, not an embedder — the store's consumer-wired `MemoryEmbedder` produces the vectors (`@fgv/ts-extras/ai-assist` `callProviderEmbedding`, `@fgv/ts-extras-transformers`, etc.).
|
|
113
161
|
- **A browser sibling.** `better-sqlite3` is Node-only. A WASM-SQLite browser variant, if ever needed, is a separate package.
|
|
114
162
|
- **Schema migration of any kind.** Re-embedding with a different-dimension model against an existing table fails loudly, and a package release that changes a `vec0` table's columns requires a drop-and-re-index — see [Upgrading](#upgrading-vec0-schema-changes-require-a-drop-and-re-index). Drop the table (or use a new `tableName`) to re-index; `vec0` cannot be altered in place.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Copyright (c) 2026 Erik Fortune
|
|
3
|
+
* SPDX-License-Identifier: MIT
|
|
4
|
+
*/
|
|
5
|
+
import { captureAsyncResult, captureResult } from '@fgv/ts-utils';
|
|
6
|
+
/**
|
|
7
|
+
* Opens a `better-sqlite3` connection this package owns.
|
|
8
|
+
*
|
|
9
|
+
* @remarks
|
|
10
|
+
* **The value import of `better-sqlite3` lives here, and it is deliberately
|
|
11
|
+
* lazy.** Every other module in this package imports the driver as `import type`
|
|
12
|
+
* only, so merely importing `@fgv/ts-agent-memory-sqlite-vec` has never loaded the
|
|
13
|
+
* native binding — a property the path-based factories must not cost consumers who
|
|
14
|
+
* only ever call `create()`. A static import would move the native load (and its
|
|
15
|
+
* failure mode) to package-load time for everyone. The dynamic `import()` keeps it
|
|
16
|
+
* at the one call that actually needs a connection.
|
|
17
|
+
*
|
|
18
|
+
* Taking this import is the entire point of the path-based factories: it is the
|
|
19
|
+
* only place this wrapper leaked its own dependency into consumer source.
|
|
20
|
+
*
|
|
21
|
+
* @param path - Filesystem path to the database file. `':memory:'` is accepted and
|
|
22
|
+
* yields an owned ephemeral connection.
|
|
23
|
+
* @param label - Package-facing prefix for the failure message.
|
|
24
|
+
* @returns `Success` with a connection this package owns, or `Failure` if the
|
|
25
|
+
* driver could not be loaded or the file could not be opened.
|
|
26
|
+
* @internal
|
|
27
|
+
*/
|
|
28
|
+
export async function openOwnedConnection(path, label) {
|
|
29
|
+
return (await captureAsyncResult(async () => (await import('better-sqlite3')).default))
|
|
30
|
+
.withErrorFormat((m) => `${label}: failed to load the 'better-sqlite3' driver: ${m}`)
|
|
31
|
+
.onSuccess((driver) => captureResult(() => new driver(path)).withErrorFormat((m) => `${label}: failed to open '${path}': ${m}`));
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Closes a connection this package opened.
|
|
35
|
+
*
|
|
36
|
+
* @remarks
|
|
37
|
+
* Only ever called on a connection produced by {@link openOwnedConnection}. A
|
|
38
|
+
* connection supplied through a `create()` param is the consumer's and is never
|
|
39
|
+
* closed here — that asymmetry is the reason `close` travels on the handle
|
|
40
|
+
* returned by `open` rather than sitting on the index class.
|
|
41
|
+
*
|
|
42
|
+
* `better-sqlite3`'s own `close()` is safe to call on an already-closed
|
|
43
|
+
* connection, so a second `close()` on the same handle succeeds rather than
|
|
44
|
+
* failing.
|
|
45
|
+
*
|
|
46
|
+
* @internal
|
|
47
|
+
*/
|
|
48
|
+
export function closeOwnedConnection(db, label) {
|
|
49
|
+
return captureResult(() => {
|
|
50
|
+
db.close();
|
|
51
|
+
return true;
|
|
52
|
+
}).withErrorFormat((m) => `${label}: failed to close the connection: ${m}`);
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=connection.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"connection.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/connection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAU,kBAAkB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE1E;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,KAAa;IAEb,OAAO,CAAC,MAAM,kBAAkB,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;SACpF,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,iDAAiD,CAAC,EAAE,CAAC;SACpF,SAAS,CAAC,CAAC,MAAM,EAAE,EAAE,CACpB,aAAa,CAAC,GAAG,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,qBAAqB,IAAI,MAAM,CAAC,EAAE,CAAC,CACzG,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,oBAAoB,CAAC,EAA0B,EAAE,KAAa;IAC5E,OAAO,aAAa,CAAO,GAAG,EAAE;QAC9B,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,qCAAqC,CAAC,EAAE,CAAC,CAAC;AAC9E,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport type BetterSqlite3 from 'better-sqlite3';\nimport { Result, captureAsyncResult, captureResult } from '@fgv/ts-utils';\n\n/**\n * Opens a `better-sqlite3` connection this package owns.\n *\n * @remarks\n * **The value import of `better-sqlite3` lives here, and it is deliberately\n * lazy.** Every other module in this package imports the driver as `import type`\n * only, so merely importing `@fgv/ts-agent-memory-sqlite-vec` has never loaded the\n * native binding — a property the path-based factories must not cost consumers who\n * only ever call `create()`. A static import would move the native load (and its\n * failure mode) to package-load time for everyone. The dynamic `import()` keeps it\n * at the one call that actually needs a connection.\n *\n * Taking this import is the entire point of the path-based factories: it is the\n * only place this wrapper leaked its own dependency into consumer source.\n *\n * @param path - Filesystem path to the database file. `':memory:'` is accepted and\n * yields an owned ephemeral connection.\n * @param label - Package-facing prefix for the failure message.\n * @returns `Success` with a connection this package owns, or `Failure` if the\n * driver could not be loaded or the file could not be opened.\n * @internal\n */\nexport async function openOwnedConnection(\n path: string,\n label: string\n): Promise<Result<BetterSqlite3.Database>> {\n return (await captureAsyncResult(async () => (await import('better-sqlite3')).default))\n .withErrorFormat((m) => `${label}: failed to load the 'better-sqlite3' driver: ${m}`)\n .onSuccess((driver) =>\n captureResult(() => new driver(path)).withErrorFormat((m) => `${label}: failed to open '${path}': ${m}`)\n );\n}\n\n/**\n * Closes a connection this package opened.\n *\n * @remarks\n * Only ever called on a connection produced by {@link openOwnedConnection}. A\n * connection supplied through a `create()` param is the consumer's and is never\n * closed here — that asymmetry is the reason `close` travels on the handle\n * returned by `open` rather than sitting on the index class.\n *\n * `better-sqlite3`'s own `close()` is safe to call on an already-closed\n * connection, so a second `close()` on the same handle succeeds rather than\n * failing.\n *\n * @internal\n */\nexport function closeOwnedConnection(db: BetterSqlite3.Database, label: string): Result<true> {\n return captureResult<true>(() => {\n db.close();\n return true;\n }).withErrorFormat((m) => `${label}: failed to close the connection: ${m}`);\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/model.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport type BetterSqlite3 from 'better-sqlite3';\n\n/**\n * Parameters for {@link SqliteVecVectorIndex.create}.\n * @public\n */\nexport interface ISqliteVecVectorIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * the boundary-package convention). The consumer opens it (`new Database(path)`\n * for a persistent file, or `new Database(':memory:')` for an ephemeral index)\n * and owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the vector table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the embeddings. Must be a simple\n * SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to `'memory_vectors'`.\n * Supply a distinct name to hold more than one independent index in a single\n * database file.\n */\n readonly tableName?: string;\n}\n\n/**\n * Parameters for {@link SqliteVecFragmentIndex.create}.\n * @public\n */\nexport interface ISqliteVecFragmentIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * {@link ISqliteVecVectorIndexCreateParams.database}). The consumer opens it and\n * owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the fragment table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the fragment embeddings. Must be a\n * simple SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to\n * `'memory_fragments'`. Supply a distinct name (distinct from any record-level\n * {@link SqliteVecVectorIndex} table) to hold more than one independent index in\n * a single database file.\n */\n readonly tableName?: string;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"model.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/model.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport type BetterSqlite3 from 'better-sqlite3';\nimport type { Result } from '@fgv/ts-utils';\nimport type { SqliteVecFragmentIndex } from './sqliteVecFragmentIndex';\nimport type { SqliteVecVectorIndex } from './sqliteVecVectorIndex';\n\n/**\n * Parameters for {@link SqliteVecVectorIndex.create}.\n * @public\n */\nexport interface ISqliteVecVectorIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * the boundary-package convention). The consumer opens it (`new Database(path)`\n * for a persistent file, or `new Database(':memory:')` for an ephemeral index)\n * and owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the vector table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the embeddings. Must be a simple\n * SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to `'memory_vectors'`.\n * Supply a distinct name to hold more than one independent index in a single\n * database file.\n */\n readonly tableName?: string;\n}\n\n/**\n * Parameters for {@link SqliteVecVectorIndex.open}.\n * @public\n */\nexport interface ISqliteVecVectorIndexOpenParams {\n /**\n * Filesystem path to the database file, opened by this package rather than by\n * the consumer. Created if it does not exist, exactly as `better-sqlite3` would.\n * `':memory:'` is accepted and yields an owned ephemeral connection.\n *\n * **Two `open` calls on one path produce two independent connections, not a\n * shared one.** That is legal in SQLite and has a different locking story than\n * the single-connection case — writes contend, and a reader can see a\n * `SQLITE_BUSY`. To put a record index and a fragment index on one connection\n * (the intended shared-handle case), open the connection yourself and pass it to\n * both `create` methods.\n */\n readonly path: string;\n\n /**\n * Name of the `vec0` virtual table that holds the embeddings. Must be a simple\n * SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to `'memory_vectors'`.\n * Supply a distinct name to hold more than one independent index in a single\n * database file.\n */\n readonly tableName?: string;\n}\n\n/**\n * An index plus the connection {@link SqliteVecVectorIndex.open} opened for it.\n *\n * @public\n */\nexport interface ISqliteVecVectorIndexHandle {\n /** The index, ready to use. */\n readonly index: SqliteVecVectorIndex;\n\n /**\n * Closes the connection **this `open` call created**. Idempotent — a second\n * `close` succeeds rather than failing.\n *\n * @remarks\n * The disposer travels on this handle rather than on the index class because an\n * index built by `create` holds a connection the **consumer** owns, and must stay\n * incapable of closing it. A `close()` method meaningful on some instances and\n * forbidden on others would be a lie in the type; here, only the caller that\n * caused the connection to exist is handed the means to end it.\n *\n * The index is unusable afterwards — every operation on it will fail against a\n * closed connection.\n */\n close(): Result<true>;\n}\n\n/**\n * Parameters for {@link SqliteVecFragmentIndex.open}.\n * @public\n */\nexport interface ISqliteVecFragmentIndexOpenParams {\n /**\n * Filesystem path to the database file, opened by this package rather than by\n * the consumer. Created if it does not exist, exactly as `better-sqlite3` would.\n * `':memory:'` is accepted and yields an owned ephemeral connection.\n *\n * **Two `open` calls on one path produce two independent connections, not a\n * shared one** — see {@link ISqliteVecVectorIndexOpenParams.path}. To put a\n * fragment index and a record index on one connection, open it yourself and pass\n * it to both `create` methods.\n */\n readonly path: string;\n\n /**\n * Name of the `vec0` virtual table that holds the fragment embeddings. Must be a\n * simple SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to\n * `'memory_fragments'`.\n */\n readonly tableName?: string;\n}\n\n/**\n * An index plus the connection {@link SqliteVecFragmentIndex.open} opened for it.\n *\n * @public\n */\nexport interface ISqliteVecFragmentIndexHandle {\n /** The index, ready to use. */\n readonly index: SqliteVecFragmentIndex;\n\n /**\n * Closes the connection **this `open` call created**. Idempotent — a second\n * `close` succeeds rather than failing. See\n * {@link ISqliteVecVectorIndexHandle.close} for why the disposer lives here\n * rather than on the index class.\n */\n close(): Result<true>;\n}\n\n/**\n * Parameters for {@link SqliteVecFragmentIndex.create}.\n * @public\n */\nexport interface ISqliteVecFragmentIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * {@link ISqliteVecVectorIndexCreateParams.database}). The consumer opens it and\n * owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the fragment table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the fragment embeddings. Must be a\n * simple SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to\n * `'memory_fragments'`. Supply a distinct name (distinct from any record-level\n * {@link SqliteVecVectorIndex} table) to hold more than one independent index in\n * a single database file.\n */\n readonly tableName?: string;\n}\n"]}
|
|
@@ -6,8 +6,11 @@ import { load as loadSqliteVec } from 'sqlite-vec';
|
|
|
6
6
|
import { captureResult, fail, failWithDetail, succeed, succeedWithDetail } from '@fgv/ts-utils';
|
|
7
7
|
import { edgeTargetKey } from '@fgv/ts-agent-memory';
|
|
8
8
|
import { invokeHook, tally, withRollbackNote } from './rebuildHelpers';
|
|
9
|
+
import { closeOwnedConnection, openOwnedConnection } from './connection';
|
|
9
10
|
/** Default name for the fragment `vec0` virtual table. */
|
|
10
11
|
const DEFAULT_TABLE_NAME = 'memory_fragments';
|
|
12
|
+
/** Package-facing prefix for this class's failure messages. */
|
|
13
|
+
const LABEL = 'sqlite-vec fragment index';
|
|
11
14
|
/** A simple SQL identifier — the only shape allowed for the table name (it is interpolated into DDL). */
|
|
12
15
|
const IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
13
16
|
/**
|
|
@@ -62,9 +65,13 @@ const AUXILIARY_COLUMN_RE = /\+\s*([A-Za-z_][A-Za-z0-9_]*)/g;
|
|
|
62
65
|
* cosine (`score = 1 - cosineDistance`), byte-identical to the in-memory index.
|
|
63
66
|
* Large-N ANN indexing is explicitly out of scope, same regime as the record index.
|
|
64
67
|
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
+
* **Connection ownership depends on which factory you use.** With
|
|
69
|
+
* {@link SqliteVecFragmentIndex.create} the `Database` is consumer-owned
|
|
70
|
+
* (bring-your-own): this index loads the `sqlite-vec` extension onto it and
|
|
71
|
+
* reads/writes the table, but never opens or closes the connection — and that is
|
|
72
|
+
* the seam for backing this index and a record index with one connection. With
|
|
73
|
+
* {@link SqliteVecFragmentIndex.open} this package opens the file itself and hands
|
|
74
|
+
* back a handle carrying the disposer for the connection it created.
|
|
68
75
|
* @public
|
|
69
76
|
*/
|
|
70
77
|
export class SqliteVecFragmentIndex {
|
|
@@ -72,10 +79,22 @@ export class SqliteVecFragmentIndex {
|
|
|
72
79
|
this._db = db;
|
|
73
80
|
this._table = table;
|
|
74
81
|
this._dimension = dimension;
|
|
82
|
+
this._released = false;
|
|
75
83
|
this._stmts = dimension === undefined ? undefined : this._prepare();
|
|
76
84
|
}
|
|
77
|
-
/**
|
|
85
|
+
/**
|
|
86
|
+
* The number of records that currently have at least one stored fragment. Zero
|
|
87
|
+
* before the first add.
|
|
88
|
+
*
|
|
89
|
+
* @remarks
|
|
90
|
+
* **Throws on a released index**, where every other member returns a `Failure` —
|
|
91
|
+
* `IFragmentVectorIndex` declares this a synchronous `number`, so there is no
|
|
92
|
+
* `Result` to fail into, and answering `0` would be a confident lie
|
|
93
|
+
* indistinguishable from an empty index. Same reasoning as
|
|
94
|
+
* {@link SqliteVecFragmentIndex.fragmentCount} and `SqliteVecVectorIndex.size`.
|
|
95
|
+
*/
|
|
78
96
|
get recordCount() {
|
|
97
|
+
this._assertUsable('read recordCount');
|
|
79
98
|
if (this._stmts === undefined) {
|
|
80
99
|
return 0;
|
|
81
100
|
}
|
|
@@ -83,8 +102,13 @@ export class SqliteVecFragmentIndex {
|
|
|
83
102
|
// safe-integer mode (which returns `count(*)` as a `bigint`).
|
|
84
103
|
return Number(this._stmts.recordCount.get().c);
|
|
85
104
|
}
|
|
86
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* The total number of fragments currently held across all records. Zero before
|
|
107
|
+
* the first add. **Throws on a released index** — see
|
|
108
|
+
* {@link SqliteVecFragmentIndex.recordCount}.
|
|
109
|
+
*/
|
|
87
110
|
get fragmentCount() {
|
|
111
|
+
this._assertUsable('read fragmentCount');
|
|
88
112
|
if (this._stmts === undefined) {
|
|
89
113
|
return 0;
|
|
90
114
|
}
|
|
@@ -115,9 +139,97 @@ export class SqliteVecFragmentIndex {
|
|
|
115
139
|
return new SqliteVecFragmentIndex(params.database, table, dimension);
|
|
116
140
|
}).withErrorFormat((e) => `sqlite-vec fragment index: failed to initialize: ${e}`));
|
|
117
141
|
}
|
|
142
|
+
/**
|
|
143
|
+
* Path-based factory. Opens the database file itself and returns the index
|
|
144
|
+
* together with a disposer for the connection it created.
|
|
145
|
+
*
|
|
146
|
+
* @remarks
|
|
147
|
+
* The fragment-granular sibling of {@link SqliteVecVectorIndex.open}, and present
|
|
148
|
+
* for the same reason: a consumer doing sub-document retrieval only would
|
|
149
|
+
* otherwise still value-import `better-sqlite3` and hand-roll a `captureResult`
|
|
150
|
+
* around a constructor that throws.
|
|
151
|
+
*
|
|
152
|
+
* **Use `create` instead when one connection must back both a fragment index and
|
|
153
|
+
* a record index** — the intended shared-handle case. Two `open` calls on one path
|
|
154
|
+
* give two independent connections, not a shared one.
|
|
155
|
+
*
|
|
156
|
+
* If initialization fails after the file is opened, the connection is closed
|
|
157
|
+
* before returning, so a failed `open` does not leak the descriptor it created.
|
|
158
|
+
* Should that close *itself* fail — the connection is then genuinely leaked — the
|
|
159
|
+
* returned message says so rather than hiding it. That includes the
|
|
160
|
+
* auxiliary-column mismatch failure, which is reported by `create` only after the
|
|
161
|
+
* file is open.
|
|
162
|
+
*
|
|
163
|
+
* @param params - See {@link ISqliteVecFragmentIndexOpenParams}.
|
|
164
|
+
* @returns `Success` with a {@link ISqliteVecFragmentIndexHandle}, or `Failure` if
|
|
165
|
+
* the driver could not be loaded, the file could not be opened, the table name is
|
|
166
|
+
* not a simple identifier, the extension fails to load, or the existing table was
|
|
167
|
+
* written by a version with a different auxiliary-column set.
|
|
168
|
+
*/
|
|
169
|
+
static async open(params) {
|
|
170
|
+
return (await openOwnedConnection(params.path, LABEL)).thenOnSuccess(async (database) => (await SqliteVecFragmentIndex.create({ database, tableName: params.tableName }))
|
|
171
|
+
.onFailure((message) =>
|
|
172
|
+
// This call opened the connection, so a failure to initialize on top of it
|
|
173
|
+
// must not leave the file handle behind. A close that ALSO fails is said out
|
|
174
|
+
// loud rather than swallowed — the same reasoning, and the same helper, as
|
|
175
|
+
// `withRollbackNote`: silently discarding it would make the "a failed open
|
|
176
|
+
// leaks nothing" guarantee untrue exactly when it stopped holding, with no
|
|
177
|
+
// way for a caller to detect it.
|
|
178
|
+
fail(withRollbackNote(message, closeOwnedConnection(database, LABEL))))
|
|
179
|
+
.onSuccess((index) => succeed({
|
|
180
|
+
index,
|
|
181
|
+
close: () => {
|
|
182
|
+
// Drop the statements BEFORE closing, so there is never a moment where
|
|
183
|
+
// a closed connection has live `Statement` objects pointing at it —
|
|
184
|
+
// see `release`.
|
|
185
|
+
index.release();
|
|
186
|
+
return closeOwnedConnection(database, LABEL);
|
|
187
|
+
}
|
|
188
|
+
})));
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Drops this index's prepared statements and marks it unusable. Does **not**
|
|
192
|
+
* touch the connection.
|
|
193
|
+
*
|
|
194
|
+
* @remarks
|
|
195
|
+
* The fragment-lane counterpart of `SqliteVecVectorIndex.release`, and it
|
|
196
|
+
* matters here for the same reason plus one more: a shared-connection
|
|
197
|
+
* deployment — the case `create({ database })` exists for — holds a record index
|
|
198
|
+
* *and* a fragment index over one connection, so it carries two instances of the
|
|
199
|
+
* statement-lifetime shape rather than one. Both must be released.
|
|
200
|
+
*
|
|
201
|
+
* `better-sqlite3` exposes no public `finalize()`, so dropping the last
|
|
202
|
+
* reference does not finalize a statement — it makes it collectable *earlier*,
|
|
203
|
+
* while the environment is alive, rather than surviving to process teardown.
|
|
204
|
+
* That narrows the window in which `Statement::~Statement()` runs against a
|
|
205
|
+
* torn-down environment; it is not a proof against it.
|
|
206
|
+
*
|
|
207
|
+
* **Call this before closing a connection you own.**
|
|
208
|
+
* {@link SqliteVecFragmentIndex.open}'s handle does it for you.
|
|
209
|
+
*
|
|
210
|
+
* Idempotent. After it, every member fails (or, for the two counts, throws)
|
|
211
|
+
* rather than answering.
|
|
212
|
+
*/
|
|
213
|
+
release() {
|
|
214
|
+
this._released = true;
|
|
215
|
+
this._stmts = undefined;
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Throw if this index has been released. The members that call it and cannot
|
|
219
|
+
* return a `Result` are the two counts; the rest convert the throw via
|
|
220
|
+
* `captureResult`.
|
|
221
|
+
*/
|
|
222
|
+
_assertUsable(what) {
|
|
223
|
+
if (this._released) {
|
|
224
|
+
throw new Error(`fragment index: cannot ${what}: the index has been released`);
|
|
225
|
+
}
|
|
226
|
+
}
|
|
118
227
|
/** {@inheritDoc IFragmentVectorIndex.addFragments} */
|
|
119
228
|
addFragments(target, fragments) {
|
|
120
229
|
const key = edgeTargetKey(target);
|
|
230
|
+
if (this._released) {
|
|
231
|
+
return Promise.resolve(fail(`fragment index: cannot add fragments for '${key}': the index has been released`));
|
|
232
|
+
}
|
|
121
233
|
// Validate every fragment before touching the database, so a bad fragment never
|
|
122
234
|
// leaves the record half-replaced or the dimension half-established (whole-record
|
|
123
235
|
// replace is all-or-nothing). The effective dimension is the established one, or —
|
|
@@ -174,6 +286,7 @@ export class SqliteVecFragmentIndex {
|
|
|
174
286
|
/** {@inheritDoc IFragmentVectorIndex.remove} */
|
|
175
287
|
remove(target) {
|
|
176
288
|
return Promise.resolve(captureResult(() => {
|
|
289
|
+
this._assertUsable(`remove '${edgeTargetKey(target)}'`);
|
|
177
290
|
// Idempotent: removing a target with no fragments (or before any add created
|
|
178
291
|
// the table) still succeeds.
|
|
179
292
|
if (this._stmts !== undefined) {
|
|
@@ -185,6 +298,7 @@ export class SqliteVecFragmentIndex {
|
|
|
185
298
|
/** {@inheritDoc IFragmentVectorIndex.has} */
|
|
186
299
|
has(target) {
|
|
187
300
|
return Promise.resolve(captureResult(() => {
|
|
301
|
+
this._assertUsable(`check '${edgeTargetKey(target)}'`);
|
|
188
302
|
// Before any add has created the table there is nothing held — a truthful
|
|
189
303
|
// `false`, matching `remove`'s idempotence and the zero counts.
|
|
190
304
|
if (this._stmts === undefined) {
|
|
@@ -270,6 +384,9 @@ export class SqliteVecFragmentIndex {
|
|
|
270
384
|
* note on `IVectorIndex.rebuild`. Tolerates a table that does not exist yet.
|
|
271
385
|
*/
|
|
272
386
|
_clear() {
|
|
387
|
+
if (this._released) {
|
|
388
|
+
return fail('fragment index: cannot clear: the index has been released');
|
|
389
|
+
}
|
|
273
390
|
if (this._stmts === undefined) {
|
|
274
391
|
return succeed(true);
|
|
275
392
|
}
|
|
@@ -278,7 +395,13 @@ export class SqliteVecFragmentIndex {
|
|
|
278
395
|
return captureResult(() => this._db.prepare(`DELETE FROM "${this._table}"`).run()).onSuccess(() => succeed(true));
|
|
279
396
|
}
|
|
280
397
|
/** {@inheritDoc IFragmentVectorIndex.query} */
|
|
281
|
-
query(vector, topK,
|
|
398
|
+
query(vector, topK, options) {
|
|
399
|
+
const maxPerRecord = options === null || options === void 0 ? void 0 : options.maxPerRecord;
|
|
400
|
+
const scope = options === null || options === void 0 ? void 0 : options.scope;
|
|
401
|
+
const id = options === null || options === void 0 ? void 0 : options.id;
|
|
402
|
+
if (this._released) {
|
|
403
|
+
return Promise.resolve(fail('fragment index: cannot query: the index has been released'));
|
|
404
|
+
}
|
|
282
405
|
if (topK <= 0 || this._stmts === undefined) {
|
|
283
406
|
return Promise.resolve(succeed([]));
|
|
284
407
|
}
|
|
@@ -292,11 +415,33 @@ export class SqliteVecFragmentIndex {
|
|
|
292
415
|
// capped record's later fragments are skipped), so fetch the full ranked set
|
|
293
416
|
// and apply the cap + topK cut here — exactly as the in-memory index does.
|
|
294
417
|
// Uncapped, KNN's own `k = topK` is already the answer.
|
|
295
|
-
|
|
418
|
+
// A scope-only narrowing (a versioned kind's per-entity subtree) spans several
|
|
419
|
+
// records, and `target_key` equality cannot express a prefix, so it is applied
|
|
420
|
+
// over the full ranked set below. Correct either way — the caller's `topK` is
|
|
421
|
+
// applied to the NARROWED set, which is the property that matters — but only
|
|
422
|
+
// the single-record case gets the partition push-down.
|
|
423
|
+
const recordKey = scope !== undefined && id !== undefined ? edgeTargetKey({ scope, id }) : undefined;
|
|
424
|
+
// The cap forces the full ranked set ONLY when other records can fill from
|
|
425
|
+
// behind a capped one. Under a single-record narrowing every row belongs to
|
|
426
|
+
// that record, so the result is exactly `min(topK, maxPerRecord, fragments)`
|
|
427
|
+
// and those are the first rows KNN returns — `k = topK` suffices, and
|
|
428
|
+
// expanding to the table-wide `fragmentCount` would ask an
|
|
429
|
+
// already-partition-restricted query for far more rows than it can use.
|
|
430
|
+
const wholeSet = recordKey === undefined && (maxPerRecord !== undefined || scope !== undefined);
|
|
431
|
+
const fetchK = wholeSet
|
|
432
|
+
? Number(stmts.fragmentCount.get().c)
|
|
433
|
+
: topK;
|
|
296
434
|
if (fetchK <= 0) {
|
|
297
435
|
return [];
|
|
298
436
|
}
|
|
299
|
-
const
|
|
437
|
+
const blob = SqliteVecFragmentIndex._toBlob(vector);
|
|
438
|
+
const rows = (recordKey !== undefined
|
|
439
|
+
? stmts.queryScopedToRecord.all(blob, fetchK, recordKey)
|
|
440
|
+
: stmts.query.all(blob, fetchK));
|
|
441
|
+
// The scope prefix every record in `scope` shares. `edgeTargetKey` joins with
|
|
442
|
+
// a NUL, so this cannot collide with a longer scope that merely starts the
|
|
443
|
+
// same way.
|
|
444
|
+
const scopePrefix = scope !== undefined && recordKey === undefined ? `${scope}\0` : undefined;
|
|
300
445
|
// sqlite-vec returns rows ascending by distance (nearest first); score is
|
|
301
446
|
// `1 - cosineDistance`, so this order is already descending score.
|
|
302
447
|
const hits = [];
|
|
@@ -305,6 +450,9 @@ export class SqliteVecFragmentIndex {
|
|
|
305
450
|
if (hits.length >= topK) {
|
|
306
451
|
break;
|
|
307
452
|
}
|
|
453
|
+
if (scopePrefix !== undefined && !row.target_key.startsWith(scopePrefix)) {
|
|
454
|
+
continue;
|
|
455
|
+
}
|
|
308
456
|
if (maxPerRecord !== undefined) {
|
|
309
457
|
const used = (_a = perRecord.get(row.target_key)) !== null && _a !== void 0 ? _a : 0;
|
|
310
458
|
if (used >= maxPerRecord) {
|
|
@@ -355,6 +503,12 @@ export class SqliteVecFragmentIndex {
|
|
|
355
503
|
},
|
|
356
504
|
query: this._db.prepare(`SELECT target_key, start_off, end_off, fragment_id, distance FROM "${this._table}" ` +
|
|
357
505
|
`WHERE embedding MATCH ? AND k = ?`),
|
|
506
|
+
// The single-record narrowing constrains `target_key`, which is the table's
|
|
507
|
+
// PARTITION KEY — so this is a partition-restricted KNN rather than a scan
|
|
508
|
+
// plus a filter. That is the performance reason this narrowing belongs in the
|
|
509
|
+
// library instead of in a bigger over-fetch on the caller's side.
|
|
510
|
+
queryScopedToRecord: this._db.prepare(`SELECT target_key, start_off, end_off, fragment_id, distance FROM "${this._table}" ` +
|
|
511
|
+
`WHERE embedding MATCH ? AND k = ? AND target_key = ?`),
|
|
358
512
|
fragmentCount: this._db.prepare(`SELECT count(*) AS c FROM "${this._table}"`),
|
|
359
513
|
recordCount: this._db.prepare(`SELECT count(DISTINCT target_key) AS c FROM "${this._table}"`),
|
|
360
514
|
// `LIMIT 1`: membership needs existence, not cardinality.
|