@fgv/ts-agent-memory-sqlite-vec 5.1.0-50 → 5.1.0-52

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. package/README.md +49 -1
  2. package/dist/packlets/sqlite-vec-index/connection.js +54 -0
  3. package/dist/packlets/sqlite-vec-index/connection.js.map +1 -0
  4. package/dist/packlets/sqlite-vec-index/model.js.map +1 -1
  5. package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +162 -8
  6. package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -1
  7. package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +115 -4
  8. package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -1
  9. package/dist/ts-agent-memory-sqlite-vec.d.ts +258 -10
  10. package/lib/packlets/sqlite-vec-index/connection.d.ts +42 -0
  11. package/lib/packlets/sqlite-vec-index/connection.d.ts.map +1 -0
  12. package/lib/packlets/sqlite-vec-index/connection.js +91 -0
  13. package/lib/packlets/sqlite-vec-index/connection.js.map +1 -0
  14. package/lib/packlets/sqlite-vec-index/model.d.ts +92 -0
  15. package/lib/packlets/sqlite-vec-index/model.d.ts.map +1 -1
  16. package/lib/packlets/sqlite-vec-index/model.js.map +1 -1
  17. package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts +89 -8
  18. package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts.map +1 -1
  19. package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +162 -8
  20. package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -1
  21. package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts +78 -5
  22. package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts.map +1 -1
  23. package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +115 -4
  24. package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -1
  25. package/package.json +7 -7
package/README.md CHANGED
@@ -25,6 +25,31 @@ rush add -p @fgv/ts-agent-memory-sqlite-vec # or npm/pnpm add
25
25
 
26
26
  ## Quick start
27
27
 
28
+ Two factories, differing only in **who owns the connection**. If this index is the only
29
+ thing on the file, `open` is the shorter path and needs no `better-sqlite3` import of
30
+ your own:
31
+
32
+ ```ts
33
+ import { SqliteVecVectorIndex } from '@fgv/ts-agent-memory-sqlite-vec';
34
+ import { FileTreeMemoryStore } from '@fgv/ts-agent-memory';
35
+
36
+ // We open the file and hand back a disposer for the connection we created.
37
+ const handle = (await SqliteVecVectorIndex.open({ path: '/path/to/vault/vectors.db' })).orThrow();
38
+
39
+ const store = (
40
+ await FileTreeMemoryStore.create({ root, registry, vectorIndex: handle.index, embed })
41
+ ).orThrow();
42
+
43
+ // ...use the store; embeddings are written to vectors.db on every put.
44
+ handle.close(); // closes the connection THIS open() created; idempotent.
45
+ ```
46
+
47
+ `SqliteVecFragmentIndex.open({ path })` is the identical shape for the fragment lane.
48
+
49
+ **Two `open` calls on one path give two independent connections, not a shared one.** To
50
+ back a record index *and* a fragment index with a single connection, own it yourself and
51
+ pass it to both `create` methods:
52
+
28
53
  ```ts
29
54
  import Database from 'better-sqlite3';
30
55
  import { SqliteVecVectorIndex } from '@fgv/ts-agent-memory-sqlite-vec';
@@ -42,9 +67,32 @@ const store = (
42
67
  ).orThrow();
43
68
 
44
69
  // ...use the store; embeddings are written to vectors.db on every put.
70
+ vectorIndex.release(); // drop this index's prepared statements — see below
45
71
  db.close(); // you own the lifecycle — this index never closes your connection.
46
72
  ```
47
73
 
74
+ ### Releasing an index over a connection you own
75
+
76
+ An index caches prepared `Statement` objects. Those hold a reference to the
77
+ connection, so if you close a connection you own while an index over it is still
78
+ reachable, its statements outlive the connection and their native destructors run
79
+ whenever GC reaches them — potentially during process teardown.
80
+
81
+ **`release()` drops those statements and marks the index unusable. It never touches
82
+ the connection**, which is why it is safe to expose on a `create()`-made index that
83
+ does not own one. `open()`'s handle calls it for you before closing; with `create()`
84
+ you own the ordering, and it is `release()` then `close()`.
85
+
86
+ A released index **fails** (or, for the synchronous counts `size` / `recordCount` /
87
+ `fragmentCount`, **throws**) rather than answering. That is deliberate: an index that
88
+ has simply never had an `add` also holds no statements, and answering `0` from a
89
+ released one would be indistinguishable from an empty one.
90
+
91
+ Note the limit honestly: `better-sqlite3` exposes no public `finalize()`, so dropping
92
+ the last reference does not finalize a statement — it makes it collectable *earlier*,
93
+ while the environment is alive, instead of surviving to teardown. That narrows the
94
+ window; it is not a proof against it.
95
+
48
96
  `SqliteVecVectorIndex` implements the full `IVectorIndex` contract — `add(target, vector)`, `remove(target)`, `query(vector, topK)` — with the **same semantics as `InMemoryCosineIndex`**:
49
97
 
50
98
  - Keyed by the canonical `edgeTargetKey` (`(scope, id)`), so records that share a filename stem across scopes never collide.
@@ -108,7 +156,7 @@ Known instance: the release that added `IEmbeddedFragment.fragmentId` added a `+
108
156
  Deliberately excluded — reach for the upstream libraries (or a different backend) directly if you need these:
109
157
 
110
158
  - **ANN / large-N indexing.** Query is a brute-force `vec0` KNN scan — correct and durable for the same "thousands of records" regime the in-memory index targets. An approximate-nearest-neighbor structure for very large N is a different backend behind the same `IVectorIndex` / `IFragmentVectorIndex` seam. (This applies to both indexes, including `SqliteVecFragmentIndex`, whose capped query fetches the full ranked set.)
111
- - **Connection lifecycle.** You open and close the `better-sqlite3` `Database`; this index never does. Pooling, WAL/pragma tuning, backups, and multi-process coordination are yours.
159
+ - **Connection lifecycle beyond plain open/close.** With `create({ database })` you open and close the `better-sqlite3` `Database` and this index never does; with `open({ path })` this package opens the file and the returned handle's `close()` disposes of exactly what it opened. Either way, pooling, WAL/pragma tuning, backups, and multi-process coordination are yours.
112
160
  - **Embedding.** This is a vector *index*, not an embedder — the store's consumer-wired `MemoryEmbedder` produces the vectors (`@fgv/ts-extras/ai-assist` `callProviderEmbedding`, `@fgv/ts-extras-transformers`, etc.).
113
161
  - **A browser sibling.** `better-sqlite3` is Node-only. A WASM-SQLite browser variant, if ever needed, is a separate package.
114
162
  - **Schema migration of any kind.** Re-embedding with a different-dimension model against an existing table fails loudly, and a package release that changes a `vec0` table's columns requires a drop-and-re-index — see [Upgrading](#upgrading-vec0-schema-changes-require-a-drop-and-re-index). Drop the table (or use a new `tableName`) to re-index; `vec0` cannot be altered in place.
@@ -0,0 +1,54 @@
1
+ /*
2
+ * Copyright (c) 2026 Erik Fortune
3
+ * SPDX-License-Identifier: MIT
4
+ */
5
+ import { captureAsyncResult, captureResult } from '@fgv/ts-utils';
6
+ /**
7
+ * Opens a `better-sqlite3` connection this package owns.
8
+ *
9
+ * @remarks
10
+ * **The value import of `better-sqlite3` lives here, and it is deliberately
11
+ * lazy.** Every other module in this package imports the driver as `import type`
12
+ * only, so merely importing `@fgv/ts-agent-memory-sqlite-vec` has never loaded the
13
+ * native binding — a property the path-based factories must not cost consumers who
14
+ * only ever call `create()`. A static import would move the native load (and its
15
+ * failure mode) to package-load time for everyone. The dynamic `import()` keeps it
16
+ * at the one call that actually needs a connection.
17
+ *
18
+ * Taking this import is the entire point of the path-based factories: it is the
19
+ * only place this wrapper leaked its own dependency into consumer source.
20
+ *
21
+ * @param path - Filesystem path to the database file. `':memory:'` is accepted and
22
+ * yields an owned ephemeral connection.
23
+ * @param label - Package-facing prefix for the failure message.
24
+ * @returns `Success` with a connection this package owns, or `Failure` if the
25
+ * driver could not be loaded or the file could not be opened.
26
+ * @internal
27
+ */
28
+ export async function openOwnedConnection(path, label) {
29
+ return (await captureAsyncResult(async () => (await import('better-sqlite3')).default))
30
+ .withErrorFormat((m) => `${label}: failed to load the 'better-sqlite3' driver: ${m}`)
31
+ .onSuccess((driver) => captureResult(() => new driver(path)).withErrorFormat((m) => `${label}: failed to open '${path}': ${m}`));
32
+ }
33
+ /**
34
+ * Closes a connection this package opened.
35
+ *
36
+ * @remarks
37
+ * Only ever called on a connection produced by {@link openOwnedConnection}. A
38
+ * connection supplied through a `create()` param is the consumer's and is never
39
+ * closed here — that asymmetry is the reason `close` travels on the handle
40
+ * returned by `open` rather than sitting on the index class.
41
+ *
42
+ * `better-sqlite3`'s own `close()` is safe to call on an already-closed
43
+ * connection, so a second `close()` on the same handle succeeds rather than
44
+ * failing.
45
+ *
46
+ * @internal
47
+ */
48
+ export function closeOwnedConnection(db, label) {
49
+ return captureResult(() => {
50
+ db.close();
51
+ return true;
52
+ }).withErrorFormat((m) => `${label}: failed to close the connection: ${m}`);
53
+ }
54
+ //# sourceMappingURL=connection.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"connection.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/connection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAU,kBAAkB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAE1E;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,IAAY,EACZ,KAAa;IAEb,OAAO,CAAC,MAAM,kBAAkB,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;SACpF,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,iDAAiD,CAAC,EAAE,CAAC;SACpF,SAAS,CAAC,CAAC,MAAM,EAAE,EAAE,CACpB,aAAa,CAAC,GAAG,EAAE,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,qBAAqB,IAAI,MAAM,CAAC,EAAE,CAAC,CACzG,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,oBAAoB,CAAC,EAA0B,EAAE,KAAa;IAC5E,OAAO,aAAa,CAAO,GAAG,EAAE;QAC9B,EAAE,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,qCAAqC,CAAC,EAAE,CAAC,CAAC;AAC9E,CAAC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport type BetterSqlite3 from 'better-sqlite3';\nimport { Result, captureAsyncResult, captureResult } from '@fgv/ts-utils';\n\n/**\n * Opens a `better-sqlite3` connection this package owns.\n *\n * @remarks\n * **The value import of `better-sqlite3` lives here, and it is deliberately\n * lazy.** Every other module in this package imports the driver as `import type`\n * only, so merely importing `@fgv/ts-agent-memory-sqlite-vec` has never loaded the\n * native binding — a property the path-based factories must not cost consumers who\n * only ever call `create()`. A static import would move the native load (and its\n * failure mode) to package-load time for everyone. The dynamic `import()` keeps it\n * at the one call that actually needs a connection.\n *\n * Taking this import is the entire point of the path-based factories: it is the\n * only place this wrapper leaked its own dependency into consumer source.\n *\n * @param path - Filesystem path to the database file. `':memory:'` is accepted and\n * yields an owned ephemeral connection.\n * @param label - Package-facing prefix for the failure message.\n * @returns `Success` with a connection this package owns, or `Failure` if the\n * driver could not be loaded or the file could not be opened.\n * @internal\n */\nexport async function openOwnedConnection(\n path: string,\n label: string\n): Promise<Result<BetterSqlite3.Database>> {\n return (await captureAsyncResult(async () => (await import('better-sqlite3')).default))\n .withErrorFormat((m) => `${label}: failed to load the 'better-sqlite3' driver: ${m}`)\n .onSuccess((driver) =>\n captureResult(() => new driver(path)).withErrorFormat((m) => `${label}: failed to open '${path}': ${m}`)\n );\n}\n\n/**\n * Closes a connection this package opened.\n *\n * @remarks\n * Only ever called on a connection produced by {@link openOwnedConnection}. A\n * connection supplied through a `create()` param is the consumer's and is never\n * closed here — that asymmetry is the reason `close` travels on the handle\n * returned by `open` rather than sitting on the index class.\n *\n * `better-sqlite3`'s own `close()` is safe to call on an already-closed\n * connection, so a second `close()` on the same handle succeeds rather than\n * failing.\n *\n * @internal\n */\nexport function closeOwnedConnection(db: BetterSqlite3.Database, label: string): Result<true> {\n return captureResult<true>(() => {\n db.close();\n return true;\n }).withErrorFormat((m) => `${label}: failed to close the connection: ${m}`);\n}\n"]}
@@ -1 +1 @@
1
- {"version":3,"file":"model.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/model.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport type BetterSqlite3 from 'better-sqlite3';\n\n/**\n * Parameters for {@link SqliteVecVectorIndex.create}.\n * @public\n */\nexport interface ISqliteVecVectorIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * the boundary-package convention). The consumer opens it (`new Database(path)`\n * for a persistent file, or `new Database(':memory:')` for an ephemeral index)\n * and owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the vector table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the embeddings. Must be a simple\n * SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to `'memory_vectors'`.\n * Supply a distinct name to hold more than one independent index in a single\n * database file.\n */\n readonly tableName?: string;\n}\n\n/**\n * Parameters for {@link SqliteVecFragmentIndex.create}.\n * @public\n */\nexport interface ISqliteVecFragmentIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * {@link ISqliteVecVectorIndexCreateParams.database}). The consumer opens it and\n * owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the fragment table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the fragment embeddings. Must be a\n * simple SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to\n * `'memory_fragments'`. Supply a distinct name (distinct from any record-level\n * {@link SqliteVecVectorIndex} table) to hold more than one independent index in\n * a single database file.\n */\n readonly tableName?: string;\n}\n"]}
1
+ {"version":3,"file":"model.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/model.ts"],"names":[],"mappings":"AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport type BetterSqlite3 from 'better-sqlite3';\nimport type { Result } from '@fgv/ts-utils';\nimport type { SqliteVecFragmentIndex } from './sqliteVecFragmentIndex';\nimport type { SqliteVecVectorIndex } from './sqliteVecVectorIndex';\n\n/**\n * Parameters for {@link SqliteVecVectorIndex.create}.\n * @public\n */\nexport interface ISqliteVecVectorIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * the boundary-package convention). The consumer opens it (`new Database(path)`\n * for a persistent file, or `new Database(':memory:')` for an ephemeral index)\n * and owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the vector table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the embeddings. Must be a simple\n * SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to `'memory_vectors'`.\n * Supply a distinct name to hold more than one independent index in a single\n * database file.\n */\n readonly tableName?: string;\n}\n\n/**\n * Parameters for {@link SqliteVecVectorIndex.open}.\n * @public\n */\nexport interface ISqliteVecVectorIndexOpenParams {\n /**\n * Filesystem path to the database file, opened by this package rather than by\n * the consumer. Created if it does not exist, exactly as `better-sqlite3` would.\n * `':memory:'` is accepted and yields an owned ephemeral connection.\n *\n * **Two `open` calls on one path produce two independent connections, not a\n * shared one.** That is legal in SQLite and has a different locking story than\n * the single-connection case — writes contend, and a reader can see a\n * `SQLITE_BUSY`. To put a record index and a fragment index on one connection\n * (the intended shared-handle case), open the connection yourself and pass it to\n * both `create` methods.\n */\n readonly path: string;\n\n /**\n * Name of the `vec0` virtual table that holds the embeddings. Must be a simple\n * SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to `'memory_vectors'`.\n * Supply a distinct name to hold more than one independent index in a single\n * database file.\n */\n readonly tableName?: string;\n}\n\n/**\n * An index plus the connection {@link SqliteVecVectorIndex.open} opened for it.\n *\n * @public\n */\nexport interface ISqliteVecVectorIndexHandle {\n /** The index, ready to use. */\n readonly index: SqliteVecVectorIndex;\n\n /**\n * Closes the connection **this `open` call created**. Idempotent — a second\n * `close` succeeds rather than failing.\n *\n * @remarks\n * The disposer travels on this handle rather than on the index class because an\n * index built by `create` holds a connection the **consumer** owns, and must stay\n * incapable of closing it. A `close()` method meaningful on some instances and\n * forbidden on others would be a lie in the type; here, only the caller that\n * caused the connection to exist is handed the means to end it.\n *\n * The index is unusable afterwards — every operation on it will fail against a\n * closed connection.\n */\n close(): Result<true>;\n}\n\n/**\n * Parameters for {@link SqliteVecFragmentIndex.open}.\n * @public\n */\nexport interface ISqliteVecFragmentIndexOpenParams {\n /**\n * Filesystem path to the database file, opened by this package rather than by\n * the consumer. Created if it does not exist, exactly as `better-sqlite3` would.\n * `':memory:'` is accepted and yields an owned ephemeral connection.\n *\n * **Two `open` calls on one path produce two independent connections, not a\n * shared one** — see {@link ISqliteVecVectorIndexOpenParams.path}. To put a\n * fragment index and a record index on one connection, open it yourself and pass\n * it to both `create` methods.\n */\n readonly path: string;\n\n /**\n * Name of the `vec0` virtual table that holds the fragment embeddings. Must be a\n * simple SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to\n * `'memory_fragments'`.\n */\n readonly tableName?: string;\n}\n\n/**\n * An index plus the connection {@link SqliteVecFragmentIndex.open} opened for it.\n *\n * @public\n */\nexport interface ISqliteVecFragmentIndexHandle {\n /** The index, ready to use. */\n readonly index: SqliteVecFragmentIndex;\n\n /**\n * Closes the connection **this `open` call created**. Idempotent — a second\n * `close` succeeds rather than failing. See\n * {@link ISqliteVecVectorIndexHandle.close} for why the disposer lives here\n * rather than on the index class.\n */\n close(): Result<true>;\n}\n\n/**\n * Parameters for {@link SqliteVecFragmentIndex.create}.\n * @public\n */\nexport interface ISqliteVecFragmentIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * {@link ISqliteVecVectorIndexCreateParams.database}). The consumer opens it and\n * owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the fragment table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the fragment embeddings. Must be a\n * simple SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to\n * `'memory_fragments'`. Supply a distinct name (distinct from any record-level\n * {@link SqliteVecVectorIndex} table) to hold more than one independent index in\n * a single database file.\n */\n readonly tableName?: string;\n}\n"]}
@@ -6,8 +6,11 @@ import { load as loadSqliteVec } from 'sqlite-vec';
6
6
  import { captureResult, fail, failWithDetail, succeed, succeedWithDetail } from '@fgv/ts-utils';
7
7
  import { edgeTargetKey } from '@fgv/ts-agent-memory';
8
8
  import { invokeHook, tally, withRollbackNote } from './rebuildHelpers';
9
+ import { closeOwnedConnection, openOwnedConnection } from './connection';
9
10
  /** Default name for the fragment `vec0` virtual table. */
10
11
  const DEFAULT_TABLE_NAME = 'memory_fragments';
12
+ /** Package-facing prefix for this class's failure messages. */
13
+ const LABEL = 'sqlite-vec fragment index';
11
14
  /** A simple SQL identifier — the only shape allowed for the table name (it is interpolated into DDL). */
12
15
  const IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
13
16
  /**
@@ -62,9 +65,13 @@ const AUXILIARY_COLUMN_RE = /\+\s*([A-Za-z_][A-Za-z0-9_]*)/g;
62
65
  * cosine (`score = 1 - cosineDistance`), byte-identical to the in-memory index.
63
66
  * Large-N ANN indexing is explicitly out of scope, same regime as the record index.
64
67
  *
65
- * The `better-sqlite3` `Database` is consumer-owned (bring-your-own): this index
66
- * loads the `sqlite-vec` extension onto it and reads/writes the table, but never
67
- * opens or closes the connection.
68
+ * **Connection ownership depends on which factory you use.** With
69
+ * {@link SqliteVecFragmentIndex.create} the `Database` is consumer-owned
70
+ * (bring-your-own): this index loads the `sqlite-vec` extension onto it and
71
+ * reads/writes the table, but never opens or closes the connection — and that is
72
+ * the seam for backing this index and a record index with one connection. With
73
+ * {@link SqliteVecFragmentIndex.open} this package opens the file itself and hands
74
+ * back a handle carrying the disposer for the connection it created.
68
75
  * @public
69
76
  */
70
77
  export class SqliteVecFragmentIndex {
@@ -72,10 +79,22 @@ export class SqliteVecFragmentIndex {
72
79
  this._db = db;
73
80
  this._table = table;
74
81
  this._dimension = dimension;
82
+ this._released = false;
75
83
  this._stmts = dimension === undefined ? undefined : this._prepare();
76
84
  }
77
- /** The number of records that currently have at least one stored fragment. Zero before the first add. */
85
+ /**
86
+ * The number of records that currently have at least one stored fragment. Zero
87
+ * before the first add.
88
+ *
89
+ * @remarks
90
+ * **Throws on a released index**, where every other member returns a `Failure` —
91
+ * `IFragmentVectorIndex` declares this a synchronous `number`, so there is no
92
+ * `Result` to fail into, and answering `0` would be a confident lie
93
+ * indistinguishable from an empty index. Same reasoning as
94
+ * {@link SqliteVecFragmentIndex.fragmentCount} and `SqliteVecVectorIndex.size`.
95
+ */
78
96
  get recordCount() {
97
+ this._assertUsable('read recordCount');
79
98
  if (this._stmts === undefined) {
80
99
  return 0;
81
100
  }
@@ -83,8 +102,13 @@ export class SqliteVecFragmentIndex {
83
102
  // safe-integer mode (which returns `count(*)` as a `bigint`).
84
103
  return Number(this._stmts.recordCount.get().c);
85
104
  }
86
- /** The total number of fragments currently held across all records. Zero before the first add. */
105
+ /**
106
+ * The total number of fragments currently held across all records. Zero before
107
+ * the first add. **Throws on a released index** — see
108
+ * {@link SqliteVecFragmentIndex.recordCount}.
109
+ */
87
110
  get fragmentCount() {
111
+ this._assertUsable('read fragmentCount');
88
112
  if (this._stmts === undefined) {
89
113
  return 0;
90
114
  }
@@ -115,9 +139,97 @@ export class SqliteVecFragmentIndex {
115
139
  return new SqliteVecFragmentIndex(params.database, table, dimension);
116
140
  }).withErrorFormat((e) => `sqlite-vec fragment index: failed to initialize: ${e}`));
117
141
  }
142
+ /**
143
+ * Path-based factory. Opens the database file itself and returns the index
144
+ * together with a disposer for the connection it created.
145
+ *
146
+ * @remarks
147
+ * The fragment-granular sibling of {@link SqliteVecVectorIndex.open}, and present
148
+ * for the same reason: a consumer doing sub-document retrieval only would
149
+ * otherwise still value-import `better-sqlite3` and hand-roll a `captureResult`
150
+ * around a constructor that throws.
151
+ *
152
+ * **Use `create` instead when one connection must back both a fragment index and
153
+ * a record index** — the intended shared-handle case. Two `open` calls on one path
154
+ * give two independent connections, not a shared one.
155
+ *
156
+ * If initialization fails after the file is opened, the connection is closed
157
+ * before returning, so a failed `open` does not leak the descriptor it created.
158
+ * Should that close *itself* fail — the connection is then genuinely leaked — the
159
+ * returned message says so rather than hiding it. That includes the
160
+ * auxiliary-column mismatch failure, which is reported by `create` only after the
161
+ * file is open.
162
+ *
163
+ * @param params - See {@link ISqliteVecFragmentIndexOpenParams}.
164
+ * @returns `Success` with a {@link ISqliteVecFragmentIndexHandle}, or `Failure` if
165
+ * the driver could not be loaded, the file could not be opened, the table name is
166
+ * not a simple identifier, the extension fails to load, or the existing table was
167
+ * written by a version with a different auxiliary-column set.
168
+ */
169
+ static async open(params) {
170
+ return (await openOwnedConnection(params.path, LABEL)).thenOnSuccess(async (database) => (await SqliteVecFragmentIndex.create({ database, tableName: params.tableName }))
171
+ .onFailure((message) =>
172
+ // This call opened the connection, so a failure to initialize on top of it
173
+ // must not leave the file handle behind. A close that ALSO fails is said out
174
+ // loud rather than swallowed — the same reasoning, and the same helper, as
175
+ // `withRollbackNote`: silently discarding it would make the "a failed open
176
+ // leaks nothing" guarantee untrue exactly when it stopped holding, with no
177
+ // way for a caller to detect it.
178
+ fail(withRollbackNote(message, closeOwnedConnection(database, LABEL))))
179
+ .onSuccess((index) => succeed({
180
+ index,
181
+ close: () => {
182
+ // Drop the statements BEFORE closing, so there is never a moment where
183
+ // a closed connection has live `Statement` objects pointing at it —
184
+ // see `release`.
185
+ index.release();
186
+ return closeOwnedConnection(database, LABEL);
187
+ }
188
+ })));
189
+ }
190
+ /**
191
+ * Drops this index's prepared statements and marks it unusable. Does **not**
192
+ * touch the connection.
193
+ *
194
+ * @remarks
195
+ * The fragment-lane counterpart of `SqliteVecVectorIndex.release`, and it
196
+ * matters here for the same reason plus one more: a shared-connection
197
+ * deployment — the case `create({ database })` exists for — holds a record index
198
+ * *and* a fragment index over one connection, so it carries two instances of the
199
+ * statement-lifetime shape rather than one. Both must be released.
200
+ *
201
+ * `better-sqlite3` exposes no public `finalize()`, so dropping the last
202
+ * reference does not finalize a statement — it makes it collectable *earlier*,
203
+ * while the environment is alive, rather than surviving to process teardown.
204
+ * That narrows the window in which `Statement::~Statement()` runs against a
205
+ * torn-down environment; it is not a proof against it.
206
+ *
207
+ * **Call this before closing a connection you own.**
208
+ * {@link SqliteVecFragmentIndex.open}'s handle does it for you.
209
+ *
210
+ * Idempotent. After it, every member fails (or, for the two counts, throws)
211
+ * rather than answering.
212
+ */
213
+ release() {
214
+ this._released = true;
215
+ this._stmts = undefined;
216
+ }
217
+ /**
218
+ * Throw if this index has been released. The members that call it and cannot
219
+ * return a `Result` are the two counts; the rest convert the throw via
220
+ * `captureResult`.
221
+ */
222
+ _assertUsable(what) {
223
+ if (this._released) {
224
+ throw new Error(`fragment index: cannot ${what}: the index has been released`);
225
+ }
226
+ }
118
227
  /** {@inheritDoc IFragmentVectorIndex.addFragments} */
119
228
  addFragments(target, fragments) {
120
229
  const key = edgeTargetKey(target);
230
+ if (this._released) {
231
+ return Promise.resolve(fail(`fragment index: cannot add fragments for '${key}': the index has been released`));
232
+ }
121
233
  // Validate every fragment before touching the database, so a bad fragment never
122
234
  // leaves the record half-replaced or the dimension half-established (whole-record
123
235
  // replace is all-or-nothing). The effective dimension is the established one, or —
@@ -174,6 +286,7 @@ export class SqliteVecFragmentIndex {
174
286
  /** {@inheritDoc IFragmentVectorIndex.remove} */
175
287
  remove(target) {
176
288
  return Promise.resolve(captureResult(() => {
289
+ this._assertUsable(`remove '${edgeTargetKey(target)}'`);
177
290
  // Idempotent: removing a target with no fragments (or before any add created
178
291
  // the table) still succeeds.
179
292
  if (this._stmts !== undefined) {
@@ -185,6 +298,7 @@ export class SqliteVecFragmentIndex {
185
298
  /** {@inheritDoc IFragmentVectorIndex.has} */
186
299
  has(target) {
187
300
  return Promise.resolve(captureResult(() => {
301
+ this._assertUsable(`check '${edgeTargetKey(target)}'`);
188
302
  // Before any add has created the table there is nothing held — a truthful
189
303
  // `false`, matching `remove`'s idempotence and the zero counts.
190
304
  if (this._stmts === undefined) {
@@ -270,6 +384,9 @@ export class SqliteVecFragmentIndex {
270
384
  * note on `IVectorIndex.rebuild`. Tolerates a table that does not exist yet.
271
385
  */
272
386
  _clear() {
387
+ if (this._released) {
388
+ return fail('fragment index: cannot clear: the index has been released');
389
+ }
273
390
  if (this._stmts === undefined) {
274
391
  return succeed(true);
275
392
  }
@@ -278,7 +395,13 @@ export class SqliteVecFragmentIndex {
278
395
  return captureResult(() => this._db.prepare(`DELETE FROM "${this._table}"`).run()).onSuccess(() => succeed(true));
279
396
  }
280
397
  /** {@inheritDoc IFragmentVectorIndex.query} */
281
- query(vector, topK, maxPerRecord) {
398
+ query(vector, topK, options) {
399
+ const maxPerRecord = options === null || options === void 0 ? void 0 : options.maxPerRecord;
400
+ const scope = options === null || options === void 0 ? void 0 : options.scope;
401
+ const id = options === null || options === void 0 ? void 0 : options.id;
402
+ if (this._released) {
403
+ return Promise.resolve(fail('fragment index: cannot query: the index has been released'));
404
+ }
282
405
  if (topK <= 0 || this._stmts === undefined) {
283
406
  return Promise.resolve(succeed([]));
284
407
  }
@@ -292,11 +415,33 @@ export class SqliteVecFragmentIndex {
292
415
  // capped record's later fragments are skipped), so fetch the full ranked set
293
416
  // and apply the cap + topK cut here — exactly as the in-memory index does.
294
417
  // Uncapped, KNN's own `k = topK` is already the answer.
295
- const fetchK = maxPerRecord === undefined ? topK : Number(stmts.fragmentCount.get().c);
418
+ // A scope-only narrowing (a versioned kind's per-entity subtree) spans several
419
+ // records, and `target_key` equality cannot express a prefix, so it is applied
420
+ // over the full ranked set below. Correct either way — the caller's `topK` is
421
+ // applied to the NARROWED set, which is the property that matters — but only
422
+ // the single-record case gets the partition push-down.
423
+ const recordKey = scope !== undefined && id !== undefined ? edgeTargetKey({ scope, id }) : undefined;
424
+ // The cap forces the full ranked set ONLY when other records can fill from
425
+ // behind a capped one. Under a single-record narrowing every row belongs to
426
+ // that record, so the result is exactly `min(topK, maxPerRecord, fragments)`
427
+ // and those are the first rows KNN returns — `k = topK` suffices, and
428
+ // expanding to the table-wide `fragmentCount` would ask an
429
+ // already-partition-restricted query for far more rows than it can use.
430
+ const wholeSet = recordKey === undefined && (maxPerRecord !== undefined || scope !== undefined);
431
+ const fetchK = wholeSet
432
+ ? Number(stmts.fragmentCount.get().c)
433
+ : topK;
296
434
  if (fetchK <= 0) {
297
435
  return [];
298
436
  }
299
- const rows = stmts.query.all(SqliteVecFragmentIndex._toBlob(vector), fetchK);
437
+ const blob = SqliteVecFragmentIndex._toBlob(vector);
438
+ const rows = (recordKey !== undefined
439
+ ? stmts.queryScopedToRecord.all(blob, fetchK, recordKey)
440
+ : stmts.query.all(blob, fetchK));
441
+ // The scope prefix every record in `scope` shares. `edgeTargetKey` joins with
442
+ // a NUL, so this cannot collide with a longer scope that merely starts the
443
+ // same way.
444
+ const scopePrefix = scope !== undefined && recordKey === undefined ? `${scope}\0` : undefined;
300
445
  // sqlite-vec returns rows ascending by distance (nearest first); score is
301
446
  // `1 - cosineDistance`, so this order is already descending score.
302
447
  const hits = [];
@@ -305,6 +450,9 @@ export class SqliteVecFragmentIndex {
305
450
  if (hits.length >= topK) {
306
451
  break;
307
452
  }
453
+ if (scopePrefix !== undefined && !row.target_key.startsWith(scopePrefix)) {
454
+ continue;
455
+ }
308
456
  if (maxPerRecord !== undefined) {
309
457
  const used = (_a = perRecord.get(row.target_key)) !== null && _a !== void 0 ? _a : 0;
310
458
  if (used >= maxPerRecord) {
@@ -355,6 +503,12 @@ export class SqliteVecFragmentIndex {
355
503
  },
356
504
  query: this._db.prepare(`SELECT target_key, start_off, end_off, fragment_id, distance FROM "${this._table}" ` +
357
505
  `WHERE embedding MATCH ? AND k = ?`),
506
+ // The single-record narrowing constrains `target_key`, which is the table's
507
+ // PARTITION KEY — so this is a partition-restricted KNN rather than a scan
508
+ // plus a filter. That is the performance reason this narrowing belongs in the
509
+ // library instead of in a bigger over-fetch on the caller's side.
510
+ queryScopedToRecord: this._db.prepare(`SELECT target_key, start_off, end_off, fragment_id, distance FROM "${this._table}" ` +
511
+ `WHERE embedding MATCH ? AND k = ? AND target_key = ?`),
358
512
  fragmentCount: this._db.prepare(`SELECT count(*) AS c FROM "${this._table}"`),
359
513
  recordCount: this._db.prepare(`SELECT count(DISTINCT target_key) AS c FROM "${this._table}"`),
360
514
  // `LIMIT 1`: membership needs existence, not cardinality.