@fgv/ts-agent-memory-sqlite-vec 5.1.0-49 → 5.1.0-51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -1
- package/dist/packlets/sqlite-vec-index/connection.js +54 -0
- package/dist/packlets/sqlite-vec-index/connection.js.map +1 -0
- package/dist/packlets/sqlite-vec-index/model.js.map +1 -1
- package/dist/packlets/sqlite-vec-index/rebuildHelpers.js +48 -0
- package/dist/packlets/sqlite-vec-index/rebuildHelpers.js.map +1 -0
- package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +189 -8
- package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -1
- package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +107 -50
- package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -1
- package/dist/ts-agent-memory-sqlite-vec.d.ts +188 -13
- package/lib/packlets/sqlite-vec-index/connection.d.ts +42 -0
- package/lib/packlets/sqlite-vec-index/connection.d.ts.map +1 -0
- package/lib/packlets/sqlite-vec-index/connection.js +91 -0
- package/lib/packlets/sqlite-vec-index/connection.js.map +1 -0
- package/lib/packlets/sqlite-vec-index/model.d.ts +92 -0
- package/lib/packlets/sqlite-vec-index/model.d.ts.map +1 -1
- package/lib/packlets/sqlite-vec-index/model.js.map +1 -1
- package/lib/packlets/sqlite-vec-index/rebuildHelpers.d.ts +38 -0
- package/lib/packlets/sqlite-vec-index/rebuildHelpers.d.ts.map +1 -0
- package/lib/packlets/sqlite-vec-index/rebuildHelpers.js +53 -0
- package/lib/packlets/sqlite-vec-index/rebuildHelpers.js.map +1 -0
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts +52 -7
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts.map +1 -1
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +188 -7
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -1
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts +44 -11
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts.map +1 -1
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +108 -51
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -1
- package/package.json +7 -7
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { Result } from '@fgv/ts-utils';
|
|
1
|
+
import { DetailedResult, Result } from '@fgv/ts-utils';
|
|
2
2
|
import { IEdgeTarget, IMemoryRecordSource, IVectorIndex, IVectorQueryHit, IVectorRebuildOptions, IVectorRebuildReport, MemoryEmbedder } from '@fgv/ts-agent-memory';
|
|
3
|
-
import { ISqliteVecVectorIndexCreateParams } from './model';
|
|
3
|
+
import { ISqliteVecVectorIndexCreateParams, ISqliteVecVectorIndexHandle, ISqliteVecVectorIndexOpenParams } from './model';
|
|
4
4
|
/**
|
|
5
5
|
* A persistent, `sqlite-vec`-backed `IVectorIndex` for `@fgv/ts-agent-memory`.
|
|
6
6
|
*
|
|
@@ -27,9 +27,13 @@ import { ISqliteVecVectorIndexCreateParams } from './model';
|
|
|
27
27
|
* durable, appropriate for the same "thousands of records" regime the in-memory
|
|
28
28
|
* index targets. Large-N ANN indexing is explicitly out of scope — see the README.
|
|
29
29
|
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* **Connection ownership depends on which factory you use.** With
|
|
31
|
+
* {@link SqliteVecVectorIndex.create} the `Database` is consumer-owned
|
|
32
|
+
* (bring-your-own): this index loads the `sqlite-vec` extension onto it and
|
|
33
|
+
* reads/writes the table, but never opens or closes the connection — and that is
|
|
34
|
+
* the seam for backing a record index and a fragment index with one connection.
|
|
35
|
+
* With {@link SqliteVecVectorIndex.open} this package opens the file itself and
|
|
36
|
+
* hands back a handle carrying the disposer for the connection it created.
|
|
33
37
|
* @public
|
|
34
38
|
*/
|
|
35
39
|
export declare class SqliteVecVectorIndex implements IVectorIndex {
|
|
@@ -53,8 +57,36 @@ export declare class SqliteVecVectorIndex implements IVectorIndex {
|
|
|
53
57
|
* simple identifier or the extension fails to load.
|
|
54
58
|
*/
|
|
55
59
|
static create(params: ISqliteVecVectorIndexCreateParams): Promise<Result<SqliteVecVectorIndex>>;
|
|
60
|
+
/**
|
|
61
|
+
* Path-based factory. Opens the database file itself and returns the index
|
|
62
|
+
* together with a disposer for the connection it created.
|
|
63
|
+
*
|
|
64
|
+
* @remarks
|
|
65
|
+
* The convenience over {@link SqliteVecVectorIndex.create} is that the consumer
|
|
66
|
+
* neither value-imports `better-sqlite3` nor re-establishes `Result` discipline
|
|
67
|
+
* around a constructor that throws — this is the one place the package leaked its
|
|
68
|
+
* own dependency into consumer source.
|
|
69
|
+
*
|
|
70
|
+
* **Use `create` instead when one connection must back more than one index** (a
|
|
71
|
+
* record index and a fragment index in the same file, the intended shared-handle
|
|
72
|
+
* case). Two `open` calls on one path give two independent connections, not a
|
|
73
|
+
* shared one.
|
|
74
|
+
*
|
|
75
|
+
* If initialization fails after the file is opened, the connection is closed
|
|
76
|
+
* before returning, so a failed `open` does not leak the descriptor it created.
|
|
77
|
+
* Should that close *itself* fail — the connection is then genuinely leaked — the
|
|
78
|
+
* returned message says so rather than hiding it.
|
|
79
|
+
*
|
|
80
|
+
* @param params - See {@link ISqliteVecVectorIndexOpenParams}.
|
|
81
|
+
* @returns `Success` with a {@link ISqliteVecVectorIndexHandle}, or `Failure` if
|
|
82
|
+
* the driver could not be loaded, the file could not be opened, the table name is
|
|
83
|
+
* not a simple identifier, or the extension fails to load.
|
|
84
|
+
*/
|
|
85
|
+
static open(params: ISqliteVecVectorIndexOpenParams): Promise<Result<ISqliteVecVectorIndexHandle>>;
|
|
56
86
|
/** {@inheritDoc IVectorIndex.add} */
|
|
57
87
|
add(target: IEdgeTarget, vector: Float32Array): Promise<Result<string>>;
|
|
88
|
+
/** {@inheritDoc IVectorIndex.has} */
|
|
89
|
+
has(target: IEdgeTarget): Promise<Result<boolean>>;
|
|
58
90
|
/** {@inheritDoc IVectorIndex.remove} */
|
|
59
91
|
remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
|
|
60
92
|
/**
|
|
@@ -70,13 +102,14 @@ export declare class SqliteVecVectorIndex implements IVectorIndex {
|
|
|
70
102
|
* a process kill mid-rebuild leaves the table holding neither the old index nor
|
|
71
103
|
* the complete new one, and the remedy is to run `rebuild` again.
|
|
72
104
|
*/
|
|
73
|
-
rebuild(source: IMemoryRecordSource, embed: MemoryEmbedder, options?: IVectorRebuildOptions): Promise<
|
|
105
|
+
rebuild(source: IMemoryRecordSource, embed: MemoryEmbedder, options?: IVectorRebuildOptions): Promise<DetailedResult<IVectorRebuildReport, IVectorRebuildReport>>;
|
|
74
106
|
/**
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
107
|
+
* **Empties the rows; does NOT release the table's declared dimension.** That
|
|
108
|
+
* is a `vec0` constraint rather than a choice — the dimension is schema, and
|
|
109
|
+
* there is no `ALTER TABLE` for it — so a rebuild at a new dimension fails
|
|
110
|
+
* here where it would succeed on the in-memory sibling, which forgets its
|
|
111
|
+
* dimension on reset. Changing dimension needs a drop-and-re-index; see the
|
|
112
|
+
* note on `IVectorIndex.rebuild`.
|
|
80
113
|
*/
|
|
81
114
|
private _clear;
|
|
82
115
|
/** {@inheritDoc IVectorIndex.query} */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqliteVecVectorIndex.d.ts","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/sqliteVecVectorIndex.ts"],"names":[],"mappings":"AAOA,OAAO,
|
|
1
|
+
{"version":3,"file":"sqliteVecVectorIndex.d.ts","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/sqliteVecVectorIndex.ts"],"names":[],"mappings":"AAOA,OAAO,EACL,cAAc,EACd,MAAM,EAMP,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,WAAW,EAEX,mBAAmB,EAEnB,YAAY,EACZ,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EAEpB,cAAc,EAIf,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACL,iCAAiC,EACjC,2BAA2B,EAC3B,+BAA+B,EAChC,MAAM,SAAS,CAAC;AAiBjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,qBAAa,oBAAqB,YAAW,YAAY;IACvD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAyB;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,2HAA2H;IAC3H,OAAO,CAAC,UAAU,CAAqB;IACvC,qFAAqF;IACrF,OAAO,CAAC,MAAM,CAAmC;IAEjD,OAAO;IAOP,yEAAyE;IACzE,IAAW,IAAI,IAAI,MAAM,CAYxB;IAED;;;;;;;;;OASG;WACW,MAAM,CAAC,MAAM,EAAE,iCAAiC,GAAG,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;IAiBtG;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;WACiB,IAAI,CACtB,MAAM,EAAE,+BAA+B,GACtC,OAAO,CAAC,MAAM,CAAC,2BAA2B,CAAC,CAAC;IAqB/C,qCAAqC;IAC9B,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAyB9E,qCAAqC;IAC9B,GAAG,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAczD,wCAAwC;IACjC,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAahE;;;;;;;;;;;;OAYG;IACU,OAAO,CAClB,MAAM,EAAE,mBAAmB,EAC3B,KAAK,EAAE,cAAc,EACrB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,cAAc,CAAC,oBAAoB,EAAE,oBAAoB,CAAC,CAAC;IAiEtE;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM;IAYd,uCAAuC;IAChC,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC;IA2BjG,sEAAsE;IACtE,OAAO,CAAC,YAAY;IAOpB,4EAA4E;IAC5E,OAAO,CAAC,QAAQ;IA6BhB;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAWrC,sHAAsH;IACtH,OAAO,CAAC,MAAM,CAAC,OAAO;IAItB;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,SAAS;CAOzB"}
|
|
@@ -8,36 +8,12 @@ exports.SqliteVecVectorIndex = void 0;
|
|
|
8
8
|
const sqlite_vec_1 = require("sqlite-vec");
|
|
9
9
|
const ts_utils_1 = require("@fgv/ts-utils");
|
|
10
10
|
const ts_agent_memory_1 = require("@fgv/ts-agent-memory");
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
* synchronous throw or a promise rejection into a `Failure` rather than letting
|
|
14
|
-
* it escape. `captureAsyncResult` wraps the hook's own `Result`, so the outcome
|
|
15
|
-
* is flattened back to one level.
|
|
16
|
-
*
|
|
17
|
-
* @remarks
|
|
18
|
-
* This is `@fgv/ts-utils`' own `_invokeDeferred` shape (see `mapResultsAsync`),
|
|
19
|
-
* which is `@internal` there and so cannot be imported. `@fgv/ts-agent-memory`
|
|
20
|
-
* carries an identical private copy for the in-memory index. Exporting a single
|
|
21
|
-
* `AsyncDeferredResult`-invoking primitive from `ts-utils` is the right home and
|
|
22
|
-
* is recorded in `docs/TECH_DEBT.md`; duplicating three lines twice is the
|
|
23
|
-
* cheaper thing to do from inside this stream than widening it to a foundational
|
|
24
|
-
* library.
|
|
25
|
-
*/
|
|
26
|
-
async function invokeHook(hook) {
|
|
27
|
-
return (await (0, ts_utils_1.captureAsyncResult)(hook)).onSuccess((inner) => inner);
|
|
28
|
-
}
|
|
29
|
-
/**
|
|
30
|
-
* Compose the failure that aborted a rebuild with the outcome of the rollback
|
|
31
|
-
* that followed it. A rollback that ALSO fails is worth saying out loud: the
|
|
32
|
-
* `'fail'` path promises an empty index, and a caller that retries against a
|
|
33
|
-
* table which is neither the old index nor empty is working from a state the
|
|
34
|
-
* contract never described.
|
|
35
|
-
*/
|
|
36
|
-
function withRollbackNote(error, rollback) {
|
|
37
|
-
return rollback.isFailure() ? `${error} (rollback also failed: ${rollback.message})` : error;
|
|
38
|
-
}
|
|
11
|
+
const rebuildHelpers_1 = require("./rebuildHelpers");
|
|
12
|
+
const connection_1 = require("./connection");
|
|
39
13
|
/** Default name for the `vec0` virtual table. */
|
|
40
14
|
const DEFAULT_TABLE_NAME = 'memory_vectors';
|
|
15
|
+
/** Package-facing prefix for this class's failure messages. */
|
|
16
|
+
const LABEL = 'sqlite-vec index';
|
|
41
17
|
/** A simple SQL identifier — the only shape allowed for the table name (it is interpolated into DDL). */
|
|
42
18
|
const IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
43
19
|
/**
|
|
@@ -66,9 +42,13 @@ const IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
|
66
42
|
* durable, appropriate for the same "thousands of records" regime the in-memory
|
|
67
43
|
* index targets. Large-N ANN indexing is explicitly out of scope — see the README.
|
|
68
44
|
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
45
|
+
* **Connection ownership depends on which factory you use.** With
|
|
46
|
+
* {@link SqliteVecVectorIndex.create} the `Database` is consumer-owned
|
|
47
|
+
* (bring-your-own): this index loads the `sqlite-vec` extension onto it and
|
|
48
|
+
* reads/writes the table, but never opens or closes the connection — and that is
|
|
49
|
+
* the seam for backing a record index and a fragment index with one connection.
|
|
50
|
+
* With {@link SqliteVecVectorIndex.open} this package opens the file itself and
|
|
51
|
+
* hands back a handle carrying the disposer for the connection it created.
|
|
72
52
|
* @public
|
|
73
53
|
*/
|
|
74
54
|
class SqliteVecVectorIndex {
|
|
@@ -83,7 +63,14 @@ class SqliteVecVectorIndex {
|
|
|
83
63
|
if (this._stmts === undefined) {
|
|
84
64
|
return 0;
|
|
85
65
|
}
|
|
86
|
-
|
|
66
|
+
// `Number(...)` narrows the count in case the consumer enabled better-sqlite3
|
|
67
|
+
// safe-integer mode (`db.defaultSafeIntegers(true)`), which returns `count(*)`
|
|
68
|
+
// as a `bigint`. Without it a `bigint` leaks through a `number`-typed contract
|
|
69
|
+
// member — and now through `IIndexCoverage.indexSize`, which is also declared
|
|
70
|
+
// `number`, so the coverage report would carry a value of the wrong runtime
|
|
71
|
+
// type. `SqliteVecFragmentIndex`'s two counts have always converted; this one
|
|
72
|
+
// was the outlier.
|
|
73
|
+
return Number(this._stmts.count.get().c);
|
|
87
74
|
}
|
|
88
75
|
/**
|
|
89
76
|
* Family-convention factory. Loads the `sqlite-vec` extension onto the supplied
|
|
@@ -107,6 +94,46 @@ class SqliteVecVectorIndex {
|
|
|
107
94
|
return new SqliteVecVectorIndex(params.database, table, dimension);
|
|
108
95
|
}).withErrorFormat((e) => `sqlite-vec index: failed to initialize: ${e}`));
|
|
109
96
|
}
|
|
97
|
+
/**
|
|
98
|
+
* Path-based factory. Opens the database file itself and returns the index
|
|
99
|
+
* together with a disposer for the connection it created.
|
|
100
|
+
*
|
|
101
|
+
* @remarks
|
|
102
|
+
* The convenience over {@link SqliteVecVectorIndex.create} is that the consumer
|
|
103
|
+
* neither value-imports `better-sqlite3` nor re-establishes `Result` discipline
|
|
104
|
+
* around a constructor that throws — this is the one place the package leaked its
|
|
105
|
+
* own dependency into consumer source.
|
|
106
|
+
*
|
|
107
|
+
* **Use `create` instead when one connection must back more than one index** (a
|
|
108
|
+
* record index and a fragment index in the same file, the intended shared-handle
|
|
109
|
+
* case). Two `open` calls on one path give two independent connections, not a
|
|
110
|
+
* shared one.
|
|
111
|
+
*
|
|
112
|
+
* If initialization fails after the file is opened, the connection is closed
|
|
113
|
+
* before returning, so a failed `open` does not leak the descriptor it created.
|
|
114
|
+
* Should that close *itself* fail — the connection is then genuinely leaked — the
|
|
115
|
+
* returned message says so rather than hiding it.
|
|
116
|
+
*
|
|
117
|
+
* @param params - See {@link ISqliteVecVectorIndexOpenParams}.
|
|
118
|
+
* @returns `Success` with a {@link ISqliteVecVectorIndexHandle}, or `Failure` if
|
|
119
|
+
* the driver could not be loaded, the file could not be opened, the table name is
|
|
120
|
+
* not a simple identifier, or the extension fails to load.
|
|
121
|
+
*/
|
|
122
|
+
static async open(params) {
|
|
123
|
+
return (await (0, connection_1.openOwnedConnection)(params.path, LABEL)).thenOnSuccess(async (database) => (await SqliteVecVectorIndex.create({ database, tableName: params.tableName }))
|
|
124
|
+
.onFailure((message) =>
|
|
125
|
+
// This call opened the connection, so a failure to initialize on top of it
|
|
126
|
+
// must not leave the file handle behind. A close that ALSO fails is said out
|
|
127
|
+
// loud rather than swallowed — the same reasoning, and the same helper, as
|
|
128
|
+
// `withRollbackNote`: silently discarding it would make the "a failed open
|
|
129
|
+
// leaks nothing" guarantee untrue exactly when it stopped holding, with no
|
|
130
|
+
// way for a caller to detect it.
|
|
131
|
+
(0, ts_utils_1.fail)((0, rebuildHelpers_1.withRollbackNote)(message, (0, connection_1.closeOwnedConnection)(database, LABEL))))
|
|
132
|
+
.onSuccess((index) => (0, ts_utils_1.succeed)({
|
|
133
|
+
index,
|
|
134
|
+
close: () => (0, connection_1.closeOwnedConnection)(database, LABEL)
|
|
135
|
+
})));
|
|
136
|
+
}
|
|
110
137
|
/** {@inheritDoc IVectorIndex.add} */
|
|
111
138
|
add(target, vector) {
|
|
112
139
|
const key = (0, ts_agent_memory_1.edgeTargetKey)(target);
|
|
@@ -126,6 +153,18 @@ class SqliteVecVectorIndex {
|
|
|
126
153
|
return key;
|
|
127
154
|
}).withErrorFormat((e) => `vector index: cannot add '${key}': ${e}`));
|
|
128
155
|
}
|
|
156
|
+
/** {@inheritDoc IVectorIndex.has} */
|
|
157
|
+
has(target) {
|
|
158
|
+
return Promise.resolve((0, ts_utils_1.captureResult)(() => {
|
|
159
|
+
// Before any add has created the table there is nothing held, which is a
|
|
160
|
+
// truthful `false` rather than an error — same posture as `remove`'s
|
|
161
|
+
// idempotence and `size`'s zero.
|
|
162
|
+
if (this._stmts === undefined) {
|
|
163
|
+
return false;
|
|
164
|
+
}
|
|
165
|
+
return this._stmts.has.get((0, ts_agent_memory_1.edgeTargetKey)(target)) !== undefined;
|
|
166
|
+
}).withErrorFormat((e) => `vector index: cannot check '${(0, ts_agent_memory_1.edgeTargetKey)(target)}': ${e}`));
|
|
167
|
+
}
|
|
129
168
|
/** {@inheritDoc IVectorIndex.remove} */
|
|
130
169
|
remove(target) {
|
|
131
170
|
return Promise.resolve((0, ts_utils_1.captureResult)(() => {
|
|
@@ -155,56 +194,71 @@ class SqliteVecVectorIndex {
|
|
|
155
194
|
const lenient = ((_a = options === null || options === void 0 ? void 0 : options.onRecordError) !== null && _a !== void 0 ? _a : 'fail') === 'skip';
|
|
156
195
|
// `source` is consumer-supplied, so a throw or rejection becomes a `Failure`
|
|
157
196
|
// here rather than escaping as an exception.
|
|
158
|
-
const listed = await invokeHook(() => source.list());
|
|
197
|
+
const listed = await (0, rebuildHelpers_1.invokeHook)(() => source.list());
|
|
159
198
|
if (listed.isFailure()) {
|
|
160
199
|
// Deliberately BEFORE any clear: a failed list is no evidence about the
|
|
161
200
|
// vectors already held, and no re-embedding has been attempted, so there is
|
|
162
201
|
// no half-rebuilt state to protect against. Clearing here would destroy a
|
|
163
|
-
// healthy persisted index over a transient read error.
|
|
164
|
-
|
|
202
|
+
// healthy persisted index over a transient read error. No report either, for
|
|
203
|
+
// the same reason — there is nothing this call disturbed to describe.
|
|
204
|
+
return (0, ts_utils_1.failWithDetail)(`vector index rebuild: failed to list records: ${listed.message}`);
|
|
165
205
|
}
|
|
166
206
|
const cleared = this._clear();
|
|
167
207
|
if (cleared.isFailure()) {
|
|
168
|
-
|
|
208
|
+
// Also nothing established: the table still holds whatever it held.
|
|
209
|
+
return (0, ts_utils_1.failWithDetail)(`vector index rebuild: failed to clear the index: ${cleared.message}`);
|
|
169
210
|
}
|
|
170
|
-
|
|
211
|
+
const indexed = new Map();
|
|
212
|
+
const declined = new Map();
|
|
171
213
|
const skipped = [];
|
|
172
|
-
|
|
214
|
+
// Absent stays absent — only the source knows whether it filtered anything.
|
|
215
|
+
const report = () => ({
|
|
216
|
+
indexed,
|
|
217
|
+
declined,
|
|
218
|
+
excluded: listed.value.excluded,
|
|
219
|
+
skipped
|
|
220
|
+
});
|
|
221
|
+
for (const scoped of listed.value.records) {
|
|
222
|
+
const kind = scoped.record.envelope.kind;
|
|
173
223
|
// Likewise capture-wrapped: an embedder that throws mid-loop would
|
|
174
224
|
// otherwise escape past the `'fail'` rollback below, leaving this DURABLE
|
|
175
225
|
// table holding a partial index that survives the process.
|
|
176
|
-
const embedded = await invokeHook(() => embed(scoped.record));
|
|
226
|
+
const embedded = await (0, rebuildHelpers_1.invokeHook)(() => embed(scoped.record));
|
|
177
227
|
if (embedded.isFailure()) {
|
|
178
228
|
const error = `vector index rebuild: embedding '${(0, ts_agent_memory_1.edgeTargetKey)(scoped.target)}' failed: ${embedded.message}`;
|
|
179
229
|
if (!lenient) {
|
|
180
|
-
return (0, ts_utils_1.
|
|
230
|
+
return (0, ts_utils_1.failWithDetail)((0, rebuildHelpers_1.withRollbackNote)(error, this._clear()), report());
|
|
181
231
|
}
|
|
182
232
|
skipped.push({ target: scoped.target, error });
|
|
183
233
|
continue;
|
|
184
234
|
}
|
|
185
235
|
if (embedded.value === undefined) {
|
|
186
|
-
declined
|
|
236
|
+
(0, rebuildHelpers_1.tally)(declined, kind);
|
|
187
237
|
continue;
|
|
188
238
|
}
|
|
189
239
|
const added = await this.add(scoped.target, embedded.value);
|
|
190
240
|
if (added.isFailure()) {
|
|
191
241
|
const error = `vector index rebuild: ${added.message}`;
|
|
192
242
|
if (!lenient) {
|
|
193
|
-
return (0, ts_utils_1.
|
|
243
|
+
return (0, ts_utils_1.failWithDetail)((0, rebuildHelpers_1.withRollbackNote)(error, this._clear()), report());
|
|
194
244
|
}
|
|
195
245
|
skipped.push({ target: scoped.target, error });
|
|
246
|
+
continue;
|
|
196
247
|
}
|
|
248
|
+
// Tallied in the loop rather than read back off `size` at the end. That
|
|
249
|
+
// `COUNT` was also the only fallible step in assembling the report, so the
|
|
250
|
+
// per-kind tally removes a failure path as well as a rounding of the answer.
|
|
251
|
+
(0, rebuildHelpers_1.tally)(indexed, kind);
|
|
197
252
|
}
|
|
198
|
-
return (0, ts_utils_1.
|
|
199
|
-
.withErrorFormat((msg) => `vector index rebuild: failed to count the rebuilt index: ${msg}`)
|
|
200
|
-
.onSuccess((indexed) => (0, ts_utils_1.succeed)({ indexed, declined, skipped }));
|
|
253
|
+
return (0, ts_utils_1.succeedWithDetail)(report());
|
|
201
254
|
}
|
|
202
255
|
/**
|
|
203
|
-
*
|
|
204
|
-
*
|
|
205
|
-
*
|
|
206
|
-
*
|
|
207
|
-
*
|
|
256
|
+
* **Empties the rows; does NOT release the table's declared dimension.** That
|
|
257
|
+
* is a `vec0` constraint rather than a choice — the dimension is schema, and
|
|
258
|
+
* there is no `ALTER TABLE` for it — so a rebuild at a new dimension fails
|
|
259
|
+
* here where it would succeed on the in-memory sibling, which forgets its
|
|
260
|
+
* dimension on reset. Changing dimension needs a drop-and-re-index; see the
|
|
261
|
+
* note on `IVectorIndex.rebuild`.
|
|
208
262
|
*/
|
|
209
263
|
_clear() {
|
|
210
264
|
if (this._stmts === undefined) {
|
|
@@ -254,7 +308,10 @@ class SqliteVecVectorIndex {
|
|
|
254
308
|
replaceTxn(key, blob);
|
|
255
309
|
},
|
|
256
310
|
query: this._db.prepare(`SELECT target_key, distance FROM "${this._table}" WHERE embedding MATCH ? AND k = ?`),
|
|
257
|
-
count: this._db.prepare(`SELECT count(*) AS c FROM "${this._table}"`)
|
|
311
|
+
count: this._db.prepare(`SELECT count(*) AS c FROM "${this._table}"`),
|
|
312
|
+
// `LIMIT 1` rather than a count: membership needs existence, not cardinality,
|
|
313
|
+
// and vec0 can stop at the first row.
|
|
314
|
+
has: this._db.prepare(`SELECT 1 FROM "${this._table}" WHERE target_key = ? LIMIT 1`)
|
|
258
315
|
};
|
|
259
316
|
}
|
|
260
317
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqliteVecVectorIndex.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/sqliteVecVectorIndex.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,2CAAmD;AACnD,4CAAyF;AACzF,0DAa8B;AAG9B;;;;;;;;;;;;;;GAcG;AACH,KAAK,UAAU,UAAU,CAAI,IAA8B;IACzD,OAAO,CAAC,MAAM,IAAA,6BAAkB,EAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;AACtE,CAAC;AAED;;;;;;GAMG;AACH,SAAS,gBAAgB,CAAC,KAAa,EAAE,QAAsB;IAC7D,OAAO,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,2BAA2B,QAAQ,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AAC/F,CAAC;AAED,iDAAiD;AACjD,MAAM,kBAAkB,GAAW,gBAAgB,CAAC;AAEpD,yGAAyG;AACzG,MAAM,aAAa,GAAW,0BAA0B,CAAC;AAQzD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,MAAa,oBAAoB;IAQ/B,YAAoB,EAA0B,EAAE,KAAa,EAAE,SAA6B;QAC1F,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;IACtE,CAAC;IAED,yEAAyE;IACzE,IAAW,IAAI;QACb,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,CAAC,CAAC;QACX,CAAC;QACD,OAAQ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;IACtD,CAAC;IAED;;;;;;;;;OASG;IACI,MAAM,CAAC,MAAM,CAAC,MAAyC;;QAC5D,MAAM,KAAK,GAAW,MAAA,MAAM,CAAC,SAAS,mCAAI,kBAAkB,CAAC;QAC7D,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,eAAI,EAAC,iCAAiC,KAAK,kCAAkC,CAAC,CAAC,CAAC;QACzG,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,wBAAa,EAAC,GAAG,EAAE;YACjB,IAAA,iBAAa,EAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,MAAM,SAAS,GAAuB,oBAAoB,CAAC,sBAAsB,CAC/E,MAAM,CAAC,QAAQ,EACf,KAAK,CACN,CAAC;YACF,OAAO,IAAI,oBAAoB,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,2CAA2C,CAAC,EAAE,CAAC,CAC1E,CAAC;IACJ,CAAC;IAED,qCAAqC;IAC9B,GAAG,CAAC,MAAmB,EAAE,MAAoB;QAClD,MAAM,GAAG,GAAW,IAAA,+BAAa,EAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,eAAI,EAAC,6BAA6B,GAAG,iBAAiB,CAAC,CAAC,CAAC;QAClF,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACvE,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,eAAI,EACF,6BAA6B,GAAG,gBAAgB,MAAM,CAAC,MAAM,mCAAmC,IAAI,CAAC,UAAU,EAAE,CAClH,CACF,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,wBAAa,EAAC,GAAG,EAAE;YACjB,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC9B,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACjC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;gBAChC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChC,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YAC/D,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,6BAA6B,GAAG,MAAM,CAAC,EAAE,CAAC,CACrE,CAAC;IACJ,CAAC;IAED,wCAAwC;IACjC,MAAM,CAAC,MAAmB;QAC/B,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,wBAAa,EAAC,GAAG,EAAE;YACjB,uEAAuE;YACvE,qCAAqC;YACrC,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,IAAA,+BAAa,EAAC,MAAM,CAAC,CAAC,CAAC;YAChD,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gCAAgC,IAAA,+BAAa,EAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAC1F,CAAC;IACJ,CAAC;IAED;;;;;;;;;;;;OAYG;IACI,KAAK,CAAC,OAAO,CAClB,MAA2B,EAC3B,KAAqB,EACrB,OAA+B;;QAE/B,MAAM,OAAO,GAAY,CAAC,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,aAAa,mCAAI,MAAM,CAAC,KAAK,MAAM,CAAC;QACvE,6EAA6E;QAC7E,6CAA6C;QAC7C,MAAM,MAAM,GAA+C,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACjG,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;YACvB,wEAAwE;YACxE,4EAA4E;YAC5E,0EAA0E;YAC1E,uDAAuD;YACvD,OAAO,IAAA,eAAI,EAAC,iDAAiD,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACjF,CAAC;QACD,MAAM,OAAO,GAAiB,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACxB,OAAO,IAAA,eAAI,EAAC,oDAAoD,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,QAAQ,GAAW,CAAC,CAAC;QACzB,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAClC,mEAAmE;YACnE,0EAA0E;YAC1E,2DAA2D;YAC3D,MAAM,QAAQ,GAAqC,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YAChG,IAAI,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAW,oCAAoC,IAAA,+BAAa,EAAC,MAAM,CAAC,MAAM,CAAC,aACpF,QAAQ,CAAC,OACX,EAAE,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAA,eAAI,EAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACtD,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/C,SAAS;YACX,CAAC;YACD,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBACjC,QAAQ,EAAE,CAAC;gBACX,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAmB,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5E,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAW,yBAAyB,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC/D,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAA,eAAI,EAAC,gBAAgB,CAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;gBACtD,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QACD,OAAO,IAAA,wBAAa,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC;aAClC,eAAe,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,4DAA4D,GAAG,EAAE,CAAC;aAC3F,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAA,kBAAO,EAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;IACrE,CAAC;IAED;;;;;;OAMG;IACK,MAAM;QACZ,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,IAAA,kBAAO,EAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QACD,6EAA6E;QAC7E,yEAAyE;QACzE,uCAAuC;QACvC,OAAO,IAAA,wBAAa,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,gBAAgB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,CAChG,IAAA,kBAAO,EAAC,IAAI,CAAC,CACd,CAAC;IACJ,CAAC;IAED,uCAAuC;IAChC,KAAK,CAAC,MAAoB,EAAE,IAAY;QAC7C,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC3C,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,kBAAO,EAAC,EAAE,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,eAAI,EACF,iCAAiC,MAAM,CAAC,MAAM,mCAAmC,IAAI,CAAC,UAAU,EAAE,CACnG,CACF,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,wBAAa,EAAiC,GAAG,EAAE;YACjD,MAAM,IAAI,GAA2B,IAAI,CAAC,MAAO,CAAC,KAAK,CAAC,GAAG,CACzD,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,EACpC,IAAI,CACqB,CAAC;YAC5B,0EAA0E;YAC1E,8EAA8E;YAC9E,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACxB,MAAM,EAAE,oBAAoB,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC;gBACtD,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC,QAAQ;aACxB,CAAC,CAAC,CAAC;QACN,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,+BAA+B,CAAC,EAAE,CAAC,CAC9D,CAAC;IACJ,CAAC;IAED,sEAAsE;IAC9D,YAAY,CAAC,SAAiB;QACpC,IAAI,CAAC,GAAG,CAAC,IAAI,CACX,uCAAuC,IAAI,CAAC,MAAM,eAAe;YAC/D,gDAAgD,SAAS,2BAA2B,CACvF,CAAC;IACJ,CAAC;IAED,4EAA4E;IACpE,QAAQ;QACd,MAAM,GAAG,GAA4B,IAAI,CAAC,GAAG,CAAC,OAAO,CACnD,gBAAgB,IAAI,CAAC,MAAM,wBAAwB,CACpD,CAAC;QACF,MAAM,GAAG,GAA4B,IAAI,CAAC,GAAG,CAAC,OAAO,CACnD,gBAAgB,IAAI,CAAC,MAAM,wCAAwC,CACpE,CAAC;QACF,wEAAwE;QACxE,kDAAkD;QAClD,MAAM,UAAU,GACd,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,GAAW,EAAE,IAAgB,EAAE,EAAE;YACrD,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACb,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;QACL,OAAO;YACL,MAAM,EAAE,GAAG;YACX,OAAO,EAAE,CAAC,GAAW,EAAE,IAAgB,EAAQ,EAAE;gBAC/C,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACxB,CAAC;YACD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CACrB,qCAAqC,IAAI,CAAC,MAAM,qCAAqC,CACtF;YACD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8BAA8B,IAAI,CAAC,MAAM,GAAG,CAAC;SACtE,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,sBAAsB,CAAC,EAA0B,EAAE,KAAa;QAC7E,MAAM,GAAG,GAAgC,EAAE;aACxC,OAAO,CAAC,iEAAiE,CAAC;aAC1E,GAAG,CAAC,KAAK,CAAgC,CAAC;QAC7C,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,KAAK,GAA4B,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACvE,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,sHAAsH;IAC9G,MAAM,CAAC,OAAO,CAAC,MAAoB;QACzC,OAAO,IAAI,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC;IAC1D,CAAC;IAED;;;OAGG;IACK,MAAM,CAAC,SAAS,CAAC,GAAW;QAClC,MAAM,GAAG,GAAW,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACtC,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAA8B;YACrD,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAwB;SAC9C,CAAC;IACJ,CAAC;CACF;AAhRD,oDAgRC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport type BetterSqlite3 from 'better-sqlite3';\nimport { load as loadSqliteVec } from 'sqlite-vec';\nimport { Result, captureAsyncResult, captureResult, fail, succeed } from '@fgv/ts-utils';\nimport {\n IEdgeTarget,\n IMemoryRecordSource,\n IScopedMemoryRecord,\n ISkippedVectorRecord,\n IVectorIndex,\n IVectorQueryHit,\n IVectorRebuildOptions,\n IVectorRebuildReport,\n MemoryEmbedder,\n MemoryId,\n MemoryScopeKey,\n edgeTargetKey\n} from '@fgv/ts-agent-memory';\nimport { ISqliteVecVectorIndexCreateParams } from './model';\n\n/**\n * Invoke a consumer-supplied hook that already returns a `Result`, converting a\n * synchronous throw or a promise rejection into a `Failure` rather than letting\n * it escape. `captureAsyncResult` wraps the hook's own `Result`, so the outcome\n * is flattened back to one level.\n *\n * @remarks\n * This is `@fgv/ts-utils`' own `_invokeDeferred` shape (see `mapResultsAsync`),\n * which is `@internal` there and so cannot be imported. `@fgv/ts-agent-memory`\n * carries an identical private copy for the in-memory index. Exporting a single\n * `AsyncDeferredResult`-invoking primitive from `ts-utils` is the right home and\n * is recorded in `docs/TECH_DEBT.md`; duplicating three lines twice is the\n * cheaper thing to do from inside this stream than widening it to a foundational\n * library.\n */\nasync function invokeHook<T>(hook: () => Promise<Result<T>>): Promise<Result<T>> {\n return (await captureAsyncResult(hook)).onSuccess((inner) => inner);\n}\n\n/**\n * Compose the failure that aborted a rebuild with the outcome of the rollback\n * that followed it. A rollback that ALSO fails is worth saying out loud: the\n * `'fail'` path promises an empty index, and a caller that retries against a\n * table which is neither the old index nor empty is working from a state the\n * contract never described.\n */\nfunction withRollbackNote(error: string, rollback: Result<true>): string {\n return rollback.isFailure() ? `${error} (rollback also failed: ${rollback.message})` : error;\n}\n\n/** Default name for the `vec0` virtual table. */\nconst DEFAULT_TABLE_NAME: string = 'memory_vectors';\n\n/** A simple SQL identifier — the only shape allowed for the table name (it is interpolated into DDL). */\nconst IDENTIFIER_RE: RegExp = /^[A-Za-z_][A-Za-z0-9_]*$/;\n\n/** One KNN row as returned by the `vec0` MATCH query. */\ninterface IKnnRow {\n readonly target_key: string;\n readonly distance: number;\n}\n\n/**\n * A persistent, `sqlite-vec`-backed `IVectorIndex` for `@fgv/ts-agent-memory`.\n *\n * @remarks\n * This is the **durable** counterpart to the in-memory `InMemoryCosineIndex`:\n * embeddings live in a `sqlite-vec` `vec0` virtual table inside a `better-sqlite3`\n * database, so they survive a process restart. A consumer that wires this index\n * into `FileTreeMemoryStore` (instead of the in-memory index) opens an existing\n * vault **without re-embedding it** — the vectors are already on disk. New writes\n * still flow through the store's incremental embed-on-write path; there is no core\n * store change.\n *\n * The index is keyed by the canonical `edgeTargetKey` of each record's\n * scope-qualified `(scope, id)` address (a `TEXT PRIMARY KEY` on the `vec0` table),\n * so two records that share a filename stem across scopes never collide. The\n * dimension is established by the first `add` (the `vec0` column is fixed-width) and\n * recovered from the table schema when a persistent file is reopened; every later\n * `add`/`query` must match it or fail loudly, exactly as the in-memory index does.\n * Similarity is cosine (`distance_metric=cosine`): the returned `score` is\n * `1 - cosineDistance`, i.e. cosine similarity in `[-1, 1]`, higher = more similar —\n * byte-for-byte the same scoring contract as `InMemoryCosineIndex`.\n *\n * Query is a brute-force `vec0` KNN scan (not an ANN structure): correct and\n * durable, appropriate for the same \"thousands of records\" regime the in-memory\n * index targets. Large-N ANN indexing is explicitly out of scope — see the README.\n *\n * The `better-sqlite3` `Database` is consumer-owned (bring-your-own): this index\n * loads the `sqlite-vec` extension onto it and reads/writes the table, but never\n * opens or closes the connection.\n * @public\n */\nexport class SqliteVecVectorIndex implements IVectorIndex {\n private readonly _db: BetterSqlite3.Database;\n private readonly _table: string;\n /** The dimension of every stored vector; `undefined` until the table exists (first `add` or a reopened non-empty file). */\n private _dimension: number | undefined;\n /** Prepared statements; created once the table exists (established or recovered). */\n private _stmts: ISqliteVecStatements | undefined;\n\n private constructor(db: BetterSqlite3.Database, table: string, dimension: number | undefined) {\n this._db = db;\n this._table = table;\n this._dimension = dimension;\n this._stmts = dimension === undefined ? undefined : this._prepare();\n }\n\n /** The number of vectors currently held. Zero before the first `add`. */\n public get size(): number {\n if (this._stmts === undefined) {\n return 0;\n }\n return (this._stmts.count.get() as { c: number }).c;\n }\n\n /**\n * Family-convention factory. Loads the `sqlite-vec` extension onto the supplied\n * `better-sqlite3` connection and, if the vector table already exists (a reopened\n * persistent file), recovers its established dimension so no re-embedding is\n * needed on open.\n *\n * @param params - See {@link ISqliteVecVectorIndexCreateParams}.\n * @returns `Success` with the index, or `Failure` if the table name is not a\n * simple identifier or the extension fails to load.\n */\n public static create(params: ISqliteVecVectorIndexCreateParams): Promise<Result<SqliteVecVectorIndex>> {\n const table: string = params.tableName ?? DEFAULT_TABLE_NAME;\n if (!IDENTIFIER_RE.test(table)) {\n return Promise.resolve(fail(`sqlite-vec index: table name '${table}' is not a simple SQL identifier`));\n }\n return Promise.resolve(\n captureResult(() => {\n loadSqliteVec(params.database);\n const dimension: number | undefined = SqliteVecVectorIndex._readExistingDimension(\n params.database,\n table\n );\n return new SqliteVecVectorIndex(params.database, table, dimension);\n }).withErrorFormat((e) => `sqlite-vec index: failed to initialize: ${e}`)\n );\n }\n\n /** {@inheritDoc IVectorIndex.add} */\n public add(target: IEdgeTarget, vector: Float32Array): Promise<Result<string>> {\n const key: string = edgeTargetKey(target);\n if (vector.length === 0) {\n return Promise.resolve(fail(`vector index: cannot add '${key}': empty vector`));\n }\n if (this._dimension !== undefined && vector.length !== this._dimension) {\n return Promise.resolve(\n fail(\n `vector index: cannot add '${key}': dimension ${vector.length} does not match index dimension ${this._dimension}`\n )\n );\n }\n return Promise.resolve(\n captureResult(() => {\n if (this._stmts === undefined) {\n this._createTable(vector.length);\n this._dimension = vector.length;\n this._stmts = this._prepare();\n }\n this._stmts.replace(key, SqliteVecVectorIndex._toBlob(vector));\n return key;\n }).withErrorFormat((e) => `vector index: cannot add '${key}': ${e}`)\n );\n }\n\n /** {@inheritDoc IVectorIndex.remove} */\n public remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>> {\n return Promise.resolve(\n captureResult(() => {\n // Idempotent: removing a target with no embedding (or before any `add`\n // created the table) still succeeds.\n if (this._stmts !== undefined) {\n this._stmts.delete.run(edgeTargetKey(target));\n }\n return target;\n }).withErrorFormat((e) => `vector index: cannot remove '${edgeTargetKey(target)}': ${e}`)\n );\n }\n\n /**\n * Re-embed every record from `source` and rebuild the persisted index — see\n * `IVectorIndex.rebuild` for the mode semantics, which this implementation\n * matches exactly.\n *\n * @remarks\n * **Not atomic, and cannot be.** `better-sqlite3` transactions are synchronous,\n * so one cannot span the `await embed(...)` calls this loop makes — unlike\n * {@link SqliteVecVectorIndex.add}, which wraps its delete-then-insert. The\n * `'fail'` / `'skip'` modes therefore cover only failures JavaScript can catch:\n * a process kill mid-rebuild leaves the table holding neither the old index nor\n * the complete new one, and the remedy is to run `rebuild` again.\n */\n public async rebuild(\n source: IMemoryRecordSource,\n embed: MemoryEmbedder,\n options?: IVectorRebuildOptions\n ): Promise<Result<IVectorRebuildReport>> {\n const lenient: boolean = (options?.onRecordError ?? 'fail') === 'skip';\n // `source` is consumer-supplied, so a throw or rejection becomes a `Failure`\n // here rather than escaping as an exception.\n const listed: Result<ReadonlyArray<IScopedMemoryRecord>> = await invokeHook(() => source.list());\n if (listed.isFailure()) {\n // Deliberately BEFORE any clear: a failed list is no evidence about the\n // vectors already held, and no re-embedding has been attempted, so there is\n // no half-rebuilt state to protect against. Clearing here would destroy a\n // healthy persisted index over a transient read error.\n return fail(`vector index rebuild: failed to list records: ${listed.message}`);\n }\n const cleared: Result<true> = this._clear();\n if (cleared.isFailure()) {\n return fail(`vector index rebuild: failed to clear the index: ${cleared.message}`);\n }\n let declined: number = 0;\n const skipped: ISkippedVectorRecord[] = [];\n for (const scoped of listed.value) {\n // Likewise capture-wrapped: an embedder that throws mid-loop would\n // otherwise escape past the `'fail'` rollback below, leaving this DURABLE\n // table holding a partial index that survives the process.\n const embedded: Result<Float32Array | undefined> = await invokeHook(() => embed(scoped.record));\n if (embedded.isFailure()) {\n const error: string = `vector index rebuild: embedding '${edgeTargetKey(scoped.target)}' failed: ${\n embedded.message\n }`;\n if (!lenient) {\n return fail(withRollbackNote(error, this._clear()));\n }\n skipped.push({ target: scoped.target, error });\n continue;\n }\n if (embedded.value === undefined) {\n declined++;\n continue;\n }\n const added: Result<string> = await this.add(scoped.target, embedded.value);\n if (added.isFailure()) {\n const error: string = `vector index rebuild: ${added.message}`;\n if (!lenient) {\n return fail(withRollbackNote(error, this._clear()));\n }\n skipped.push({ target: scoped.target, error });\n }\n }\n return captureResult(() => this.size)\n .withErrorFormat((msg) => `vector index rebuild: failed to count the rebuilt index: ${msg}`)\n .onSuccess((indexed) => succeed({ indexed, declined, skipped }));\n }\n\n /**\n * Empty the table. Deliberately does NOT drop it or forget the established\n * dimension: the `vec0` table's dimension is fixed at creation and a re-embed at\n * a different dimension needs a drop-and-re-index, which is a consumer decision\n * (see the package README on `vec0` schema changes), not something a rebuild\n * should do silently.\n */\n private _clear(): Result<true> {\n if (this._stmts === undefined) {\n return succeed(true);\n }\n // Capture-wrapped like `add` / `remove` / `query`: a closed connection or an\n // I/O error here is a `Failure`, not an exception thrown out of a method\n // whose signature promises a `Result`.\n return captureResult(() => this._db.prepare(`DELETE FROM \"${this._table}\"`).run()).onSuccess(() =>\n succeed(true)\n );\n }\n\n /** {@inheritDoc IVectorIndex.query} */\n public query(vector: Float32Array, topK: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>> {\n if (topK <= 0 || this._stmts === undefined) {\n return Promise.resolve(succeed([]));\n }\n if (vector.length !== this._dimension) {\n return Promise.resolve(\n fail(\n `vector index: query dimension ${vector.length} does not match index dimension ${this._dimension}`\n )\n );\n }\n return Promise.resolve(\n captureResult<ReadonlyArray<IVectorQueryHit>>(() => {\n const rows: ReadonlyArray<IKnnRow> = this._stmts!.query.all(\n SqliteVecVectorIndex._toBlob(vector),\n topK\n ) as ReadonlyArray<IKnnRow>;\n // sqlite-vec returns rows in ascending distance (nearest first); score is\n // `1 - cosineDistance` = cosine similarity, so descending score is preserved.\n return rows.map((row) => ({\n target: SqliteVecVectorIndex._parseKey(row.target_key),\n score: 1 - row.distance\n }));\n }).withErrorFormat((e) => `vector index: query failed: ${e}`)\n );\n }\n\n /** Create the `vec0` virtual table with the established dimension. */\n private _createTable(dimension: number): void {\n this._db.exec(\n `CREATE VIRTUAL TABLE IF NOT EXISTS \"${this._table}\" USING vec0(` +\n `target_key TEXT PRIMARY KEY, embedding float[${dimension}] distance_metric=cosine)`\n );\n }\n\n /** Prepare the statements the index reuses. Requires the table to exist. */\n private _prepare(): ISqliteVecStatements {\n const del: BetterSqlite3.Statement = this._db.prepare(\n `DELETE FROM \"${this._table}\" WHERE target_key = ?`\n );\n const ins: BetterSqlite3.Statement = this._db.prepare(\n `INSERT INTO \"${this._table}\"(target_key, embedding) VALUES (?, ?)`\n );\n // vec0 rejects INSERT OR REPLACE on a TEXT primary key, so replace is a\n // delete-then-insert inside a single transaction.\n const replaceTxn: BetterSqlite3.Transaction<(key: string, blob: Uint8Array) => void> =\n this._db.transaction((key: string, blob: Uint8Array) => {\n del.run(key);\n ins.run(key, blob);\n });\n return {\n delete: del,\n replace: (key: string, blob: Uint8Array): void => {\n replaceTxn(key, blob);\n },\n query: this._db.prepare(\n `SELECT target_key, distance FROM \"${this._table}\" WHERE embedding MATCH ? AND k = ?`\n ),\n count: this._db.prepare(`SELECT count(*) AS c FROM \"${this._table}\"`)\n };\n }\n\n /**\n * Recover the established dimension of an existing `vec0` table from its stored\n * `CREATE VIRTUAL TABLE` SQL (`float[<n>]`). Returns `undefined` when the table\n * does not exist yet (a fresh database — dimension is set by the first `add`).\n */\n private static _readExistingDimension(db: BetterSqlite3.Database, table: string): number | undefined {\n const row: { sql: string } | undefined = db\n .prepare(\"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?\")\n .get(table) as { sql: string } | undefined;\n if (row === undefined) {\n return undefined;\n }\n const match: RegExpMatchArray | null = row.sql.match(/float\\[(\\d+)\\]/);\n return match === null ? undefined : Number(match[1]);\n }\n\n /** Pack a `Float32Array` as the little-endian byte blob `vec0` stores. Copies, so the caller may reuse its buffer. */\n private static _toBlob(vector: Float32Array): Uint8Array {\n return new Uint8Array(Float32Array.from(vector).buffer);\n }\n\n /**\n * Reverse `edgeTargetKey` — the canonical key is `scope\\0id` with NUL\n * excluded from both components, so the first NUL splits it unambiguously.\n */\n private static _parseKey(key: string): IEdgeTarget {\n const nul: number = key.indexOf('\\0');\n return {\n scope: key.slice(0, nul) as unknown as MemoryScopeKey,\n id: key.slice(nul + 1) as unknown as MemoryId\n };\n }\n}\n\n/** The prepared statements / helpers the index reuses once its table exists. */\ninterface ISqliteVecStatements {\n readonly delete: BetterSqlite3.Statement;\n readonly replace: (key: string, blob: Uint8Array) => void;\n readonly query: BetterSqlite3.Statement;\n readonly count: BetterSqlite3.Statement;\n}\n"]}
|
|
1
|
+
{"version":3,"file":"sqliteVecVectorIndex.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/sqliteVecVectorIndex.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAGH,2CAAmD;AACnD,4CAQuB;AACvB,0DAc8B;AAC9B,qDAAuE;AACvE,6CAAyE;AAOzE,iDAAiD;AACjD,MAAM,kBAAkB,GAAW,gBAAgB,CAAC;AAEpD,+DAA+D;AAC/D,MAAM,KAAK,GAAW,kBAAkB,CAAC;AAEzC,yGAAyG;AACzG,MAAM,aAAa,GAAW,0BAA0B,CAAC;AAQzD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,MAAa,oBAAoB;IAQ/B,YAAoB,EAA0B,EAAE,KAAa,EAAE,SAA6B;QAC1F,IAAI,CAAC,GAAG,GAAG,EAAE,CAAC;QACd,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;IACtE,CAAC;IAED,yEAAyE;IACzE,IAAW,IAAI;QACb,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,CAAC,CAAC;QACX,CAAC;QACD,8EAA8E;QAC9E,+EAA+E;QAC/E,+EAA+E;QAC/E,8EAA8E;QAC9E,4EAA4E;QAC5E,8EAA8E;QAC9E,mBAAmB;QACnB,OAAO,MAAM,CAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAA6B,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IAED;;;;;;;;;OASG;IACI,MAAM,CAAC,MAAM,CAAC,MAAyC;;QAC5D,MAAM,KAAK,GAAW,MAAA,MAAM,CAAC,SAAS,mCAAI,kBAAkB,CAAC;QAC7D,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,eAAI,EAAC,iCAAiC,KAAK,kCAAkC,CAAC,CAAC,CAAC;QACzG,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,wBAAa,EAAC,GAAG,EAAE;YACjB,IAAA,iBAAa,EAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC/B,MAAM,SAAS,GAAuB,oBAAoB,CAAC,sBAAsB,CAC/E,MAAM,CAAC,QAAQ,EACf,KAAK,CACN,CAAC;YACF,OAAO,IAAI,oBAAoB,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,2CAA2C,CAAC,EAAE,CAAC,CAC1E,CAAC;IACJ,CAAC;IAED;;;;;;;;;;;;;;;;;;;;;;;;OAwBG;IACI,MAAM,CAAC,KAAK,CAAC,IAAI,CACtB,MAAuC;QAEvC,OAAO,CAAC,MAAM,IAAA,gCAAmB,EAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE,CACtF,CAAC,MAAM,oBAAoB,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC;aAC3E,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE;QACrB,2EAA2E;QAC3E,6EAA6E;QAC7E,2EAA2E;QAC3E,2EAA2E;QAC3E,2EAA2E;QAC3E,iCAAiC;QACjC,IAAA,eAAI,EAAC,IAAA,iCAAgB,EAAC,OAAO,EAAE,IAAA,iCAAoB,EAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,CACvE;aACA,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CACnB,IAAA,kBAAO,EAAC;YACN,KAAK;YACL,KAAK,EAAE,GAAG,EAAE,CAAC,IAAA,iCAAoB,EAAC,QAAQ,EAAE,KAAK,CAAC;SACnD,CAAC,CACH,CACJ,CAAC;IACJ,CAAC;IAED,qCAAqC;IAC9B,GAAG,CAAC,MAAmB,EAAE,MAAoB;QAClD,MAAM,GAAG,GAAW,IAAA,+BAAa,EAAC,MAAM,CAAC,CAAC;QAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,eAAI,EAAC,6BAA6B,GAAG,iBAAiB,CAAC,CAAC,CAAC;QAClF,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACvE,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,eAAI,EACF,6BAA6B,GAAG,gBAAgB,MAAM,CAAC,MAAM,mCAAmC,IAAI,CAAC,UAAU,EAAE,CAClH,CACF,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,wBAAa,EAAC,GAAG,EAAE;YACjB,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC9B,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACjC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC;gBAChC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChC,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;YAC/D,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,6BAA6B,GAAG,MAAM,CAAC,EAAE,CAAC,CACrE,CAAC;IACJ,CAAC;IAED,qCAAqC;IAC9B,GAAG,CAAC,MAAmB;QAC5B,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,wBAAa,EAAC,GAAG,EAAE;YACjB,yEAAyE;YACzE,qEAAqE;YACrE,iCAAiC;YACjC,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC9B,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAA,+BAAa,EAAC,MAAM,CAAC,CAAC,KAAK,SAAS,CAAC;QAClE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,+BAA+B,IAAA,+BAAa,EAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CACzF,CAAC;IACJ,CAAC;IAED,wCAAwC;IACjC,MAAM,CAAC,MAAmB;QAC/B,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,wBAAa,EAAC,GAAG,EAAE;YACjB,uEAAuE;YACvE,qCAAqC;YACrC,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,IAAA,+BAAa,EAAC,MAAM,CAAC,CAAC,CAAC;YAChD,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gCAAgC,IAAA,+BAAa,EAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAC1F,CAAC;IACJ,CAAC;IAED;;;;;;;;;;;;OAYG;IACI,KAAK,CAAC,OAAO,CAClB,MAA2B,EAC3B,KAAqB,EACrB,OAA+B;;QAE/B,MAAM,OAAO,GAAY,CAAC,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,aAAa,mCAAI,MAAM,CAAC,KAAK,MAAM,CAAC;QACvE,6EAA6E;QAC7E,6CAA6C;QAC7C,MAAM,MAAM,GAAiC,MAAM,IAAA,2BAAU,EAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACnF,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;YACvB,wEAAwE;YACxE,4EAA4E;YAC5E,0EAA0E;YAC1E,6EAA6E;YAC7E,sEAAsE;YACtE,OAAO,IAAA,yBAAc,EAAC,iDAAiD,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3F,CAAC;QACD,MAAM,OAAO,GAAiB,IAAI,CAAC,MAAM,EAAE,CAAC;QAC5C,IAAI,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC;YACxB,oEAAoE;YACpE,OAAO,IAAA,yBAAc,EAAC,oDAAoD,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,MAAM,OAAO,GAAsB,IAAI,GAAG,EAAgB,CAAC;QAC3D,MAAM,QAAQ,GAAsB,IAAI,GAAG,EAAgB,CAAC;QAC5D,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,4EAA4E;QAC5E,MAAM,MAAM,GAAG,GAAyB,EAAE,CAAC,CAAC;YAC1C,OAAO;YACP,QAAQ;YACR,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,QAAQ;YAC/B,OAAO;SACR,CAAC,CAAC;QACH,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAS,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;YAC/C,mEAAmE;YACnE,0EAA0E;YAC1E,2DAA2D;YAC3D,MAAM,QAAQ,GAAqC,MAAM,IAAA,2BAAU,EAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;YAChG,IAAI,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAW,oCAAoC,IAAA,+BAAa,EAAC,MAAM,CAAC,MAAM,CAAC,aACpF,QAAQ,CAAC,OACX,EAAE,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAA,yBAAc,EAAC,IAAA,iCAAgB,EAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;gBAC1E,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/C,SAAS;YACX,CAAC;YACD,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBACjC,IAAA,sBAAK,EAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBACtB,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAmB,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC5E,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC;gBACtB,MAAM,KAAK,GAAW,yBAAyB,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC/D,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,OAAO,IAAA,yBAAc,EAAC,IAAA,iCAAgB,EAAC,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;gBAC1E,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC/C,SAAS;YACX,CAAC;YACD,wEAAwE;YACxE,2EAA2E;YAC3E,6EAA6E;YAC7E,IAAA,sBAAK,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC;QACvB,CAAC;QACD,OAAO,IAAA,4BAAiB,EAAC,MAAM,EAAE,CAAC,CAAC;IACrC,CAAC;IAED;;;;;;;OAOG;IACK,MAAM;QACZ,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC9B,OAAO,IAAA,kBAAO,EAAC,IAAI,CAAC,CAAC;QACvB,CAAC;QACD,6EAA6E;QAC7E,yEAAyE;QACzE,uCAAuC;QACvC,OAAO,IAAA,wBAAa,EAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,gBAAgB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,CAChG,IAAA,kBAAO,EAAC,IAAI,CAAC,CACd,CAAC;IACJ,CAAC;IAED,uCAAuC;IAChC,KAAK,CAAC,MAAoB,EAAE,IAAY;QAC7C,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC3C,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,kBAAO,EAAC,EAAE,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,eAAI,EACF,iCAAiC,MAAM,CAAC,MAAM,mCAAmC,IAAI,CAAC,UAAU,EAAE,CACnG,CACF,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,wBAAa,EAAiC,GAAG,EAAE;YACjD,MAAM,IAAI,GAA2B,IAAI,CAAC,MAAO,CAAC,KAAK,CAAC,GAAG,CACzD,oBAAoB,CAAC,OAAO,CAAC,MAAM,CAAC,EACpC,IAAI,CACqB,CAAC;YAC5B,0EAA0E;YAC1E,8EAA8E;YAC9E,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACxB,MAAM,EAAE,oBAAoB,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC;gBACtD,KAAK,EAAE,CAAC,GAAG,GAAG,CAAC,QAAQ;aACxB,CAAC,CAAC,CAAC;QACN,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,+BAA+B,CAAC,EAAE,CAAC,CAC9D,CAAC;IACJ,CAAC;IAED,sEAAsE;IAC9D,YAAY,CAAC,SAAiB;QACpC,IAAI,CAAC,GAAG,CAAC,IAAI,CACX,uCAAuC,IAAI,CAAC,MAAM,eAAe;YAC/D,gDAAgD,SAAS,2BAA2B,CACvF,CAAC;IACJ,CAAC;IAED,4EAA4E;IACpE,QAAQ;QACd,MAAM,GAAG,GAA4B,IAAI,CAAC,GAAG,CAAC,OAAO,CACnD,gBAAgB,IAAI,CAAC,MAAM,wBAAwB,CACpD,CAAC;QACF,MAAM,GAAG,GAA4B,IAAI,CAAC,GAAG,CAAC,OAAO,CACnD,gBAAgB,IAAI,CAAC,MAAM,wCAAwC,CACpE,CAAC;QACF,wEAAwE;QACxE,kDAAkD;QAClD,MAAM,UAAU,GACd,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,GAAW,EAAE,IAAgB,EAAE,EAAE;YACrD,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACb,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;QACrB,CAAC,CAAC,CAAC;QACL,OAAO;YACL,MAAM,EAAE,GAAG;YACX,OAAO,EAAE,CAAC,GAAW,EAAE,IAAgB,EAAQ,EAAE;gBAC/C,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;YACxB,CAAC;YACD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CACrB,qCAAqC,IAAI,CAAC,MAAM,qCAAqC,CACtF;YACD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,8BAA8B,IAAI,CAAC,MAAM,GAAG,CAAC;YACrE,8EAA8E;YAC9E,sCAAsC;YACtC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,kBAAkB,IAAI,CAAC,MAAM,gCAAgC,CAAC;SACrF,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,sBAAsB,CAAC,EAA0B,EAAE,KAAa;QAC7E,MAAM,GAAG,GAAgC,EAAE;aACxC,OAAO,CAAC,iEAAiE,CAAC;aAC1E,GAAG,CAAC,KAAK,CAAgC,CAAC;QAC7C,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,MAAM,KAAK,GAA4B,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;QACvE,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC;IAED,sHAAsH;IAC9G,MAAM,CAAC,OAAO,CAAC,MAAoB;QACzC,OAAO,IAAI,UAAU,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC;IAC1D,CAAC;IAED;;;OAGG;IACK,MAAM,CAAC,SAAS,CAAC,GAAW;QAClC,MAAM,GAAG,GAAW,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACtC,OAAO;YACL,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAA8B;YACrD,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAwB;SAC9C,CAAC;IACJ,CAAC;CACF;AAxWD,oDAwWC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport type BetterSqlite3 from 'better-sqlite3';\nimport { load as loadSqliteVec } from 'sqlite-vec';\nimport {\n DetailedResult,\n Result,\n captureResult,\n fail,\n failWithDetail,\n succeed,\n succeedWithDetail\n} from '@fgv/ts-utils';\nimport {\n IEdgeTarget,\n IMemoryRecordListing,\n IMemoryRecordSource,\n ISkippedVectorRecord,\n IVectorIndex,\n IVectorQueryHit,\n IVectorRebuildOptions,\n IVectorRebuildReport,\n Kind,\n MemoryEmbedder,\n MemoryId,\n MemoryScopeKey,\n edgeTargetKey\n} from '@fgv/ts-agent-memory';\nimport { invokeHook, tally, withRollbackNote } from './rebuildHelpers';\nimport { closeOwnedConnection, openOwnedConnection } from './connection';\nimport {\n ISqliteVecVectorIndexCreateParams,\n ISqliteVecVectorIndexHandle,\n ISqliteVecVectorIndexOpenParams\n} from './model';\n\n/** Default name for the `vec0` virtual table. */\nconst DEFAULT_TABLE_NAME: string = 'memory_vectors';\n\n/** Package-facing prefix for this class's failure messages. */\nconst LABEL: string = 'sqlite-vec index';\n\n/** A simple SQL identifier — the only shape allowed for the table name (it is interpolated into DDL). */\nconst IDENTIFIER_RE: RegExp = /^[A-Za-z_][A-Za-z0-9_]*$/;\n\n/** One KNN row as returned by the `vec0` MATCH query. */\ninterface IKnnRow {\n readonly target_key: string;\n readonly distance: number;\n}\n\n/**\n * A persistent, `sqlite-vec`-backed `IVectorIndex` for `@fgv/ts-agent-memory`.\n *\n * @remarks\n * This is the **durable** counterpart to the in-memory `InMemoryCosineIndex`:\n * embeddings live in a `sqlite-vec` `vec0` virtual table inside a `better-sqlite3`\n * database, so they survive a process restart. A consumer that wires this index\n * into `FileTreeMemoryStore` (instead of the in-memory index) opens an existing\n * vault **without re-embedding it** — the vectors are already on disk. New writes\n * still flow through the store's incremental embed-on-write path; there is no core\n * store change.\n *\n * The index is keyed by the canonical `edgeTargetKey` of each record's\n * scope-qualified `(scope, id)` address (a `TEXT PRIMARY KEY` on the `vec0` table),\n * so two records that share a filename stem across scopes never collide. The\n * dimension is established by the first `add` (the `vec0` column is fixed-width) and\n * recovered from the table schema when a persistent file is reopened; every later\n * `add`/`query` must match it or fail loudly, exactly as the in-memory index does.\n * Similarity is cosine (`distance_metric=cosine`): the returned `score` is\n * `1 - cosineDistance`, i.e. cosine similarity in `[-1, 1]`, higher = more similar —\n * byte-for-byte the same scoring contract as `InMemoryCosineIndex`.\n *\n * Query is a brute-force `vec0` KNN scan (not an ANN structure): correct and\n * durable, appropriate for the same \"thousands of records\" regime the in-memory\n * index targets. Large-N ANN indexing is explicitly out of scope — see the README.\n *\n * **Connection ownership depends on which factory you use.** With\n * {@link SqliteVecVectorIndex.create} the `Database` is consumer-owned\n * (bring-your-own): this index loads the `sqlite-vec` extension onto it and\n * reads/writes the table, but never opens or closes the connection — and that is\n * the seam for backing a record index and a fragment index with one connection.\n * With {@link SqliteVecVectorIndex.open} this package opens the file itself and\n * hands back a handle carrying the disposer for the connection it created.\n * @public\n */\nexport class SqliteVecVectorIndex implements IVectorIndex {\n private readonly _db: BetterSqlite3.Database;\n private readonly _table: string;\n /** The dimension of every stored vector; `undefined` until the table exists (first `add` or a reopened non-empty file). */\n private _dimension: number | undefined;\n /** Prepared statements; created once the table exists (established or recovered). */\n private _stmts: ISqliteVecStatements | undefined;\n\n private constructor(db: BetterSqlite3.Database, table: string, dimension: number | undefined) {\n this._db = db;\n this._table = table;\n this._dimension = dimension;\n this._stmts = dimension === undefined ? undefined : this._prepare();\n }\n\n /** The number of vectors currently held. Zero before the first `add`. */\n public get size(): number {\n if (this._stmts === undefined) {\n return 0;\n }\n // `Number(...)` narrows the count in case the consumer enabled better-sqlite3\n // safe-integer mode (`db.defaultSafeIntegers(true)`), which returns `count(*)`\n // as a `bigint`. Without it a `bigint` leaks through a `number`-typed contract\n // member — and now through `IIndexCoverage.indexSize`, which is also declared\n // `number`, so the coverage report would carry a value of the wrong runtime\n // type. `SqliteVecFragmentIndex`'s two counts have always converted; this one\n // was the outlier.\n return Number((this._stmts.count.get() as { c: number | bigint }).c);\n }\n\n /**\n * Family-convention factory. Loads the `sqlite-vec` extension onto the supplied\n * `better-sqlite3` connection and, if the vector table already exists (a reopened\n * persistent file), recovers its established dimension so no re-embedding is\n * needed on open.\n *\n * @param params - See {@link ISqliteVecVectorIndexCreateParams}.\n * @returns `Success` with the index, or `Failure` if the table name is not a\n * simple identifier or the extension fails to load.\n */\n public static create(params: ISqliteVecVectorIndexCreateParams): Promise<Result<SqliteVecVectorIndex>> {\n const table: string = params.tableName ?? DEFAULT_TABLE_NAME;\n if (!IDENTIFIER_RE.test(table)) {\n return Promise.resolve(fail(`sqlite-vec index: table name '${table}' is not a simple SQL identifier`));\n }\n return Promise.resolve(\n captureResult(() => {\n loadSqliteVec(params.database);\n const dimension: number | undefined = SqliteVecVectorIndex._readExistingDimension(\n params.database,\n table\n );\n return new SqliteVecVectorIndex(params.database, table, dimension);\n }).withErrorFormat((e) => `sqlite-vec index: failed to initialize: ${e}`)\n );\n }\n\n /**\n * Path-based factory. Opens the database file itself and returns the index\n * together with a disposer for the connection it created.\n *\n * @remarks\n * The convenience over {@link SqliteVecVectorIndex.create} is that the consumer\n * neither value-imports `better-sqlite3` nor re-establishes `Result` discipline\n * around a constructor that throws — this is the one place the package leaked its\n * own dependency into consumer source.\n *\n * **Use `create` instead when one connection must back more than one index** (a\n * record index and a fragment index in the same file, the intended shared-handle\n * case). Two `open` calls on one path give two independent connections, not a\n * shared one.\n *\n * If initialization fails after the file is opened, the connection is closed\n * before returning, so a failed `open` does not leak the descriptor it created.\n * Should that close *itself* fail — the connection is then genuinely leaked — the\n * returned message says so rather than hiding it.\n *\n * @param params - See {@link ISqliteVecVectorIndexOpenParams}.\n * @returns `Success` with a {@link ISqliteVecVectorIndexHandle}, or `Failure` if\n * the driver could not be loaded, the file could not be opened, the table name is\n * not a simple identifier, or the extension fails to load.\n */\n public static async open(\n params: ISqliteVecVectorIndexOpenParams\n ): Promise<Result<ISqliteVecVectorIndexHandle>> {\n return (await openOwnedConnection(params.path, LABEL)).thenOnSuccess(async (database) =>\n (await SqliteVecVectorIndex.create({ database, tableName: params.tableName }))\n .onFailure((message) =>\n // This call opened the connection, so a failure to initialize on top of it\n // must not leave the file handle behind. A close that ALSO fails is said out\n // loud rather than swallowed — the same reasoning, and the same helper, as\n // `withRollbackNote`: silently discarding it would make the \"a failed open\n // leaks nothing\" guarantee untrue exactly when it stopped holding, with no\n // way for a caller to detect it.\n fail(withRollbackNote(message, closeOwnedConnection(database, LABEL)))\n )\n .onSuccess((index) =>\n succeed({\n index,\n close: () => closeOwnedConnection(database, LABEL)\n })\n )\n );\n }\n\n /** {@inheritDoc IVectorIndex.add} */\n public add(target: IEdgeTarget, vector: Float32Array): Promise<Result<string>> {\n const key: string = edgeTargetKey(target);\n if (vector.length === 0) {\n return Promise.resolve(fail(`vector index: cannot add '${key}': empty vector`));\n }\n if (this._dimension !== undefined && vector.length !== this._dimension) {\n return Promise.resolve(\n fail(\n `vector index: cannot add '${key}': dimension ${vector.length} does not match index dimension ${this._dimension}`\n )\n );\n }\n return Promise.resolve(\n captureResult(() => {\n if (this._stmts === undefined) {\n this._createTable(vector.length);\n this._dimension = vector.length;\n this._stmts = this._prepare();\n }\n this._stmts.replace(key, SqliteVecVectorIndex._toBlob(vector));\n return key;\n }).withErrorFormat((e) => `vector index: cannot add '${key}': ${e}`)\n );\n }\n\n /** {@inheritDoc IVectorIndex.has} */\n public has(target: IEdgeTarget): Promise<Result<boolean>> {\n return Promise.resolve(\n captureResult(() => {\n // Before any add has created the table there is nothing held, which is a\n // truthful `false` rather than an error — same posture as `remove`'s\n // idempotence and `size`'s zero.\n if (this._stmts === undefined) {\n return false;\n }\n return this._stmts.has.get(edgeTargetKey(target)) !== undefined;\n }).withErrorFormat((e) => `vector index: cannot check '${edgeTargetKey(target)}': ${e}`)\n );\n }\n\n /** {@inheritDoc IVectorIndex.remove} */\n public remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>> {\n return Promise.resolve(\n captureResult(() => {\n // Idempotent: removing a target with no embedding (or before any `add`\n // created the table) still succeeds.\n if (this._stmts !== undefined) {\n this._stmts.delete.run(edgeTargetKey(target));\n }\n return target;\n }).withErrorFormat((e) => `vector index: cannot remove '${edgeTargetKey(target)}': ${e}`)\n );\n }\n\n /**\n * Re-embed every record from `source` and rebuild the persisted index — see\n * `IVectorIndex.rebuild` for the mode semantics, which this implementation\n * matches exactly.\n *\n * @remarks\n * **Not atomic, and cannot be.** `better-sqlite3` transactions are synchronous,\n * so one cannot span the `await embed(...)` calls this loop makes — unlike\n * {@link SqliteVecVectorIndex.add}, which wraps its delete-then-insert. The\n * `'fail'` / `'skip'` modes therefore cover only failures JavaScript can catch:\n * a process kill mid-rebuild leaves the table holding neither the old index nor\n * the complete new one, and the remedy is to run `rebuild` again.\n */\n public async rebuild(\n source: IMemoryRecordSource,\n embed: MemoryEmbedder,\n options?: IVectorRebuildOptions\n ): Promise<DetailedResult<IVectorRebuildReport, IVectorRebuildReport>> {\n const lenient: boolean = (options?.onRecordError ?? 'fail') === 'skip';\n // `source` is consumer-supplied, so a throw or rejection becomes a `Failure`\n // here rather than escaping as an exception.\n const listed: Result<IMemoryRecordListing> = await invokeHook(() => source.list());\n if (listed.isFailure()) {\n // Deliberately BEFORE any clear: a failed list is no evidence about the\n // vectors already held, and no re-embedding has been attempted, so there is\n // no half-rebuilt state to protect against. Clearing here would destroy a\n // healthy persisted index over a transient read error. No report either, for\n // the same reason — there is nothing this call disturbed to describe.\n return failWithDetail(`vector index rebuild: failed to list records: ${listed.message}`);\n }\n const cleared: Result<true> = this._clear();\n if (cleared.isFailure()) {\n // Also nothing established: the table still holds whatever it held.\n return failWithDetail(`vector index rebuild: failed to clear the index: ${cleared.message}`);\n }\n const indexed: Map<Kind, number> = new Map<Kind, number>();\n const declined: Map<Kind, number> = new Map<Kind, number>();\n const skipped: ISkippedVectorRecord[] = [];\n // Absent stays absent — only the source knows whether it filtered anything.\n const report = (): IVectorRebuildReport => ({\n indexed,\n declined,\n excluded: listed.value.excluded,\n skipped\n });\n for (const scoped of listed.value.records) {\n const kind: Kind = scoped.record.envelope.kind;\n // Likewise capture-wrapped: an embedder that throws mid-loop would\n // otherwise escape past the `'fail'` rollback below, leaving this DURABLE\n // table holding a partial index that survives the process.\n const embedded: Result<Float32Array | undefined> = await invokeHook(() => embed(scoped.record));\n if (embedded.isFailure()) {\n const error: string = `vector index rebuild: embedding '${edgeTargetKey(scoped.target)}' failed: ${\n embedded.message\n }`;\n if (!lenient) {\n return failWithDetail(withRollbackNote(error, this._clear()), report());\n }\n skipped.push({ target: scoped.target, error });\n continue;\n }\n if (embedded.value === undefined) {\n tally(declined, kind);\n continue;\n }\n const added: Result<string> = await this.add(scoped.target, embedded.value);\n if (added.isFailure()) {\n const error: string = `vector index rebuild: ${added.message}`;\n if (!lenient) {\n return failWithDetail(withRollbackNote(error, this._clear()), report());\n }\n skipped.push({ target: scoped.target, error });\n continue;\n }\n // Tallied in the loop rather than read back off `size` at the end. That\n // `COUNT` was also the only fallible step in assembling the report, so the\n // per-kind tally removes a failure path as well as a rounding of the answer.\n tally(indexed, kind);\n }\n return succeedWithDetail(report());\n }\n\n /**\n * **Empties the rows; does NOT release the table's declared dimension.** That\n * is a `vec0` constraint rather than a choice — the dimension is schema, and\n * there is no `ALTER TABLE` for it — so a rebuild at a new dimension fails\n * here where it would succeed on the in-memory sibling, which forgets its\n * dimension on reset. Changing dimension needs a drop-and-re-index; see the\n * note on `IVectorIndex.rebuild`.\n */\n private _clear(): Result<true> {\n if (this._stmts === undefined) {\n return succeed(true);\n }\n // Capture-wrapped like `add` / `remove` / `query`: a closed connection or an\n // I/O error here is a `Failure`, not an exception thrown out of a method\n // whose signature promises a `Result`.\n return captureResult(() => this._db.prepare(`DELETE FROM \"${this._table}\"`).run()).onSuccess(() =>\n succeed(true)\n );\n }\n\n /** {@inheritDoc IVectorIndex.query} */\n public query(vector: Float32Array, topK: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>> {\n if (topK <= 0 || this._stmts === undefined) {\n return Promise.resolve(succeed([]));\n }\n if (vector.length !== this._dimension) {\n return Promise.resolve(\n fail(\n `vector index: query dimension ${vector.length} does not match index dimension ${this._dimension}`\n )\n );\n }\n return Promise.resolve(\n captureResult<ReadonlyArray<IVectorQueryHit>>(() => {\n const rows: ReadonlyArray<IKnnRow> = this._stmts!.query.all(\n SqliteVecVectorIndex._toBlob(vector),\n topK\n ) as ReadonlyArray<IKnnRow>;\n // sqlite-vec returns rows in ascending distance (nearest first); score is\n // `1 - cosineDistance` = cosine similarity, so descending score is preserved.\n return rows.map((row) => ({\n target: SqliteVecVectorIndex._parseKey(row.target_key),\n score: 1 - row.distance\n }));\n }).withErrorFormat((e) => `vector index: query failed: ${e}`)\n );\n }\n\n /** Create the `vec0` virtual table with the established dimension. */\n private _createTable(dimension: number): void {\n this._db.exec(\n `CREATE VIRTUAL TABLE IF NOT EXISTS \"${this._table}\" USING vec0(` +\n `target_key TEXT PRIMARY KEY, embedding float[${dimension}] distance_metric=cosine)`\n );\n }\n\n /** Prepare the statements the index reuses. Requires the table to exist. */\n private _prepare(): ISqliteVecStatements {\n const del: BetterSqlite3.Statement = this._db.prepare(\n `DELETE FROM \"${this._table}\" WHERE target_key = ?`\n );\n const ins: BetterSqlite3.Statement = this._db.prepare(\n `INSERT INTO \"${this._table}\"(target_key, embedding) VALUES (?, ?)`\n );\n // vec0 rejects INSERT OR REPLACE on a TEXT primary key, so replace is a\n // delete-then-insert inside a single transaction.\n const replaceTxn: BetterSqlite3.Transaction<(key: string, blob: Uint8Array) => void> =\n this._db.transaction((key: string, blob: Uint8Array) => {\n del.run(key);\n ins.run(key, blob);\n });\n return {\n delete: del,\n replace: (key: string, blob: Uint8Array): void => {\n replaceTxn(key, blob);\n },\n query: this._db.prepare(\n `SELECT target_key, distance FROM \"${this._table}\" WHERE embedding MATCH ? AND k = ?`\n ),\n count: this._db.prepare(`SELECT count(*) AS c FROM \"${this._table}\"`),\n // `LIMIT 1` rather than a count: membership needs existence, not cardinality,\n // and vec0 can stop at the first row.\n has: this._db.prepare(`SELECT 1 FROM \"${this._table}\" WHERE target_key = ? LIMIT 1`)\n };\n }\n\n /**\n * Recover the established dimension of an existing `vec0` table from its stored\n * `CREATE VIRTUAL TABLE` SQL (`float[<n>]`). Returns `undefined` when the table\n * does not exist yet (a fresh database — dimension is set by the first `add`).\n */\n private static _readExistingDimension(db: BetterSqlite3.Database, table: string): number | undefined {\n const row: { sql: string } | undefined = db\n .prepare(\"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?\")\n .get(table) as { sql: string } | undefined;\n if (row === undefined) {\n return undefined;\n }\n const match: RegExpMatchArray | null = row.sql.match(/float\\[(\\d+)\\]/);\n return match === null ? undefined : Number(match[1]);\n }\n\n /** Pack a `Float32Array` as the little-endian byte blob `vec0` stores. Copies, so the caller may reuse its buffer. */\n private static _toBlob(vector: Float32Array): Uint8Array {\n return new Uint8Array(Float32Array.from(vector).buffer);\n }\n\n /**\n * Reverse `edgeTargetKey` — the canonical key is `scope\\0id` with NUL\n * excluded from both components, so the first NUL splits it unambiguously.\n */\n private static _parseKey(key: string): IEdgeTarget {\n const nul: number = key.indexOf('\\0');\n return {\n scope: key.slice(0, nul) as unknown as MemoryScopeKey,\n id: key.slice(nul + 1) as unknown as MemoryId\n };\n }\n}\n\n/** The prepared statements / helpers the index reuses once its table exists. */\ninterface ISqliteVecStatements {\n readonly delete: BetterSqlite3.Statement;\n readonly replace: (key: string, blob: Uint8Array) => void;\n readonly query: BetterSqlite3.Statement;\n readonly count: BetterSqlite3.Statement;\n readonly has: BetterSqlite3.Statement;\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fgv/ts-agent-memory-sqlite-vec",
|
|
3
|
-
"version": "5.1.0-
|
|
3
|
+
"version": "5.1.0-51",
|
|
4
4
|
"description": "Result-integration boundary providing a persistent, sqlite-vec-backed IVectorIndex for @fgv/ts-agent-memory (survives restarts — no re-embed on open)",
|
|
5
5
|
"main": "lib/index.js",
|
|
6
6
|
"types": "dist/ts-agent-memory-sqlite-vec.d.ts",
|
|
@@ -72,16 +72,16 @@
|
|
|
72
72
|
"ts-jest": "^29.4.6",
|
|
73
73
|
"ts-node": "^10.9.2",
|
|
74
74
|
"typescript": "5.9.3",
|
|
75
|
-
"@fgv/heft-dual-rig": "5.1.0-
|
|
76
|
-
"@fgv/ts-utils": "5.1.0-
|
|
77
|
-
"@fgv/ts-agent-memory": "5.1.0-
|
|
78
|
-
"@fgv/ts-utils-jest": "5.1.0-
|
|
75
|
+
"@fgv/heft-dual-rig": "5.1.0-51",
|
|
76
|
+
"@fgv/ts-utils": "5.1.0-51",
|
|
77
|
+
"@fgv/ts-agent-memory": "5.1.0-51",
|
|
78
|
+
"@fgv/ts-utils-jest": "5.1.0-51"
|
|
79
79
|
},
|
|
80
80
|
"peerDependencies": {
|
|
81
81
|
"better-sqlite3": "^12.0.0",
|
|
82
82
|
"sqlite-vec": "^0.1.9",
|
|
83
|
-
"@fgv/ts-utils": "5.1.0-
|
|
84
|
-
"@fgv/ts-agent-memory": "5.1.0-
|
|
83
|
+
"@fgv/ts-utils": "5.1.0-51",
|
|
84
|
+
"@fgv/ts-agent-memory": "5.1.0-51"
|
|
85
85
|
},
|
|
86
86
|
"scripts": {
|
|
87
87
|
"build": "heft build --clean",
|