@fgv/ts-agent-memory-sqlite-vec 5.1.0-50 → 5.1.0-52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -1
- package/dist/packlets/sqlite-vec-index/connection.js +54 -0
- package/dist/packlets/sqlite-vec-index/connection.js.map +1 -0
- package/dist/packlets/sqlite-vec-index/model.js.map +1 -1
- package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +162 -8
- package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -1
- package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +115 -4
- package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -1
- package/dist/ts-agent-memory-sqlite-vec.d.ts +258 -10
- package/lib/packlets/sqlite-vec-index/connection.d.ts +42 -0
- package/lib/packlets/sqlite-vec-index/connection.d.ts.map +1 -0
- package/lib/packlets/sqlite-vec-index/connection.js +91 -0
- package/lib/packlets/sqlite-vec-index/connection.js.map +1 -0
- package/lib/packlets/sqlite-vec-index/model.d.ts +92 -0
- package/lib/packlets/sqlite-vec-index/model.d.ts.map +1 -1
- package/lib/packlets/sqlite-vec-index/model.js.map +1 -1
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts +89 -8
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts.map +1 -1
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +162 -8
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -1
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts +78 -5
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts.map +1 -1
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +115 -4
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -1
- package/package.json +7 -7
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { DetailedResult, Result } from '@fgv/ts-utils';
|
|
2
|
-
import { FragmentEmbedder, IEdgeTarget, IEmbeddedFragment, IFragmentVectorIndex, IFragmentVectorRebuildReport, IMemoryRecordSource, IVectorQueryHit, IVectorRebuildOptions } from '@fgv/ts-agent-memory';
|
|
3
|
-
import { ISqliteVecFragmentIndexCreateParams } from './model';
|
|
2
|
+
import { FragmentEmbedder, IEdgeTarget, IEmbeddedFragment, IFragmentVectorIndex, IFragmentVectorRebuildReport, IMemoryRecordSource, IVectorQueryHit, IVectorRebuildOptions, IFragmentQueryOptions } from '@fgv/ts-agent-memory';
|
|
3
|
+
import { ISqliteVecFragmentIndexCreateParams, ISqliteVecFragmentIndexHandle, ISqliteVecFragmentIndexOpenParams } from './model';
|
|
4
4
|
/**
|
|
5
5
|
* A persistent, `sqlite-vec`-backed `IFragmentVectorIndex` (from
|
|
6
6
|
* `@fgv/ts-agent-memory`) — the fragment-granular sibling of
|
|
@@ -40,9 +40,13 @@ import { ISqliteVecFragmentIndexCreateParams } from './model';
|
|
|
40
40
|
* cosine (`score = 1 - cosineDistance`), byte-identical to the in-memory index.
|
|
41
41
|
* Large-N ANN indexing is explicitly out of scope, same regime as the record index.
|
|
42
42
|
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
43
|
+
* **Connection ownership depends on which factory you use.** With
|
|
44
|
+
* {@link SqliteVecFragmentIndex.create} the `Database` is consumer-owned
|
|
45
|
+
* (bring-your-own): this index loads the `sqlite-vec` extension onto it and
|
|
46
|
+
* reads/writes the table, but never opens or closes the connection — and that is
|
|
47
|
+
* the seam for backing this index and a record index with one connection. With
|
|
48
|
+
* {@link SqliteVecFragmentIndex.open} this package opens the file itself and hands
|
|
49
|
+
* back a handle carrying the disposer for the connection it created.
|
|
46
50
|
* @public
|
|
47
51
|
*/
|
|
48
52
|
export declare class SqliteVecFragmentIndex implements IFragmentVectorIndex {
|
|
@@ -52,10 +56,29 @@ export declare class SqliteVecFragmentIndex implements IFragmentVectorIndex {
|
|
|
52
56
|
private _dimension;
|
|
53
57
|
/** Prepared statements; created once the table exists (established or recovered). */
|
|
54
58
|
private _stmts;
|
|
59
|
+
/**
|
|
60
|
+
* Set by {@link SqliteVecFragmentIndex.release}. Distinct from `_stmts === undefined`,
|
|
61
|
+
* which means *no dimension established yet* — see the remarks on `release`.
|
|
62
|
+
*/
|
|
63
|
+
private _released;
|
|
55
64
|
private constructor();
|
|
56
|
-
/**
|
|
65
|
+
/**
|
|
66
|
+
* The number of records that currently have at least one stored fragment. Zero
|
|
67
|
+
* before the first add.
|
|
68
|
+
*
|
|
69
|
+
* @remarks
|
|
70
|
+
* **Throws on a released index**, where every other member returns a `Failure` —
|
|
71
|
+
* `IFragmentVectorIndex` declares this a synchronous `number`, so there is no
|
|
72
|
+
* `Result` to fail into, and answering `0` would be a confident lie
|
|
73
|
+
* indistinguishable from an empty index. Same reasoning as
|
|
74
|
+
* {@link SqliteVecFragmentIndex.fragmentCount} and `SqliteVecVectorIndex.size`.
|
|
75
|
+
*/
|
|
57
76
|
get recordCount(): number;
|
|
58
|
-
/**
|
|
77
|
+
/**
|
|
78
|
+
* The total number of fragments currently held across all records. Zero before
|
|
79
|
+
* the first add. **Throws on a released index** — see
|
|
80
|
+
* {@link SqliteVecFragmentIndex.recordCount}.
|
|
81
|
+
*/
|
|
59
82
|
get fragmentCount(): number;
|
|
60
83
|
/**
|
|
61
84
|
* Family-convention factory. Loads the `sqlite-vec` extension onto the supplied
|
|
@@ -71,6 +94,64 @@ export declare class SqliteVecFragmentIndex implements IFragmentVectorIndex {
|
|
|
71
94
|
* drop-and-re-index — `vec0` cannot be altered in place).
|
|
72
95
|
*/
|
|
73
96
|
static create(params: ISqliteVecFragmentIndexCreateParams): Promise<Result<SqliteVecFragmentIndex>>;
|
|
97
|
+
/**
|
|
98
|
+
* Path-based factory. Opens the database file itself and returns the index
|
|
99
|
+
* together with a disposer for the connection it created.
|
|
100
|
+
*
|
|
101
|
+
* @remarks
|
|
102
|
+
* The fragment-granular sibling of {@link SqliteVecVectorIndex.open}, and present
|
|
103
|
+
* for the same reason: a consumer doing sub-document retrieval only would
|
|
104
|
+
* otherwise still value-import `better-sqlite3` and hand-roll a `captureResult`
|
|
105
|
+
* around a constructor that throws.
|
|
106
|
+
*
|
|
107
|
+
* **Use `create` instead when one connection must back both a fragment index and
|
|
108
|
+
* a record index** — the intended shared-handle case. Two `open` calls on one path
|
|
109
|
+
* give two independent connections, not a shared one.
|
|
110
|
+
*
|
|
111
|
+
* If initialization fails after the file is opened, the connection is closed
|
|
112
|
+
* before returning, so a failed `open` does not leak the descriptor it created.
|
|
113
|
+
* Should that close *itself* fail — the connection is then genuinely leaked — the
|
|
114
|
+
* returned message says so rather than hiding it. That includes the
|
|
115
|
+
* auxiliary-column mismatch failure, which is reported by `create` only after the
|
|
116
|
+
* file is open.
|
|
117
|
+
*
|
|
118
|
+
* @param params - See {@link ISqliteVecFragmentIndexOpenParams}.
|
|
119
|
+
* @returns `Success` with a {@link ISqliteVecFragmentIndexHandle}, or `Failure` if
|
|
120
|
+
* the driver could not be loaded, the file could not be opened, the table name is
|
|
121
|
+
* not a simple identifier, the extension fails to load, or the existing table was
|
|
122
|
+
* written by a version with a different auxiliary-column set.
|
|
123
|
+
*/
|
|
124
|
+
static open(params: ISqliteVecFragmentIndexOpenParams): Promise<Result<ISqliteVecFragmentIndexHandle>>;
|
|
125
|
+
/**
|
|
126
|
+
* Drops this index's prepared statements and marks it unusable. Does **not**
|
|
127
|
+
* touch the connection.
|
|
128
|
+
*
|
|
129
|
+
* @remarks
|
|
130
|
+
* The fragment-lane counterpart of `SqliteVecVectorIndex.release`, and it
|
|
131
|
+
* matters here for the same reason plus one more: a shared-connection
|
|
132
|
+
* deployment — the case `create({ database })` exists for — holds a record index
|
|
133
|
+
* *and* a fragment index over one connection, so it carries two instances of the
|
|
134
|
+
* statement-lifetime shape rather than one. Both must be released.
|
|
135
|
+
*
|
|
136
|
+
* `better-sqlite3` exposes no public `finalize()`, so dropping the last
|
|
137
|
+
* reference does not finalize a statement — it makes it collectable *earlier*,
|
|
138
|
+
* while the environment is alive, rather than surviving to process teardown.
|
|
139
|
+
* That narrows the window in which `Statement::~Statement()` runs against a
|
|
140
|
+
* torn-down environment; it is not a proof against it.
|
|
141
|
+
*
|
|
142
|
+
* **Call this before closing a connection you own.**
|
|
143
|
+
* {@link SqliteVecFragmentIndex.open}'s handle does it for you.
|
|
144
|
+
*
|
|
145
|
+
* Idempotent. After it, every member fails (or, for the two counts, throws)
|
|
146
|
+
* rather than answering.
|
|
147
|
+
*/
|
|
148
|
+
release(): void;
|
|
149
|
+
/**
|
|
150
|
+
* Throw if this index has been released. The members that call it and cannot
|
|
151
|
+
* return a `Result` are the two counts; the rest convert the throw via
|
|
152
|
+
* `captureResult`.
|
|
153
|
+
*/
|
|
154
|
+
private _assertUsable;
|
|
74
155
|
/** {@inheritDoc IFragmentVectorIndex.addFragments} */
|
|
75
156
|
addFragments(target: IEdgeTarget, fragments: ReadonlyArray<IEmbeddedFragment>): Promise<Result<number>>;
|
|
76
157
|
/** {@inheritDoc IFragmentVectorIndex.remove} */
|
|
@@ -89,7 +170,7 @@ export declare class SqliteVecFragmentIndex implements IFragmentVectorIndex {
|
|
|
89
170
|
*/
|
|
90
171
|
private _clear;
|
|
91
172
|
/** {@inheritDoc IFragmentVectorIndex.query} */
|
|
92
|
-
query(vector: Float32Array, topK: number,
|
|
173
|
+
query(vector: Float32Array, topK: number, options?: IFragmentQueryOptions): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
|
|
93
174
|
/**
|
|
94
175
|
* Create the fragment `vec0` virtual table with the established dimension. The
|
|
95
176
|
* auxiliary columns must stay in sync with `AUXILIARY_COLUMNS`, which
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"sqliteVecFragmentIndex.d.ts","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/sqliteVecFragmentIndex.ts"],"names":[],"mappings":"AAOA,OAAO,EACL,cAAc,EACd,MAAM,EAMP,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,iBAAiB,EAEjB,oBAAoB,EACpB,4BAA4B,EAE5B,mBAAmB,EAEnB,eAAe,EACf,qBAAqB,
|
|
1
|
+
{"version":3,"file":"sqliteVecFragmentIndex.d.ts","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/sqliteVecFragmentIndex.ts"],"names":[],"mappings":"AAOA,OAAO,EACL,cAAc,EACd,MAAM,EAMP,MAAM,eAAe,CAAC;AACvB,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,iBAAiB,EAEjB,oBAAoB,EACpB,4BAA4B,EAE5B,mBAAmB,EAEnB,eAAe,EACf,qBAAqB,EAErB,qBAAqB,EAItB,MAAM,sBAAsB,CAAC;AAG9B,OAAO,EACL,mCAAmC,EACnC,6BAA6B,EAC7B,iCAAiC,EAClC,MAAM,SAAS,CAAC;AAsDjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AACH,qBAAa,sBAAuB,YAAW,oBAAoB;IACjE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAyB;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,yFAAyF;IACzF,OAAO,CAAC,UAAU,CAAqB;IACvC,qFAAqF;IACrF,OAAO,CAAC,MAAM,CAAkC;IAChD;;;OAGG;IACH,OAAO,CAAC,SAAS,CAAU;IAE3B,OAAO;IAQP;;;;;;;;;;OAUG;IACH,IAAW,WAAW,IAAI,MAAM,CAQ/B;IAED;;;;OAIG;IACH,IAAW,aAAa,IAAI,MAAM,CAMjC;IAED;;;;;;;;;;;;OAYG;WACW,MAAM,CAAC,MAAM,EAAE,mCAAmC,GAAG,OAAO,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAmB1G;;;;;;;;;;;;;;;;;;;;;;;;;;OA0BG;WACiB,IAAI,CACtB,MAAM,EAAE,iCAAiC,GACxC,OAAO,CAAC,MAAM,CAAC,6BAA6B,CAAC,CAAC;IA2BjD;;;;;;;;;;;;;;;;;;;;;;OAsBG;IACI,OAAO,IAAI,IAAI;IAKtB;;;;OAIG;IACH,OAAO,CAAC,aAAa;IAMrB,sDAAsD;IAC/C,YAAY,CACjB,MAAM,EAAE,WAAW,EACnB,SAAS,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAC1C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IA4E1B,gDAAgD;IACzC,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAchE,6CAA6C;IACtC,GAAG,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAczD,iDAAiD;IACpC,OAAO,CAClB,MAAM,EAAE,mBAAmB,EAC3B,KAAK,EAAE,gBAAgB,EACvB,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,cAAc,CAAC,4BAA4B,EAAE,4BAA4B,CAAC,CAAC;IAqEtF;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM;IAcd,+CAA+C;IACxC,KAAK,CACV,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC;IAsFlD;;;;OAIG;IACH,OAAO,CAAC,YAAY;IAQpB,4EAA4E;IAC5E,OAAO,CAAC,QAAQ;IAoDhB;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAsBrC;;;;;;;;;OASG;IACH,OAAO,CAAC,MAAM,CAAC,uBAAuB;IAiBtC;;;;;;;;OAQG;IACH,OAAO,CAAC,MAAM,CAAC,WAAW;IAa1B;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM,CAAC,UAAU;IAiBzB,sHAAsH;IACtH,OAAO,CAAC,MAAM,CAAC,OAAO;IAItB;;;;;;OAMG;IACH,OAAO,CAAC,MAAM,CAAC,SAAS;IAWxB;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM,CAAC,SAAS;CASzB"}
|
|
@@ -9,8 +9,11 @@ const sqlite_vec_1 = require("sqlite-vec");
|
|
|
9
9
|
const ts_utils_1 = require("@fgv/ts-utils");
|
|
10
10
|
const ts_agent_memory_1 = require("@fgv/ts-agent-memory");
|
|
11
11
|
const rebuildHelpers_1 = require("./rebuildHelpers");
|
|
12
|
+
const connection_1 = require("./connection");
|
|
12
13
|
/** Default name for the fragment `vec0` virtual table. */
|
|
13
14
|
const DEFAULT_TABLE_NAME = 'memory_fragments';
|
|
15
|
+
/** Package-facing prefix for this class's failure messages. */
|
|
16
|
+
const LABEL = 'sqlite-vec fragment index';
|
|
14
17
|
/** A simple SQL identifier — the only shape allowed for the table name (it is interpolated into DDL). */
|
|
15
18
|
const IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
16
19
|
/**
|
|
@@ -65,9 +68,13 @@ const AUXILIARY_COLUMN_RE = /\+\s*([A-Za-z_][A-Za-z0-9_]*)/g;
|
|
|
65
68
|
* cosine (`score = 1 - cosineDistance`), byte-identical to the in-memory index.
|
|
66
69
|
* Large-N ANN indexing is explicitly out of scope, same regime as the record index.
|
|
67
70
|
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
+
* **Connection ownership depends on which factory you use.** With
|
|
72
|
+
* {@link SqliteVecFragmentIndex.create} the `Database` is consumer-owned
|
|
73
|
+
* (bring-your-own): this index loads the `sqlite-vec` extension onto it and
|
|
74
|
+
* reads/writes the table, but never opens or closes the connection — and that is
|
|
75
|
+
* the seam for backing this index and a record index with one connection. With
|
|
76
|
+
* {@link SqliteVecFragmentIndex.open} this package opens the file itself and hands
|
|
77
|
+
* back a handle carrying the disposer for the connection it created.
|
|
71
78
|
* @public
|
|
72
79
|
*/
|
|
73
80
|
class SqliteVecFragmentIndex {
|
|
@@ -75,10 +82,22 @@ class SqliteVecFragmentIndex {
|
|
|
75
82
|
this._db = db;
|
|
76
83
|
this._table = table;
|
|
77
84
|
this._dimension = dimension;
|
|
85
|
+
this._released = false;
|
|
78
86
|
this._stmts = dimension === undefined ? undefined : this._prepare();
|
|
79
87
|
}
|
|
80
|
-
/**
|
|
88
|
+
/**
|
|
89
|
+
* The number of records that currently have at least one stored fragment. Zero
|
|
90
|
+
* before the first add.
|
|
91
|
+
*
|
|
92
|
+
* @remarks
|
|
93
|
+
* **Throws on a released index**, where every other member returns a `Failure` —
|
|
94
|
+
* `IFragmentVectorIndex` declares this a synchronous `number`, so there is no
|
|
95
|
+
* `Result` to fail into, and answering `0` would be a confident lie
|
|
96
|
+
* indistinguishable from an empty index. Same reasoning as
|
|
97
|
+
* {@link SqliteVecFragmentIndex.fragmentCount} and `SqliteVecVectorIndex.size`.
|
|
98
|
+
*/
|
|
81
99
|
get recordCount() {
|
|
100
|
+
this._assertUsable('read recordCount');
|
|
82
101
|
if (this._stmts === undefined) {
|
|
83
102
|
return 0;
|
|
84
103
|
}
|
|
@@ -86,8 +105,13 @@ class SqliteVecFragmentIndex {
|
|
|
86
105
|
// safe-integer mode (which returns `count(*)` as a `bigint`).
|
|
87
106
|
return Number(this._stmts.recordCount.get().c);
|
|
88
107
|
}
|
|
89
|
-
/**
|
|
108
|
+
/**
|
|
109
|
+
* The total number of fragments currently held across all records. Zero before
|
|
110
|
+
* the first add. **Throws on a released index** — see
|
|
111
|
+
* {@link SqliteVecFragmentIndex.recordCount}.
|
|
112
|
+
*/
|
|
90
113
|
get fragmentCount() {
|
|
114
|
+
this._assertUsable('read fragmentCount');
|
|
91
115
|
if (this._stmts === undefined) {
|
|
92
116
|
return 0;
|
|
93
117
|
}
|
|
@@ -118,9 +142,97 @@ class SqliteVecFragmentIndex {
|
|
|
118
142
|
return new SqliteVecFragmentIndex(params.database, table, dimension);
|
|
119
143
|
}).withErrorFormat((e) => `sqlite-vec fragment index: failed to initialize: ${e}`));
|
|
120
144
|
}
|
|
145
|
+
/**
|
|
146
|
+
* Path-based factory. Opens the database file itself and returns the index
|
|
147
|
+
* together with a disposer for the connection it created.
|
|
148
|
+
*
|
|
149
|
+
* @remarks
|
|
150
|
+
* The fragment-granular sibling of {@link SqliteVecVectorIndex.open}, and present
|
|
151
|
+
* for the same reason: a consumer doing sub-document retrieval only would
|
|
152
|
+
* otherwise still value-import `better-sqlite3` and hand-roll a `captureResult`
|
|
153
|
+
* around a constructor that throws.
|
|
154
|
+
*
|
|
155
|
+
* **Use `create` instead when one connection must back both a fragment index and
|
|
156
|
+
* a record index** — the intended shared-handle case. Two `open` calls on one path
|
|
157
|
+
* give two independent connections, not a shared one.
|
|
158
|
+
*
|
|
159
|
+
* If initialization fails after the file is opened, the connection is closed
|
|
160
|
+
* before returning, so a failed `open` does not leak the descriptor it created.
|
|
161
|
+
* Should that close *itself* fail — the connection is then genuinely leaked — the
|
|
162
|
+
* returned message says so rather than hiding it. That includes the
|
|
163
|
+
* auxiliary-column mismatch failure, which is reported by `create` only after the
|
|
164
|
+
* file is open.
|
|
165
|
+
*
|
|
166
|
+
* @param params - See {@link ISqliteVecFragmentIndexOpenParams}.
|
|
167
|
+
* @returns `Success` with a {@link ISqliteVecFragmentIndexHandle}, or `Failure` if
|
|
168
|
+
* the driver could not be loaded, the file could not be opened, the table name is
|
|
169
|
+
* not a simple identifier, the extension fails to load, or the existing table was
|
|
170
|
+
* written by a version with a different auxiliary-column set.
|
|
171
|
+
*/
|
|
172
|
+
static async open(params) {
|
|
173
|
+
return (await (0, connection_1.openOwnedConnection)(params.path, LABEL)).thenOnSuccess(async (database) => (await SqliteVecFragmentIndex.create({ database, tableName: params.tableName }))
|
|
174
|
+
.onFailure((message) =>
|
|
175
|
+
// This call opened the connection, so a failure to initialize on top of it
|
|
176
|
+
// must not leave the file handle behind. A close that ALSO fails is said out
|
|
177
|
+
// loud rather than swallowed — the same reasoning, and the same helper, as
|
|
178
|
+
// `withRollbackNote`: silently discarding it would make the "a failed open
|
|
179
|
+
// leaks nothing" guarantee untrue exactly when it stopped holding, with no
|
|
180
|
+
// way for a caller to detect it.
|
|
181
|
+
(0, ts_utils_1.fail)((0, rebuildHelpers_1.withRollbackNote)(message, (0, connection_1.closeOwnedConnection)(database, LABEL))))
|
|
182
|
+
.onSuccess((index) => (0, ts_utils_1.succeed)({
|
|
183
|
+
index,
|
|
184
|
+
close: () => {
|
|
185
|
+
// Drop the statements BEFORE closing, so there is never a moment where
|
|
186
|
+
// a closed connection has live `Statement` objects pointing at it —
|
|
187
|
+
// see `release`.
|
|
188
|
+
index.release();
|
|
189
|
+
return (0, connection_1.closeOwnedConnection)(database, LABEL);
|
|
190
|
+
}
|
|
191
|
+
})));
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Drops this index's prepared statements and marks it unusable. Does **not**
|
|
195
|
+
* touch the connection.
|
|
196
|
+
*
|
|
197
|
+
* @remarks
|
|
198
|
+
* The fragment-lane counterpart of `SqliteVecVectorIndex.release`, and it
|
|
199
|
+
* matters here for the same reason plus one more: a shared-connection
|
|
200
|
+
* deployment — the case `create({ database })` exists for — holds a record index
|
|
201
|
+
* *and* a fragment index over one connection, so it carries two instances of the
|
|
202
|
+
* statement-lifetime shape rather than one. Both must be released.
|
|
203
|
+
*
|
|
204
|
+
* `better-sqlite3` exposes no public `finalize()`, so dropping the last
|
|
205
|
+
* reference does not finalize a statement — it makes it collectable *earlier*,
|
|
206
|
+
* while the environment is alive, rather than surviving to process teardown.
|
|
207
|
+
* That narrows the window in which `Statement::~Statement()` runs against a
|
|
208
|
+
* torn-down environment; it is not a proof against it.
|
|
209
|
+
*
|
|
210
|
+
* **Call this before closing a connection you own.**
|
|
211
|
+
* {@link SqliteVecFragmentIndex.open}'s handle does it for you.
|
|
212
|
+
*
|
|
213
|
+
* Idempotent. After it, every member fails (or, for the two counts, throws)
|
|
214
|
+
* rather than answering.
|
|
215
|
+
*/
|
|
216
|
+
release() {
|
|
217
|
+
this._released = true;
|
|
218
|
+
this._stmts = undefined;
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* Throw if this index has been released. The members that call it and cannot
|
|
222
|
+
* return a `Result` are the two counts; the rest convert the throw via
|
|
223
|
+
* `captureResult`.
|
|
224
|
+
*/
|
|
225
|
+
_assertUsable(what) {
|
|
226
|
+
if (this._released) {
|
|
227
|
+
throw new Error(`fragment index: cannot ${what}: the index has been released`);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
121
230
|
/** {@inheritDoc IFragmentVectorIndex.addFragments} */
|
|
122
231
|
addFragments(target, fragments) {
|
|
123
232
|
const key = (0, ts_agent_memory_1.edgeTargetKey)(target);
|
|
233
|
+
if (this._released) {
|
|
234
|
+
return Promise.resolve((0, ts_utils_1.fail)(`fragment index: cannot add fragments for '${key}': the index has been released`));
|
|
235
|
+
}
|
|
124
236
|
// Validate every fragment before touching the database, so a bad fragment never
|
|
125
237
|
// leaves the record half-replaced or the dimension half-established (whole-record
|
|
126
238
|
// replace is all-or-nothing). The effective dimension is the established one, or —
|
|
@@ -177,6 +289,7 @@ class SqliteVecFragmentIndex {
|
|
|
177
289
|
/** {@inheritDoc IFragmentVectorIndex.remove} */
|
|
178
290
|
remove(target) {
|
|
179
291
|
return Promise.resolve((0, ts_utils_1.captureResult)(() => {
|
|
292
|
+
this._assertUsable(`remove '${(0, ts_agent_memory_1.edgeTargetKey)(target)}'`);
|
|
180
293
|
// Idempotent: removing a target with no fragments (or before any add created
|
|
181
294
|
// the table) still succeeds.
|
|
182
295
|
if (this._stmts !== undefined) {
|
|
@@ -188,6 +301,7 @@ class SqliteVecFragmentIndex {
|
|
|
188
301
|
/** {@inheritDoc IFragmentVectorIndex.has} */
|
|
189
302
|
has(target) {
|
|
190
303
|
return Promise.resolve((0, ts_utils_1.captureResult)(() => {
|
|
304
|
+
this._assertUsable(`check '${(0, ts_agent_memory_1.edgeTargetKey)(target)}'`);
|
|
191
305
|
// Before any add has created the table there is nothing held — a truthful
|
|
192
306
|
// `false`, matching `remove`'s idempotence and the zero counts.
|
|
193
307
|
if (this._stmts === undefined) {
|
|
@@ -273,6 +387,9 @@ class SqliteVecFragmentIndex {
|
|
|
273
387
|
* note on `IVectorIndex.rebuild`. Tolerates a table that does not exist yet.
|
|
274
388
|
*/
|
|
275
389
|
_clear() {
|
|
390
|
+
if (this._released) {
|
|
391
|
+
return (0, ts_utils_1.fail)('fragment index: cannot clear: the index has been released');
|
|
392
|
+
}
|
|
276
393
|
if (this._stmts === undefined) {
|
|
277
394
|
return (0, ts_utils_1.succeed)(true);
|
|
278
395
|
}
|
|
@@ -281,7 +398,13 @@ class SqliteVecFragmentIndex {
|
|
|
281
398
|
return (0, ts_utils_1.captureResult)(() => this._db.prepare(`DELETE FROM "${this._table}"`).run()).onSuccess(() => (0, ts_utils_1.succeed)(true));
|
|
282
399
|
}
|
|
283
400
|
/** {@inheritDoc IFragmentVectorIndex.query} */
|
|
284
|
-
query(vector, topK,
|
|
401
|
+
query(vector, topK, options) {
|
|
402
|
+
const maxPerRecord = options === null || options === void 0 ? void 0 : options.maxPerRecord;
|
|
403
|
+
const scope = options === null || options === void 0 ? void 0 : options.scope;
|
|
404
|
+
const id = options === null || options === void 0 ? void 0 : options.id;
|
|
405
|
+
if (this._released) {
|
|
406
|
+
return Promise.resolve((0, ts_utils_1.fail)('fragment index: cannot query: the index has been released'));
|
|
407
|
+
}
|
|
285
408
|
if (topK <= 0 || this._stmts === undefined) {
|
|
286
409
|
return Promise.resolve((0, ts_utils_1.succeed)([]));
|
|
287
410
|
}
|
|
@@ -295,11 +418,33 @@ class SqliteVecFragmentIndex {
|
|
|
295
418
|
// capped record's later fragments are skipped), so fetch the full ranked set
|
|
296
419
|
// and apply the cap + topK cut here — exactly as the in-memory index does.
|
|
297
420
|
// Uncapped, KNN's own `k = topK` is already the answer.
|
|
298
|
-
|
|
421
|
+
// A scope-only narrowing (a versioned kind's per-entity subtree) spans several
|
|
422
|
+
// records, and `target_key` equality cannot express a prefix, so it is applied
|
|
423
|
+
// over the full ranked set below. Correct either way — the caller's `topK` is
|
|
424
|
+
// applied to the NARROWED set, which is the property that matters — but only
|
|
425
|
+
// the single-record case gets the partition push-down.
|
|
426
|
+
const recordKey = scope !== undefined && id !== undefined ? (0, ts_agent_memory_1.edgeTargetKey)({ scope, id }) : undefined;
|
|
427
|
+
// The cap forces the full ranked set ONLY when other records can fill from
|
|
428
|
+
// behind a capped one. Under a single-record narrowing every row belongs to
|
|
429
|
+
// that record, so the result is exactly `min(topK, maxPerRecord, fragments)`
|
|
430
|
+
// and those are the first rows KNN returns — `k = topK` suffices, and
|
|
431
|
+
// expanding to the table-wide `fragmentCount` would ask an
|
|
432
|
+
// already-partition-restricted query for far more rows than it can use.
|
|
433
|
+
const wholeSet = recordKey === undefined && (maxPerRecord !== undefined || scope !== undefined);
|
|
434
|
+
const fetchK = wholeSet
|
|
435
|
+
? Number(stmts.fragmentCount.get().c)
|
|
436
|
+
: topK;
|
|
299
437
|
if (fetchK <= 0) {
|
|
300
438
|
return [];
|
|
301
439
|
}
|
|
302
|
-
const
|
|
440
|
+
const blob = SqliteVecFragmentIndex._toBlob(vector);
|
|
441
|
+
const rows = (recordKey !== undefined
|
|
442
|
+
? stmts.queryScopedToRecord.all(blob, fetchK, recordKey)
|
|
443
|
+
: stmts.query.all(blob, fetchK));
|
|
444
|
+
// The scope prefix every record in `scope` shares. `edgeTargetKey` joins with
|
|
445
|
+
// a NUL, so this cannot collide with a longer scope that merely starts the
|
|
446
|
+
// same way.
|
|
447
|
+
const scopePrefix = scope !== undefined && recordKey === undefined ? `${scope}\0` : undefined;
|
|
303
448
|
// sqlite-vec returns rows ascending by distance (nearest first); score is
|
|
304
449
|
// `1 - cosineDistance`, so this order is already descending score.
|
|
305
450
|
const hits = [];
|
|
@@ -308,6 +453,9 @@ class SqliteVecFragmentIndex {
|
|
|
308
453
|
if (hits.length >= topK) {
|
|
309
454
|
break;
|
|
310
455
|
}
|
|
456
|
+
if (scopePrefix !== undefined && !row.target_key.startsWith(scopePrefix)) {
|
|
457
|
+
continue;
|
|
458
|
+
}
|
|
311
459
|
if (maxPerRecord !== undefined) {
|
|
312
460
|
const used = (_a = perRecord.get(row.target_key)) !== null && _a !== void 0 ? _a : 0;
|
|
313
461
|
if (used >= maxPerRecord) {
|
|
@@ -358,6 +506,12 @@ class SqliteVecFragmentIndex {
|
|
|
358
506
|
},
|
|
359
507
|
query: this._db.prepare(`SELECT target_key, start_off, end_off, fragment_id, distance FROM "${this._table}" ` +
|
|
360
508
|
`WHERE embedding MATCH ? AND k = ?`),
|
|
509
|
+
// The single-record narrowing constrains `target_key`, which is the table's
|
|
510
|
+
// PARTITION KEY — so this is a partition-restricted KNN rather than a scan
|
|
511
|
+
// plus a filter. That is the performance reason this narrowing belongs in the
|
|
512
|
+
// library instead of in a bigger over-fetch on the caller's side.
|
|
513
|
+
queryScopedToRecord: this._db.prepare(`SELECT target_key, start_off, end_off, fragment_id, distance FROM "${this._table}" ` +
|
|
514
|
+
`WHERE embedding MATCH ? AND k = ? AND target_key = ?`),
|
|
361
515
|
fragmentCount: this._db.prepare(`SELECT count(*) AS c FROM "${this._table}"`),
|
|
362
516
|
recordCount: this._db.prepare(`SELECT count(DISTINCT target_key) AS c FROM "${this._table}"`),
|
|
363
517
|
// `LIMIT 1`: membership needs existence, not cardinality.
|