@fgv/ts-agent-memory-sqlite-vec 5.1.0-42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.rush/temp/689b960e7e1a3b63bd68fbb91474d5c28d567861.tar.log +60 -0
- package/.rush/temp/chunked-rush-logs/ts-agent-memory-sqlite-vec.build.chunks.jsonl +9 -0
- package/.rush/temp/operation/build/all.log +9 -0
- package/.rush/temp/operation/build/log-chunks.jsonl +9 -0
- package/.rush/temp/operation/build/state.json +3 -0
- package/.rush/temp/shrinkwrap-deps.json +720 -0
- package/README.md +109 -0
- package/config/api-extractor.json +38 -0
- package/config/jest.config.json +13 -0
- package/config/rig.json +6 -0
- package/dist/index.js +6 -0
- package/dist/index.js.map +1 -0
- package/dist/packlets/sqlite-vec-index/index.js +8 -0
- package/dist/packlets/sqlite-vec-index/index.js.map +1 -0
- package/dist/packlets/sqlite-vec-index/model.js +6 -0
- package/dist/packlets/sqlite-vec-index/model.js.map +1 -0
- package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +277 -0
- package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -0
- package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +182 -0
- package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -0
- package/dist/test/unit/sqliteVecFragmentIndex.test.js +352 -0
- package/dist/test/unit/sqliteVecFragmentIndex.test.js.map +1 -0
- package/dist/test/unit/sqliteVecVectorIndex.test.js +199 -0
- package/dist/test/unit/sqliteVecVectorIndex.test.js.map +1 -0
- package/dist/ts-agent-memory-sqlite-vec.d.ts +225 -0
- package/dist/tsdoc-metadata.json +11 -0
- package/eslint.config.js +15 -0
- package/etc/ts-agent-memory-sqlite-vec.api.md +66 -0
- package/lib/index.d.ts +2 -0
- package/lib/index.d.ts.map +1 -0
- package/lib/index.js +22 -0
- package/lib/index.js.map +1 -0
- package/lib/packlets/sqlite-vec-index/index.d.ts +4 -0
- package/lib/packlets/sqlite-vec-index/index.d.ts.map +1 -0
- package/lib/packlets/sqlite-vec-index/index.js +24 -0
- package/lib/packlets/sqlite-vec-index/index.js.map +1 -0
- package/lib/packlets/sqlite-vec-index/model.d.ts +48 -0
- package/lib/packlets/sqlite-vec-index/model.d.ts.map +1 -0
- package/lib/packlets/sqlite-vec-index/model.js +7 -0
- package/lib/packlets/sqlite-vec-index/model.js.map +1 -0
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts +94 -0
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts.map +1 -0
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +281 -0
- package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -0
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts +80 -0
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts.map +1 -0
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +186 -0
- package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -0
- package/lib/test/unit/sqliteVecFragmentIndex.test.d.ts +2 -0
- package/lib/test/unit/sqliteVecFragmentIndex.test.d.ts.map +1 -0
- package/lib/test/unit/sqliteVecFragmentIndex.test.js +390 -0
- package/lib/test/unit/sqliteVecFragmentIndex.test.js.map +1 -0
- package/lib/test/unit/sqliteVecVectorIndex.test.d.ts +2 -0
- package/lib/test/unit/sqliteVecVectorIndex.test.d.ts.map +1 -0
- package/lib/test/unit/sqliteVecVectorIndex.test.js +237 -0
- package/lib/test/unit/sqliteVecVectorIndex.test.js.map +1 -0
- package/package.json +87 -0
- package/rush-logs/ts-agent-memory-sqlite-vec.build.cache.log +3 -0
- package/rush-logs/ts-agent-memory-sqlite-vec.build.log +9 -0
- package/src/index.ts +6 -0
- package/src/packlets/sqlite-vec-index/index.ts +8 -0
- package/src/packlets/sqlite-vec-index/model.ts +56 -0
- package/src/packlets/sqlite-vec-index/sqliteVecFragmentIndex.ts +381 -0
- package/src/packlets/sqlite-vec-index/sqliteVecVectorIndex.ts +255 -0
- package/src/test/unit/sqliteVecFragmentIndex.test.ts +466 -0
- package/src/test/unit/sqliteVecVectorIndex.test.ts +253 -0
- package/temp/build/lint/_eslint-5eVG3S6w.json +34 -0
- package/temp/build/typescript/ts_8nwakTlr.json +1 -0
- package/temp/ts-agent-memory-sqlite-vec.api.json +1167 -0
- package/temp/ts-agent-memory-sqlite-vec.api.md +66 -0
- package/tsconfig.json +8 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
## API Report File for "@fgv/ts-agent-memory-sqlite-vec"
|
|
2
|
+
|
|
3
|
+
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
|
4
|
+
|
|
5
|
+
```ts
|
|
6
|
+
|
|
7
|
+
import type BetterSqlite3 from 'better-sqlite3';
|
|
8
|
+
import { IEdgeTarget } from '@fgv/ts-agent-memory';
|
|
9
|
+
import { IEmbeddedFragment } from '@fgv/ts-agent-memory';
|
|
10
|
+
import { IFragmentVectorIndex } from '@fgv/ts-agent-memory';
|
|
11
|
+
import { IVectorIndex } from '@fgv/ts-agent-memory';
|
|
12
|
+
import { IVectorQueryHit } from '@fgv/ts-agent-memory';
|
|
13
|
+
import { Result } from '@fgv/ts-utils';
|
|
14
|
+
|
|
15
|
+
// @public
|
|
16
|
+
export interface ISqliteVecFragmentIndexCreateParams {
|
|
17
|
+
readonly database: BetterSqlite3.Database;
|
|
18
|
+
readonly tableName?: string;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// @public
|
|
22
|
+
export interface ISqliteVecVectorIndexCreateParams {
|
|
23
|
+
readonly database: BetterSqlite3.Database;
|
|
24
|
+
readonly tableName?: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// @public
|
|
28
|
+
export class SqliteVecFragmentIndex implements IFragmentVectorIndex {
|
|
29
|
+
// Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IFragmentVectorIndex"
|
|
30
|
+
//
|
|
31
|
+
// (undocumented)
|
|
32
|
+
addFragments(target: IEdgeTarget, fragments: ReadonlyArray<IEmbeddedFragment>): Promise<Result<number>>;
|
|
33
|
+
static create(params: ISqliteVecFragmentIndexCreateParams): Promise<Result<SqliteVecFragmentIndex>>;
|
|
34
|
+
get fragmentCount(): number;
|
|
35
|
+
// Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IFragmentVectorIndex"
|
|
36
|
+
//
|
|
37
|
+
// (undocumented)
|
|
38
|
+
query(vector: Float32Array, topK: number, maxPerRecord?: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
|
|
39
|
+
get recordCount(): number;
|
|
40
|
+
// Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IFragmentVectorIndex"
|
|
41
|
+
//
|
|
42
|
+
// (undocumented)
|
|
43
|
+
remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// @public
|
|
47
|
+
export class SqliteVecVectorIndex implements IVectorIndex {
|
|
48
|
+
// Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IVectorIndex"
|
|
49
|
+
//
|
|
50
|
+
// (undocumented)
|
|
51
|
+
add(target: IEdgeTarget, vector: Float32Array): Promise<Result<string>>;
|
|
52
|
+
static create(params: ISqliteVecVectorIndexCreateParams): Promise<Result<SqliteVecVectorIndex>>;
|
|
53
|
+
// Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IVectorIndex"
|
|
54
|
+
//
|
|
55
|
+
// (undocumented)
|
|
56
|
+
query(vector: Float32Array, topK: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
|
|
57
|
+
// Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IVectorIndex"
|
|
58
|
+
//
|
|
59
|
+
// (undocumented)
|
|
60
|
+
remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
|
|
61
|
+
get size(): number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// (No @packageDocumentation comment for this package)
|
|
65
|
+
|
|
66
|
+
```
|
package/lib/index.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,cAAc,6BAA6B,CAAC"}
|
package/lib/index.js
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright (c) 2026 Erik Fortune
|
|
4
|
+
* SPDX-License-Identifier: MIT
|
|
5
|
+
*/
|
|
6
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
7
|
+
if (k2 === undefined) k2 = k;
|
|
8
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
9
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
10
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
11
|
+
}
|
|
12
|
+
Object.defineProperty(o, k2, desc);
|
|
13
|
+
}) : (function(o, m, k, k2) {
|
|
14
|
+
if (k2 === undefined) k2 = k;
|
|
15
|
+
o[k2] = m[k];
|
|
16
|
+
}));
|
|
17
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
18
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
19
|
+
};
|
|
20
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
+
__exportStar(require("./packlets/sqlite-vec-index"), exports);
|
|
22
|
+
//# sourceMappingURL=index.js.map
|
package/lib/index.js.map
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;AAEH,8DAA4C","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nexport * from './packlets/sqlite-vec-index';\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/index.ts"],"names":[],"mappings":"AAKA,cAAc,SAAS,CAAC;AACxB,cAAc,wBAAwB,CAAC;AACvC,cAAc,0BAA0B,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright (c) 2026 Erik Fortune
|
|
4
|
+
* SPDX-License-Identifier: MIT
|
|
5
|
+
*/
|
|
6
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
7
|
+
if (k2 === undefined) k2 = k;
|
|
8
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
9
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
10
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
11
|
+
}
|
|
12
|
+
Object.defineProperty(o, k2, desc);
|
|
13
|
+
}) : (function(o, m, k, k2) {
|
|
14
|
+
if (k2 === undefined) k2 = k;
|
|
15
|
+
o[k2] = m[k];
|
|
16
|
+
}));
|
|
17
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
18
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
19
|
+
};
|
|
20
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
+
__exportStar(require("./model"), exports);
|
|
22
|
+
__exportStar(require("./sqliteVecVectorIndex"), exports);
|
|
23
|
+
__exportStar(require("./sqliteVecFragmentIndex"), exports);
|
|
24
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;AAEH,0CAAwB;AACxB,yDAAuC;AACvC,2DAAyC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nexport * from './model';\nexport * from './sqliteVecVectorIndex';\nexport * from './sqliteVecFragmentIndex';\n"]}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type BetterSqlite3 from 'better-sqlite3';
|
|
2
|
+
/**
|
|
3
|
+
* Parameters for {@link SqliteVecVectorIndex.create}.
|
|
4
|
+
* @public
|
|
5
|
+
*/
|
|
6
|
+
export interface ISqliteVecVectorIndexCreateParams {
|
|
7
|
+
/**
|
|
8
|
+
* A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring
|
|
9
|
+
* the boundary-package convention). The consumer opens it (`new Database(path)`
|
|
10
|
+
* for a persistent file, or `new Database(':memory:')` for an ephemeral index)
|
|
11
|
+
* and owns its lifecycle — this index never closes it. `create` loads the
|
|
12
|
+
* `sqlite-vec` extension onto the connection and, if the vector table already
|
|
13
|
+
* exists (a reopened persistent file), recovers its established dimension so no
|
|
14
|
+
* re-embedding is required on open.
|
|
15
|
+
*/
|
|
16
|
+
readonly database: BetterSqlite3.Database;
|
|
17
|
+
/**
|
|
18
|
+
* Name of the `vec0` virtual table that holds the embeddings. Must be a simple
|
|
19
|
+
* SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to `'memory_vectors'`.
|
|
20
|
+
* Supply a distinct name to hold more than one independent index in a single
|
|
21
|
+
* database file.
|
|
22
|
+
*/
|
|
23
|
+
readonly tableName?: string;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Parameters for {@link SqliteVecFragmentIndex.create}.
|
|
27
|
+
* @public
|
|
28
|
+
*/
|
|
29
|
+
export interface ISqliteVecFragmentIndexCreateParams {
|
|
30
|
+
/**
|
|
31
|
+
* A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring
|
|
32
|
+
* {@link ISqliteVecVectorIndexCreateParams.database}). The consumer opens it and
|
|
33
|
+
* owns its lifecycle — this index never closes it. `create` loads the
|
|
34
|
+
* `sqlite-vec` extension onto the connection and, if the fragment table already
|
|
35
|
+
* exists (a reopened persistent file), recovers its established dimension so no
|
|
36
|
+
* re-embedding is required on open.
|
|
37
|
+
*/
|
|
38
|
+
readonly database: BetterSqlite3.Database;
|
|
39
|
+
/**
|
|
40
|
+
* Name of the `vec0` virtual table that holds the fragment embeddings. Must be a
|
|
41
|
+
* simple SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to
|
|
42
|
+
* `'memory_fragments'`. Supply a distinct name (distinct from any record-level
|
|
43
|
+
* {@link SqliteVecVectorIndex} table) to hold more than one independent index in
|
|
44
|
+
* a single database file.
|
|
45
|
+
*/
|
|
46
|
+
readonly tableName?: string;
|
|
47
|
+
}
|
|
48
|
+
//# sourceMappingURL=model.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/model.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,aAAa,MAAM,gBAAgB,CAAC;AAEhD;;;GAGG;AACH,MAAM,WAAW,iCAAiC;IAChD;;;;;;;;OAQG;IACH,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC;IAE1C;;;;;OAKG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;GAGG;AACH,MAAM,WAAW,mCAAmC;IAClD;;;;;;;OAOG;IACH,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC;IAE1C;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/model.ts"],"names":[],"mappings":";AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport type BetterSqlite3 from 'better-sqlite3';\n\n/**\n * Parameters for {@link SqliteVecVectorIndex.create}.\n * @public\n */\nexport interface ISqliteVecVectorIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * the boundary-package convention). The consumer opens it (`new Database(path)`\n * for a persistent file, or `new Database(':memory:')` for an ephemeral index)\n * and owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the vector table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the embeddings. Must be a simple\n * SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to `'memory_vectors'`.\n * Supply a distinct name to hold more than one independent index in a single\n * database file.\n */\n readonly tableName?: string;\n}\n\n/**\n * Parameters for {@link SqliteVecFragmentIndex.create}.\n * @public\n */\nexport interface ISqliteVecFragmentIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * {@link ISqliteVecVectorIndexCreateParams.database}). The consumer opens it and\n * owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the fragment table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the fragment embeddings. Must be a\n * simple SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to\n * `'memory_fragments'`. Supply a distinct name (distinct from any record-level\n * {@link SqliteVecVectorIndex} table) to hold more than one independent index in\n * a single database file.\n */\n readonly tableName?: string;\n}\n"]}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
import { Result } from '@fgv/ts-utils';
|
|
2
|
+
import { IEdgeTarget, IEmbeddedFragment, IFragmentVectorIndex, IVectorQueryHit } from '@fgv/ts-agent-memory';
|
|
3
|
+
import { ISqliteVecFragmentIndexCreateParams } from './model';
|
|
4
|
+
/**
|
|
5
|
+
* A persistent, `sqlite-vec`-backed `IFragmentVectorIndex` (from
|
|
6
|
+
* `@fgv/ts-agent-memory`) — the fragment-granular sibling of
|
|
7
|
+
* {@link SqliteVecVectorIndex}, and the **durable** counterpart to the in-memory
|
|
8
|
+
* `InMemoryFragmentCosineIndex`.
|
|
9
|
+
*
|
|
10
|
+
* @remarks
|
|
11
|
+
* Where {@link SqliteVecVectorIndex} keys one vector per record on a
|
|
12
|
+
* `target_key` primary key, this index holds **many** vectors per record — one per
|
|
13
|
+
* in-record `[start, end)` span — so it keys the `vec0` table on `target_key` as a
|
|
14
|
+
* **`PARTITION KEY`** (many rows may share it) and stores each fragment's locator
|
|
15
|
+
* offsets in two auxiliary columns (`+start_off`, `+end_off`) that ride alongside
|
|
16
|
+
* the vector and are returned on query but never filtered. A query is a brute-force
|
|
17
|
+
* `vec0` KNN scan across all partitions returning per-fragment hits, each carrying
|
|
18
|
+
* its record `target` and the matched `locator`.
|
|
19
|
+
*
|
|
20
|
+
* Semantics match `InMemoryFragmentCosineIndex` exactly: `addFragments` is
|
|
21
|
+
* whole-record-replace (a single transaction deletes every prior fragment of the
|
|
22
|
+
* target, then inserts the new set), `remove` drops every fragment of a target,
|
|
23
|
+
* and `query` applies the optional `maxPerRecord` cap **during selection, before
|
|
24
|
+
* the topK cut** — so one long document cannot crowd others out. The dimension is
|
|
25
|
+
* established by the first `addFragments` (the `vec0` column is fixed-width) and
|
|
26
|
+
* recovered from the table schema when a persistent file is reopened; similarity is
|
|
27
|
+
* cosine (`score = 1 - cosineDistance`), byte-identical to the in-memory index.
|
|
28
|
+
* Large-N ANN indexing is explicitly out of scope, same regime as the record index.
|
|
29
|
+
*
|
|
30
|
+
* The `better-sqlite3` `Database` is consumer-owned (bring-your-own): this index
|
|
31
|
+
* loads the `sqlite-vec` extension onto it and reads/writes the table, but never
|
|
32
|
+
* opens or closes the connection.
|
|
33
|
+
* @public
|
|
34
|
+
*/
|
|
35
|
+
export declare class SqliteVecFragmentIndex implements IFragmentVectorIndex {
|
|
36
|
+
private readonly _db;
|
|
37
|
+
private readonly _table;
|
|
38
|
+
/** The dimension of every stored fragment vector; `undefined` until the table exists. */
|
|
39
|
+
private _dimension;
|
|
40
|
+
/** Prepared statements; created once the table exists (established or recovered). */
|
|
41
|
+
private _stmts;
|
|
42
|
+
private constructor();
|
|
43
|
+
/** The number of records that currently have at least one stored fragment. Zero before the first add. */
|
|
44
|
+
get recordCount(): number;
|
|
45
|
+
/** The total number of fragments currently held across all records. Zero before the first add. */
|
|
46
|
+
get fragmentCount(): number;
|
|
47
|
+
/**
|
|
48
|
+
* Family-convention factory. Loads the `sqlite-vec` extension onto the supplied
|
|
49
|
+
* `better-sqlite3` connection and, if the fragment table already exists (a
|
|
50
|
+
* reopened persistent file), recovers its established dimension so no
|
|
51
|
+
* re-embedding is needed on open.
|
|
52
|
+
*
|
|
53
|
+
* @param params - See {@link ISqliteVecFragmentIndexCreateParams}.
|
|
54
|
+
* @returns `Success` with the index, or `Failure` if the table name is not a
|
|
55
|
+
* simple identifier or the extension fails to load.
|
|
56
|
+
*/
|
|
57
|
+
static create(params: ISqliteVecFragmentIndexCreateParams): Promise<Result<SqliteVecFragmentIndex>>;
|
|
58
|
+
/** {@inheritDoc IFragmentVectorIndex.addFragments} */
|
|
59
|
+
addFragments(target: IEdgeTarget, fragments: ReadonlyArray<IEmbeddedFragment>): Promise<Result<number>>;
|
|
60
|
+
/** {@inheritDoc IFragmentVectorIndex.remove} */
|
|
61
|
+
remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
|
|
62
|
+
/** {@inheritDoc IFragmentVectorIndex.query} */
|
|
63
|
+
query(vector: Float32Array, topK: number, maxPerRecord?: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
|
|
64
|
+
/** Create the fragment `vec0` virtual table with the established dimension. */
|
|
65
|
+
private _createTable;
|
|
66
|
+
/** Prepare the statements the index reuses. Requires the table to exist. */
|
|
67
|
+
private _prepare;
|
|
68
|
+
/**
|
|
69
|
+
* Recover the established dimension of an existing fragment `vec0` table from its
|
|
70
|
+
* stored `CREATE VIRTUAL TABLE` SQL (`float[<n>]`). Returns `undefined` when the
|
|
71
|
+
* table does not exist yet (a fresh database — dimension is set by the first add).
|
|
72
|
+
*/
|
|
73
|
+
private static _readExistingDimension;
|
|
74
|
+
/** Pack a `Float32Array` as the little-endian byte blob `vec0` stores. Copies, so the caller may reuse its buffer. */
|
|
75
|
+
private static _toBlob;
|
|
76
|
+
/**
|
|
77
|
+
* Reverse `edgeTargetKey` — the canonical key is `scope\0id` with NUL excluded
|
|
78
|
+
* from both components, so the first NUL splits it unambiguously. A key with no
|
|
79
|
+
* NUL cannot have been written by `edgeTargetKey`; rather than fabricate a wrong
|
|
80
|
+
* `(scope, id)` from corrupt / externally-edited table data, throw so the query
|
|
81
|
+
* surfaces it as a loud `Failure`.
|
|
82
|
+
*/
|
|
83
|
+
private static _parseKey;
|
|
84
|
+
/**
|
|
85
|
+
* Coerce a persisted locator offset to a plain `number`. `better-sqlite3` returns
|
|
86
|
+
* integer columns as `bigint` under safe-integer mode, so an offset can arrive as
|
|
87
|
+
* either; both narrow to `number` here. A value outside the safe-integer range
|
|
88
|
+
* (only reachable via corrupt / externally-edited data — the index only ever
|
|
89
|
+
* writes in-document offsets) throws rather than silently losing precision, so the
|
|
90
|
+
* query surfaces it as a loud `Failure`.
|
|
91
|
+
*/
|
|
92
|
+
private static _toOffset;
|
|
93
|
+
}
|
|
94
|
+
//# sourceMappingURL=sqliteVecFragmentIndex.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sqliteVecFragmentIndex.d.ts","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/sqliteVecFragmentIndex.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,MAAM,EAAgC,MAAM,eAAe,CAAC;AACrE,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,oBAAoB,EACpB,eAAe,EAIhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,mCAAmC,EAAE,MAAM,SAAS,CAAC;AAsB9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,sBAAuB,YAAW,oBAAoB;IACjE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAyB;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,yFAAyF;IACzF,OAAO,CAAC,UAAU,CAAqB;IACvC,qFAAqF;IACrF,OAAO,CAAC,MAAM,CAAkC;IAEhD,OAAO;IAOP,yGAAyG;IACzG,IAAW,WAAW,IAAI,MAAM,CAO/B;IAED,kGAAkG;IAClG,IAAW,aAAa,IAAI,MAAM,CAKjC;IAED;;;;;;;;;OASG;WACW,MAAM,CAAC,MAAM,EAAE,mCAAmC,GAAG,OAAO,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAmB1G,sDAAsD;IAC/C,YAAY,CACjB,MAAM,EAAE,WAAW,EACnB,SAAS,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAC1C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAyD1B,gDAAgD;IACzC,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAahE,+CAA+C;IACxC,KAAK,CACV,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,MAAM,EACZ,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC;IAyDlD,+EAA+E;IAC/E,OAAO,CAAC,YAAY;IAQpB,4EAA4E;IAC5E,OAAO,CAAC,QAAQ;IAoChB;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAWrC,sHAAsH;IACtH,OAAO,CAAC,MAAM,CAAC,OAAO;IAItB;;;;;;OAMG;IACH,OAAO,CAAC,MAAM,CAAC,SAAS;IAWxB;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM,CAAC,SAAS;CASzB"}
|
|
@@ -0,0 +1,281 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright (c) 2026 Erik Fortune
|
|
4
|
+
* SPDX-License-Identifier: MIT
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.SqliteVecFragmentIndex = void 0;
|
|
8
|
+
const sqlite_vec_1 = require("sqlite-vec");
|
|
9
|
+
const ts_utils_1 = require("@fgv/ts-utils");
|
|
10
|
+
const ts_agent_memory_1 = require("@fgv/ts-agent-memory");
|
|
11
|
+
/** Default name for the fragment `vec0` virtual table. */
|
|
12
|
+
const DEFAULT_TABLE_NAME = 'memory_fragments';
|
|
13
|
+
/** A simple SQL identifier — the only shape allowed for the table name (it is interpolated into DDL). */
|
|
14
|
+
const IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
15
|
+
/**
|
|
16
|
+
* A persistent, `sqlite-vec`-backed `IFragmentVectorIndex` (from
|
|
17
|
+
* `@fgv/ts-agent-memory`) — the fragment-granular sibling of
|
|
18
|
+
* {@link SqliteVecVectorIndex}, and the **durable** counterpart to the in-memory
|
|
19
|
+
* `InMemoryFragmentCosineIndex`.
|
|
20
|
+
*
|
|
21
|
+
* @remarks
|
|
22
|
+
* Where {@link SqliteVecVectorIndex} keys one vector per record on a
|
|
23
|
+
* `target_key` primary key, this index holds **many** vectors per record — one per
|
|
24
|
+
* in-record `[start, end)` span — so it keys the `vec0` table on `target_key` as a
|
|
25
|
+
* **`PARTITION KEY`** (many rows may share it) and stores each fragment's locator
|
|
26
|
+
* offsets in two auxiliary columns (`+start_off`, `+end_off`) that ride alongside
|
|
27
|
+
* the vector and are returned on query but never filtered. A query is a brute-force
|
|
28
|
+
* `vec0` KNN scan across all partitions returning per-fragment hits, each carrying
|
|
29
|
+
* its record `target` and the matched `locator`.
|
|
30
|
+
*
|
|
31
|
+
* Semantics match `InMemoryFragmentCosineIndex` exactly: `addFragments` is
|
|
32
|
+
* whole-record-replace (a single transaction deletes every prior fragment of the
|
|
33
|
+
* target, then inserts the new set), `remove` drops every fragment of a target,
|
|
34
|
+
* and `query` applies the optional `maxPerRecord` cap **during selection, before
|
|
35
|
+
* the topK cut** — so one long document cannot crowd others out. The dimension is
|
|
36
|
+
* established by the first `addFragments` (the `vec0` column is fixed-width) and
|
|
37
|
+
* recovered from the table schema when a persistent file is reopened; similarity is
|
|
38
|
+
* cosine (`score = 1 - cosineDistance`), byte-identical to the in-memory index.
|
|
39
|
+
* Large-N ANN indexing is explicitly out of scope, same regime as the record index.
|
|
40
|
+
*
|
|
41
|
+
* The `better-sqlite3` `Database` is consumer-owned (bring-your-own): this index
|
|
42
|
+
* loads the `sqlite-vec` extension onto it and reads/writes the table, but never
|
|
43
|
+
* opens or closes the connection.
|
|
44
|
+
* @public
|
|
45
|
+
*/
|
|
46
|
+
class SqliteVecFragmentIndex {
|
|
47
|
+
constructor(db, table, dimension) {
|
|
48
|
+
this._db = db;
|
|
49
|
+
this._table = table;
|
|
50
|
+
this._dimension = dimension;
|
|
51
|
+
this._stmts = dimension === undefined ? undefined : this._prepare();
|
|
52
|
+
}
|
|
53
|
+
/** The number of records that currently have at least one stored fragment. Zero before the first add. */
|
|
54
|
+
get recordCount() {
|
|
55
|
+
if (this._stmts === undefined) {
|
|
56
|
+
return 0;
|
|
57
|
+
}
|
|
58
|
+
// `Number(...)` narrows the count in case the consumer enabled better-sqlite3
|
|
59
|
+
// safe-integer mode (which returns `count(*)` as a `bigint`).
|
|
60
|
+
return Number(this._stmts.recordCount.get().c);
|
|
61
|
+
}
|
|
62
|
+
/** The total number of fragments currently held across all records. Zero before the first add. */
|
|
63
|
+
get fragmentCount() {
|
|
64
|
+
if (this._stmts === undefined) {
|
|
65
|
+
return 0;
|
|
66
|
+
}
|
|
67
|
+
return Number(this._stmts.fragmentCount.get().c);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Family-convention factory. Loads the `sqlite-vec` extension onto the supplied
|
|
71
|
+
* `better-sqlite3` connection and, if the fragment table already exists (a
|
|
72
|
+
* reopened persistent file), recovers its established dimension so no
|
|
73
|
+
* re-embedding is needed on open.
|
|
74
|
+
*
|
|
75
|
+
* @param params - See {@link ISqliteVecFragmentIndexCreateParams}.
|
|
76
|
+
* @returns `Success` with the index, or `Failure` if the table name is not a
|
|
77
|
+
* simple identifier or the extension fails to load.
|
|
78
|
+
*/
|
|
79
|
+
static create(params) {
|
|
80
|
+
var _a;
|
|
81
|
+
const table = (_a = params.tableName) !== null && _a !== void 0 ? _a : DEFAULT_TABLE_NAME;
|
|
82
|
+
if (!IDENTIFIER_RE.test(table)) {
|
|
83
|
+
return Promise.resolve((0, ts_utils_1.fail)(`sqlite-vec fragment index: table name '${table}' is not a simple SQL identifier`));
|
|
84
|
+
}
|
|
85
|
+
return Promise.resolve((0, ts_utils_1.captureResult)(() => {
|
|
86
|
+
(0, sqlite_vec_1.load)(params.database);
|
|
87
|
+
const dimension = SqliteVecFragmentIndex._readExistingDimension(params.database, table);
|
|
88
|
+
return new SqliteVecFragmentIndex(params.database, table, dimension);
|
|
89
|
+
}).withErrorFormat((e) => `sqlite-vec fragment index: failed to initialize: ${e}`));
|
|
90
|
+
}
|
|
91
|
+
/** {@inheritDoc IFragmentVectorIndex.addFragments} */
|
|
92
|
+
addFragments(target, fragments) {
|
|
93
|
+
const key = (0, ts_agent_memory_1.edgeTargetKey)(target);
|
|
94
|
+
// Validate every fragment before touching the database, so a bad fragment never
|
|
95
|
+
// leaves the record half-replaced or the dimension half-established (whole-record
|
|
96
|
+
// replace is all-or-nothing). The effective dimension is the established one, or —
|
|
97
|
+
// on a still-dimensionless index — the first fragment's length; it is committed
|
|
98
|
+
// (via table creation) only once the whole batch validates.
|
|
99
|
+
let dimension = this._dimension;
|
|
100
|
+
for (const fragment of fragments) {
|
|
101
|
+
if (fragment.vector.length === 0) {
|
|
102
|
+
return Promise.resolve((0, ts_utils_1.fail)(`fragment index: cannot add '${key}': empty fragment vector`));
|
|
103
|
+
}
|
|
104
|
+
// Locator offsets are persisted as SQLite integers (bound via BigInt). Reject a
|
|
105
|
+
// non-safe-integer offset up front with a clear message, rather than letting
|
|
106
|
+
// `BigInt(nonInteger)` throw cryptically inside the write transaction OR storing
|
|
107
|
+
// a value the read-side `_toOffset` guard would later reject on every query.
|
|
108
|
+
if (!Number.isSafeInteger(fragment.locator.start) || !Number.isSafeInteger(fragment.locator.end)) {
|
|
109
|
+
return Promise.resolve((0, ts_utils_1.fail)(`fragment index: cannot add '${key}': locator [${fragment.locator.start}, ${fragment.locator.end}) offsets must be safe integers`));
|
|
110
|
+
}
|
|
111
|
+
if (dimension === undefined) {
|
|
112
|
+
dimension = fragment.vector.length;
|
|
113
|
+
}
|
|
114
|
+
else if (fragment.vector.length !== dimension) {
|
|
115
|
+
return Promise.resolve((0, ts_utils_1.fail)(`fragment index: cannot add '${key}': fragment dimension ${fragment.vector.length} does not match index dimension ${dimension}`));
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return Promise.resolve((0, ts_utils_1.captureResult)(() => {
|
|
119
|
+
// A same-target re-author (or an empty batch) still needs the table to exist
|
|
120
|
+
// to delete prior fragments; create it lazily on the first non-empty add.
|
|
121
|
+
if (this._stmts === undefined) {
|
|
122
|
+
if (fragments.length === 0) {
|
|
123
|
+
// Nothing stored yet and nothing to store: no table, no work.
|
|
124
|
+
return 0;
|
|
125
|
+
}
|
|
126
|
+
// `fragments` is non-empty here (the empty case returned above), so the
|
|
127
|
+
// validation loop proved every fragment shares `fragments[0]`'s length —
|
|
128
|
+
// which IS the dimension to establish. Read it straight from the first
|
|
129
|
+
// fragment: no cast, no invariant-dependent narrowing.
|
|
130
|
+
const established = fragments[0].vector.length;
|
|
131
|
+
this._createTable(established);
|
|
132
|
+
this._dimension = established;
|
|
133
|
+
this._stmts = this._prepare();
|
|
134
|
+
}
|
|
135
|
+
this._stmts.replace(key, fragments);
|
|
136
|
+
return fragments.length;
|
|
137
|
+
}).withErrorFormat((e) => `fragment index: cannot add '${key}': ${e}`));
|
|
138
|
+
}
|
|
139
|
+
/** {@inheritDoc IFragmentVectorIndex.remove} */
|
|
140
|
+
remove(target) {
|
|
141
|
+
return Promise.resolve((0, ts_utils_1.captureResult)(() => {
|
|
142
|
+
// Idempotent: removing a target with no fragments (or before any add created
|
|
143
|
+
// the table) still succeeds.
|
|
144
|
+
if (this._stmts !== undefined) {
|
|
145
|
+
this._stmts.deleteByTarget.run((0, ts_agent_memory_1.edgeTargetKey)(target));
|
|
146
|
+
}
|
|
147
|
+
return target;
|
|
148
|
+
}).withErrorFormat((e) => `fragment index: cannot remove '${(0, ts_agent_memory_1.edgeTargetKey)(target)}': ${e}`));
|
|
149
|
+
}
|
|
150
|
+
/** {@inheritDoc IFragmentVectorIndex.query} */
|
|
151
|
+
query(vector, topK, maxPerRecord) {
|
|
152
|
+
if (topK <= 0 || this._stmts === undefined) {
|
|
153
|
+
return Promise.resolve((0, ts_utils_1.succeed)([]));
|
|
154
|
+
}
|
|
155
|
+
if (vector.length !== this._dimension) {
|
|
156
|
+
return Promise.resolve((0, ts_utils_1.fail)(`fragment index: query dimension ${vector.length} does not match index dimension ${this._dimension}`));
|
|
157
|
+
}
|
|
158
|
+
const stmts = this._stmts;
|
|
159
|
+
return Promise.resolve((0, ts_utils_1.captureResult)(() => {
|
|
160
|
+
var _a;
|
|
161
|
+
// With a per-record cap the topK winners may lie past the first topK rows (a
|
|
162
|
+
// capped record's later fragments are skipped), so fetch the full ranked set
|
|
163
|
+
// and apply the cap + topK cut here — exactly as the in-memory index does.
|
|
164
|
+
// Uncapped, KNN's own `k = topK` is already the answer.
|
|
165
|
+
const fetchK = maxPerRecord === undefined ? topK : Number(stmts.fragmentCount.get().c);
|
|
166
|
+
if (fetchK <= 0) {
|
|
167
|
+
return [];
|
|
168
|
+
}
|
|
169
|
+
const rows = stmts.query.all(SqliteVecFragmentIndex._toBlob(vector), fetchK);
|
|
170
|
+
// sqlite-vec returns rows ascending by distance (nearest first); score is
|
|
171
|
+
// `1 - cosineDistance`, so this order is already descending score.
|
|
172
|
+
const hits = [];
|
|
173
|
+
const perRecord = new Map();
|
|
174
|
+
for (const row of rows) {
|
|
175
|
+
if (hits.length >= topK) {
|
|
176
|
+
break;
|
|
177
|
+
}
|
|
178
|
+
if (maxPerRecord !== undefined) {
|
|
179
|
+
const used = (_a = perRecord.get(row.target_key)) !== null && _a !== void 0 ? _a : 0;
|
|
180
|
+
if (used >= maxPerRecord) {
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
perRecord.set(row.target_key, used + 1);
|
|
184
|
+
}
|
|
185
|
+
const key = row.target_key;
|
|
186
|
+
hits.push({
|
|
187
|
+
target: SqliteVecFragmentIndex._parseKey(key),
|
|
188
|
+
score: 1 - row.distance,
|
|
189
|
+
locator: {
|
|
190
|
+
start: SqliteVecFragmentIndex._toOffset(row.start_off, key),
|
|
191
|
+
end: SqliteVecFragmentIndex._toOffset(row.end_off, key)
|
|
192
|
+
}
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
return hits;
|
|
196
|
+
}).withErrorFormat((e) => `fragment index: query failed: ${e}`));
|
|
197
|
+
}
|
|
198
|
+
/** Create the fragment `vec0` virtual table with the established dimension. */
|
|
199
|
+
_createTable(dimension) {
|
|
200
|
+
this._db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS "${this._table}" USING vec0(` +
|
|
201
|
+
`target_key TEXT PARTITION KEY, embedding float[${dimension}] distance_metric=cosine, ` +
|
|
202
|
+
`+start_off integer, +end_off integer)`);
|
|
203
|
+
}
|
|
204
|
+
/** Prepare the statements the index reuses. Requires the table to exist. */
|
|
205
|
+
_prepare() {
|
|
206
|
+
const del = this._db.prepare(`DELETE FROM "${this._table}" WHERE target_key = ?`);
|
|
207
|
+
const ins = this._db.prepare(`INSERT INTO "${this._table}"(target_key, embedding, start_off, end_off) VALUES (?, ?, ?, ?)`);
|
|
208
|
+
// Whole-record replace: drop every prior fragment of the target, then insert the
|
|
209
|
+
// new set, atomically. An empty set collapses to a pure delete.
|
|
210
|
+
const replaceTxn = this._db.transaction((key, fragments) => {
|
|
211
|
+
del.run(key);
|
|
212
|
+
for (const fragment of fragments) {
|
|
213
|
+
ins.run(key, SqliteVecFragmentIndex._toBlob(fragment.vector),
|
|
214
|
+
// vec0 typed columns reject a JS float; bind the offsets as integers.
|
|
215
|
+
BigInt(fragment.locator.start), BigInt(fragment.locator.end));
|
|
216
|
+
}
|
|
217
|
+
});
|
|
218
|
+
return {
|
|
219
|
+
deleteByTarget: del,
|
|
220
|
+
replace: (key, fragments) => {
|
|
221
|
+
replaceTxn(key, fragments);
|
|
222
|
+
},
|
|
223
|
+
query: this._db.prepare(`SELECT target_key, start_off, end_off, distance FROM "${this._table}" WHERE embedding MATCH ? AND k = ?`),
|
|
224
|
+
fragmentCount: this._db.prepare(`SELECT count(*) AS c FROM "${this._table}"`),
|
|
225
|
+
recordCount: this._db.prepare(`SELECT count(DISTINCT target_key) AS c FROM "${this._table}"`)
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Recover the established dimension of an existing fragment `vec0` table from its
|
|
230
|
+
* stored `CREATE VIRTUAL TABLE` SQL (`float[<n>]`). Returns `undefined` when the
|
|
231
|
+
* table does not exist yet (a fresh database — dimension is set by the first add).
|
|
232
|
+
*/
|
|
233
|
+
static _readExistingDimension(db, table) {
|
|
234
|
+
const row = db
|
|
235
|
+
.prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
|
|
236
|
+
.get(table);
|
|
237
|
+
if (row === undefined) {
|
|
238
|
+
return undefined;
|
|
239
|
+
}
|
|
240
|
+
const match = row.sql.match(/float\[(\d+)\]/);
|
|
241
|
+
return match === null ? undefined : Number(match[1]);
|
|
242
|
+
}
|
|
243
|
+
/** Pack a `Float32Array` as the little-endian byte blob `vec0` stores. Copies, so the caller may reuse its buffer. */
|
|
244
|
+
static _toBlob(vector) {
|
|
245
|
+
return new Uint8Array(Float32Array.from(vector).buffer);
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* Reverse `edgeTargetKey` — the canonical key is `scope\0id` with NUL excluded
|
|
249
|
+
* from both components, so the first NUL splits it unambiguously. A key with no
|
|
250
|
+
* NUL cannot have been written by `edgeTargetKey`; rather than fabricate a wrong
|
|
251
|
+
* `(scope, id)` from corrupt / externally-edited table data, throw so the query
|
|
252
|
+
* surfaces it as a loud `Failure`.
|
|
253
|
+
*/
|
|
254
|
+
static _parseKey(key) {
|
|
255
|
+
const nul = key.indexOf('\0');
|
|
256
|
+
if (nul < 0) {
|
|
257
|
+
throw new Error(`malformed target key '${key}': missing scope/id separator (corrupt persisted data)`);
|
|
258
|
+
}
|
|
259
|
+
return {
|
|
260
|
+
scope: key.slice(0, nul),
|
|
261
|
+
id: key.slice(nul + 1)
|
|
262
|
+
};
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* Coerce a persisted locator offset to a plain `number`. `better-sqlite3` returns
|
|
266
|
+
* integer columns as `bigint` under safe-integer mode, so an offset can arrive as
|
|
267
|
+
* either; both narrow to `number` here. A value outside the safe-integer range
|
|
268
|
+
* (only reachable via corrupt / externally-edited data — the index only ever
|
|
269
|
+
* writes in-document offsets) throws rather than silently losing precision, so the
|
|
270
|
+
* query surfaces it as a loud `Failure`.
|
|
271
|
+
*/
|
|
272
|
+
static _toOffset(value, key) {
|
|
273
|
+
const n = Number(value);
|
|
274
|
+
if (!Number.isSafeInteger(n)) {
|
|
275
|
+
throw new Error(`fragment '${key}': locator offset ${String(value)} is not a safe integer (corrupt persisted data)`);
|
|
276
|
+
}
|
|
277
|
+
return n;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
exports.SqliteVecFragmentIndex = SqliteVecFragmentIndex;
|
|
281
|
+
//# sourceMappingURL=sqliteVecFragmentIndex.js.map
|