@fgv/ts-agent-memory-sqlite-vec 5.1.0-42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/.rush/temp/689b960e7e1a3b63bd68fbb91474d5c28d567861.tar.log +60 -0
  2. package/.rush/temp/chunked-rush-logs/ts-agent-memory-sqlite-vec.build.chunks.jsonl +9 -0
  3. package/.rush/temp/operation/build/all.log +9 -0
  4. package/.rush/temp/operation/build/log-chunks.jsonl +9 -0
  5. package/.rush/temp/operation/build/state.json +3 -0
  6. package/.rush/temp/shrinkwrap-deps.json +720 -0
  7. package/README.md +109 -0
  8. package/config/api-extractor.json +38 -0
  9. package/config/jest.config.json +13 -0
  10. package/config/rig.json +6 -0
  11. package/dist/index.js +6 -0
  12. package/dist/index.js.map +1 -0
  13. package/dist/packlets/sqlite-vec-index/index.js +8 -0
  14. package/dist/packlets/sqlite-vec-index/index.js.map +1 -0
  15. package/dist/packlets/sqlite-vec-index/model.js +6 -0
  16. package/dist/packlets/sqlite-vec-index/model.js.map +1 -0
  17. package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +277 -0
  18. package/dist/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -0
  19. package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +182 -0
  20. package/dist/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -0
  21. package/dist/test/unit/sqliteVecFragmentIndex.test.js +352 -0
  22. package/dist/test/unit/sqliteVecFragmentIndex.test.js.map +1 -0
  23. package/dist/test/unit/sqliteVecVectorIndex.test.js +199 -0
  24. package/dist/test/unit/sqliteVecVectorIndex.test.js.map +1 -0
  25. package/dist/ts-agent-memory-sqlite-vec.d.ts +225 -0
  26. package/dist/tsdoc-metadata.json +11 -0
  27. package/eslint.config.js +15 -0
  28. package/etc/ts-agent-memory-sqlite-vec.api.md +66 -0
  29. package/lib/index.d.ts +2 -0
  30. package/lib/index.d.ts.map +1 -0
  31. package/lib/index.js +22 -0
  32. package/lib/index.js.map +1 -0
  33. package/lib/packlets/sqlite-vec-index/index.d.ts +4 -0
  34. package/lib/packlets/sqlite-vec-index/index.d.ts.map +1 -0
  35. package/lib/packlets/sqlite-vec-index/index.js +24 -0
  36. package/lib/packlets/sqlite-vec-index/index.js.map +1 -0
  37. package/lib/packlets/sqlite-vec-index/model.d.ts +48 -0
  38. package/lib/packlets/sqlite-vec-index/model.d.ts.map +1 -0
  39. package/lib/packlets/sqlite-vec-index/model.js +7 -0
  40. package/lib/packlets/sqlite-vec-index/model.js.map +1 -0
  41. package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts +94 -0
  42. package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.d.ts.map +1 -0
  43. package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js +281 -0
  44. package/lib/packlets/sqlite-vec-index/sqliteVecFragmentIndex.js.map +1 -0
  45. package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts +80 -0
  46. package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.d.ts.map +1 -0
  47. package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js +186 -0
  48. package/lib/packlets/sqlite-vec-index/sqliteVecVectorIndex.js.map +1 -0
  49. package/lib/test/unit/sqliteVecFragmentIndex.test.d.ts +2 -0
  50. package/lib/test/unit/sqliteVecFragmentIndex.test.d.ts.map +1 -0
  51. package/lib/test/unit/sqliteVecFragmentIndex.test.js +390 -0
  52. package/lib/test/unit/sqliteVecFragmentIndex.test.js.map +1 -0
  53. package/lib/test/unit/sqliteVecVectorIndex.test.d.ts +2 -0
  54. package/lib/test/unit/sqliteVecVectorIndex.test.d.ts.map +1 -0
  55. package/lib/test/unit/sqliteVecVectorIndex.test.js +237 -0
  56. package/lib/test/unit/sqliteVecVectorIndex.test.js.map +1 -0
  57. package/package.json +87 -0
  58. package/rush-logs/ts-agent-memory-sqlite-vec.build.cache.log +3 -0
  59. package/rush-logs/ts-agent-memory-sqlite-vec.build.log +9 -0
  60. package/src/index.ts +6 -0
  61. package/src/packlets/sqlite-vec-index/index.ts +8 -0
  62. package/src/packlets/sqlite-vec-index/model.ts +56 -0
  63. package/src/packlets/sqlite-vec-index/sqliteVecFragmentIndex.ts +381 -0
  64. package/src/packlets/sqlite-vec-index/sqliteVecVectorIndex.ts +255 -0
  65. package/src/test/unit/sqliteVecFragmentIndex.test.ts +466 -0
  66. package/src/test/unit/sqliteVecVectorIndex.test.ts +253 -0
  67. package/temp/build/lint/_eslint-5eVG3S6w.json +34 -0
  68. package/temp/build/typescript/ts_8nwakTlr.json +1 -0
  69. package/temp/ts-agent-memory-sqlite-vec.api.json +1167 -0
  70. package/temp/ts-agent-memory-sqlite-vec.api.md +66 -0
  71. package/tsconfig.json +8 -0
@@ -0,0 +1,66 @@
1
+ ## API Report File for "@fgv/ts-agent-memory-sqlite-vec"
2
+
3
+ > Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
4
+
5
+ ```ts
6
+
7
+ import type BetterSqlite3 from 'better-sqlite3';
8
+ import { IEdgeTarget } from '@fgv/ts-agent-memory';
9
+ import { IEmbeddedFragment } from '@fgv/ts-agent-memory';
10
+ import { IFragmentVectorIndex } from '@fgv/ts-agent-memory';
11
+ import { IVectorIndex } from '@fgv/ts-agent-memory';
12
+ import { IVectorQueryHit } from '@fgv/ts-agent-memory';
13
+ import { Result } from '@fgv/ts-utils';
14
+
15
+ // @public
16
+ export interface ISqliteVecFragmentIndexCreateParams {
17
+ readonly database: BetterSqlite3.Database;
18
+ readonly tableName?: string;
19
+ }
20
+
21
+ // @public
22
+ export interface ISqliteVecVectorIndexCreateParams {
23
+ readonly database: BetterSqlite3.Database;
24
+ readonly tableName?: string;
25
+ }
26
+
27
+ // @public
28
+ export class SqliteVecFragmentIndex implements IFragmentVectorIndex {
29
+ // Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IFragmentVectorIndex"
30
+ //
31
+ // (undocumented)
32
+ addFragments(target: IEdgeTarget, fragments: ReadonlyArray<IEmbeddedFragment>): Promise<Result<number>>;
33
+ static create(params: ISqliteVecFragmentIndexCreateParams): Promise<Result<SqliteVecFragmentIndex>>;
34
+ get fragmentCount(): number;
35
+ // Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IFragmentVectorIndex"
36
+ //
37
+ // (undocumented)
38
+ query(vector: Float32Array, topK: number, maxPerRecord?: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
39
+ get recordCount(): number;
40
+ // Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IFragmentVectorIndex"
41
+ //
42
+ // (undocumented)
43
+ remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
44
+ }
45
+
46
+ // @public
47
+ export class SqliteVecVectorIndex implements IVectorIndex {
48
+ // Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IVectorIndex"
49
+ //
50
+ // (undocumented)
51
+ add(target: IEdgeTarget, vector: Float32Array): Promise<Result<string>>;
52
+ static create(params: ISqliteVecVectorIndexCreateParams): Promise<Result<SqliteVecVectorIndex>>;
53
+ // Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IVectorIndex"
54
+ //
55
+ // (undocumented)
56
+ query(vector: Float32Array, topK: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
57
+ // Warning: (ae-unresolved-inheritdoc-reference) The @inheritDoc reference could not be resolved: The package "@fgv/ts-agent-memory-sqlite-vec" does not have an export "IVectorIndex"
58
+ //
59
+ // (undocumented)
60
+ remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
61
+ get size(): number;
62
+ }
63
+
64
+ // (No @packageDocumentation comment for this package)
65
+
66
+ ```
package/lib/index.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ export * from './packlets/sqlite-vec-index';
2
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAKA,cAAc,6BAA6B,CAAC"}
package/lib/index.js ADDED
@@ -0,0 +1,22 @@
1
+ "use strict";
2
+ /*
3
+ * Copyright (c) 2026 Erik Fortune
4
+ * SPDX-License-Identifier: MIT
5
+ */
6
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
7
+ if (k2 === undefined) k2 = k;
8
+ var desc = Object.getOwnPropertyDescriptor(m, k);
9
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
10
+ desc = { enumerable: true, get: function() { return m[k]; } };
11
+ }
12
+ Object.defineProperty(o, k2, desc);
13
+ }) : (function(o, m, k, k2) {
14
+ if (k2 === undefined) k2 = k;
15
+ o[k2] = m[k];
16
+ }));
17
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
18
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
19
+ };
20
+ Object.defineProperty(exports, "__esModule", { value: true });
21
+ __exportStar(require("./packlets/sqlite-vec-index"), exports);
22
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;AAEH,8DAA4C","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nexport * from './packlets/sqlite-vec-index';\n"]}
@@ -0,0 +1,4 @@
1
+ export * from './model';
2
+ export * from './sqliteVecVectorIndex';
3
+ export * from './sqliteVecFragmentIndex';
4
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/index.ts"],"names":[],"mappings":"AAKA,cAAc,SAAS,CAAC;AACxB,cAAc,wBAAwB,CAAC;AACvC,cAAc,0BAA0B,CAAC"}
@@ -0,0 +1,24 @@
1
+ "use strict";
2
+ /*
3
+ * Copyright (c) 2026 Erik Fortune
4
+ * SPDX-License-Identifier: MIT
5
+ */
6
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
7
+ if (k2 === undefined) k2 = k;
8
+ var desc = Object.getOwnPropertyDescriptor(m, k);
9
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
10
+ desc = { enumerable: true, get: function() { return m[k]; } };
11
+ }
12
+ Object.defineProperty(o, k2, desc);
13
+ }) : (function(o, m, k, k2) {
14
+ if (k2 === undefined) k2 = k;
15
+ o[k2] = m[k];
16
+ }));
17
+ var __exportStar = (this && this.__exportStar) || function(m, exports) {
18
+ for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
19
+ };
20
+ Object.defineProperty(exports, "__esModule", { value: true });
21
+ __exportStar(require("./model"), exports);
22
+ __exportStar(require("./sqliteVecVectorIndex"), exports);
23
+ __exportStar(require("./sqliteVecFragmentIndex"), exports);
24
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;AAEH,0CAAwB;AACxB,yDAAuC;AACvC,2DAAyC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nexport * from './model';\nexport * from './sqliteVecVectorIndex';\nexport * from './sqliteVecFragmentIndex';\n"]}
@@ -0,0 +1,48 @@
1
+ import type BetterSqlite3 from 'better-sqlite3';
2
+ /**
3
+ * Parameters for {@link SqliteVecVectorIndex.create}.
4
+ * @public
5
+ */
6
+ export interface ISqliteVecVectorIndexCreateParams {
7
+ /**
8
+ * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring
9
+ * the boundary-package convention). The consumer opens it (`new Database(path)`
10
+ * for a persistent file, or `new Database(':memory:')` for an ephemeral index)
11
+ * and owns its lifecycle — this index never closes it. `create` loads the
12
+ * `sqlite-vec` extension onto the connection and, if the vector table already
13
+ * exists (a reopened persistent file), recovers its established dimension so no
14
+ * re-embedding is required on open.
15
+ */
16
+ readonly database: BetterSqlite3.Database;
17
+ /**
18
+ * Name of the `vec0` virtual table that holds the embeddings. Must be a simple
19
+ * SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to `'memory_vectors'`.
20
+ * Supply a distinct name to hold more than one independent index in a single
21
+ * database file.
22
+ */
23
+ readonly tableName?: string;
24
+ }
25
+ /**
26
+ * Parameters for {@link SqliteVecFragmentIndex.create}.
27
+ * @public
28
+ */
29
+ export interface ISqliteVecFragmentIndexCreateParams {
30
+ /**
31
+ * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring
32
+ * {@link ISqliteVecVectorIndexCreateParams.database}). The consumer opens it and
33
+ * owns its lifecycle — this index never closes it. `create` loads the
34
+ * `sqlite-vec` extension onto the connection and, if the fragment table already
35
+ * exists (a reopened persistent file), recovers its established dimension so no
36
+ * re-embedding is required on open.
37
+ */
38
+ readonly database: BetterSqlite3.Database;
39
+ /**
40
+ * Name of the `vec0` virtual table that holds the fragment embeddings. Must be a
41
+ * simple SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to
42
+ * `'memory_fragments'`. Supply a distinct name (distinct from any record-level
43
+ * {@link SqliteVecVectorIndex} table) to hold more than one independent index in
44
+ * a single database file.
45
+ */
46
+ readonly tableName?: string;
47
+ }
48
+ //# sourceMappingURL=model.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"model.d.ts","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/model.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,aAAa,MAAM,gBAAgB,CAAC;AAEhD;;;GAGG;AACH,MAAM,WAAW,iCAAiC;IAChD;;;;;;;;OAQG;IACH,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC;IAE1C;;;;;OAKG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;;GAGG;AACH,MAAM,WAAW,mCAAmC;IAClD;;;;;;;OAOG;IACH,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC;IAE1C;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B"}
@@ -0,0 +1,7 @@
1
+ "use strict";
2
+ /*
3
+ * Copyright (c) 2026 Erik Fortune
4
+ * SPDX-License-Identifier: MIT
5
+ */
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ //# sourceMappingURL=model.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"model.js","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/model.ts"],"names":[],"mappings":";AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport type BetterSqlite3 from 'better-sqlite3';\n\n/**\n * Parameters for {@link SqliteVecVectorIndex.create}.\n * @public\n */\nexport interface ISqliteVecVectorIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * the boundary-package convention). The consumer opens it (`new Database(path)`\n * for a persistent file, or `new Database(':memory:')` for an ephemeral index)\n * and owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the vector table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the embeddings. Must be a simple\n * SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to `'memory_vectors'`.\n * Supply a distinct name to hold more than one independent index in a single\n * database file.\n */\n readonly tableName?: string;\n}\n\n/**\n * Parameters for {@link SqliteVecFragmentIndex.create}.\n * @public\n */\nexport interface ISqliteVecFragmentIndexCreateParams {\n /**\n * A `better-sqlite3` `Database` the consumer owns (bring-your-own, mirroring\n * {@link ISqliteVecVectorIndexCreateParams.database}). The consumer opens it and\n * owns its lifecycle — this index never closes it. `create` loads the\n * `sqlite-vec` extension onto the connection and, if the fragment table already\n * exists (a reopened persistent file), recovers its established dimension so no\n * re-embedding is required on open.\n */\n readonly database: BetterSqlite3.Database;\n\n /**\n * Name of the `vec0` virtual table that holds the fragment embeddings. Must be a\n * simple SQL identifier (`[A-Za-z_][A-Za-z0-9_]*`). Defaults to\n * `'memory_fragments'`. Supply a distinct name (distinct from any record-level\n * {@link SqliteVecVectorIndex} table) to hold more than one independent index in\n * a single database file.\n */\n readonly tableName?: string;\n}\n"]}
@@ -0,0 +1,94 @@
1
+ import { Result } from '@fgv/ts-utils';
2
+ import { IEdgeTarget, IEmbeddedFragment, IFragmentVectorIndex, IVectorQueryHit } from '@fgv/ts-agent-memory';
3
+ import { ISqliteVecFragmentIndexCreateParams } from './model';
4
+ /**
5
+ * A persistent, `sqlite-vec`-backed `IFragmentVectorIndex` (from
6
+ * `@fgv/ts-agent-memory`) — the fragment-granular sibling of
7
+ * {@link SqliteVecVectorIndex}, and the **durable** counterpart to the in-memory
8
+ * `InMemoryFragmentCosineIndex`.
9
+ *
10
+ * @remarks
11
+ * Where {@link SqliteVecVectorIndex} keys one vector per record on a
12
+ * `target_key` primary key, this index holds **many** vectors per record — one per
13
+ * in-record `[start, end)` span — so it keys the `vec0` table on `target_key` as a
14
+ * **`PARTITION KEY`** (many rows may share it) and stores each fragment's locator
15
+ * offsets in two auxiliary columns (`+start_off`, `+end_off`) that ride alongside
16
+ * the vector and are returned on query but never filtered. A query is a brute-force
17
+ * `vec0` KNN scan across all partitions returning per-fragment hits, each carrying
18
+ * its record `target` and the matched `locator`.
19
+ *
20
+ * Semantics match `InMemoryFragmentCosineIndex` exactly: `addFragments` is
21
+ * whole-record-replace (a single transaction deletes every prior fragment of the
22
+ * target, then inserts the new set), `remove` drops every fragment of a target,
23
+ * and `query` applies the optional `maxPerRecord` cap **during selection, before
24
+ * the topK cut** — so one long document cannot crowd others out. The dimension is
25
+ * established by the first `addFragments` (the `vec0` column is fixed-width) and
26
+ * recovered from the table schema when a persistent file is reopened; similarity is
27
+ * cosine (`score = 1 - cosineDistance`), byte-identical to the in-memory index.
28
+ * Large-N ANN indexing is explicitly out of scope, same regime as the record index.
29
+ *
30
+ * The `better-sqlite3` `Database` is consumer-owned (bring-your-own): this index
31
+ * loads the `sqlite-vec` extension onto it and reads/writes the table, but never
32
+ * opens or closes the connection.
33
+ * @public
34
+ */
35
+ export declare class SqliteVecFragmentIndex implements IFragmentVectorIndex {
36
+ private readonly _db;
37
+ private readonly _table;
38
+ /** The dimension of every stored fragment vector; `undefined` until the table exists. */
39
+ private _dimension;
40
+ /** Prepared statements; created once the table exists (established or recovered). */
41
+ private _stmts;
42
+ private constructor();
43
+ /** The number of records that currently have at least one stored fragment. Zero before the first add. */
44
+ get recordCount(): number;
45
+ /** The total number of fragments currently held across all records. Zero before the first add. */
46
+ get fragmentCount(): number;
47
+ /**
48
+ * Family-convention factory. Loads the `sqlite-vec` extension onto the supplied
49
+ * `better-sqlite3` connection and, if the fragment table already exists (a
50
+ * reopened persistent file), recovers its established dimension so no
51
+ * re-embedding is needed on open.
52
+ *
53
+ * @param params - See {@link ISqliteVecFragmentIndexCreateParams}.
54
+ * @returns `Success` with the index, or `Failure` if the table name is not a
55
+ * simple identifier or the extension fails to load.
56
+ */
57
+ static create(params: ISqliteVecFragmentIndexCreateParams): Promise<Result<SqliteVecFragmentIndex>>;
58
+ /** {@inheritDoc IFragmentVectorIndex.addFragments} */
59
+ addFragments(target: IEdgeTarget, fragments: ReadonlyArray<IEmbeddedFragment>): Promise<Result<number>>;
60
+ /** {@inheritDoc IFragmentVectorIndex.remove} */
61
+ remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
62
+ /** {@inheritDoc IFragmentVectorIndex.query} */
63
+ query(vector: Float32Array, topK: number, maxPerRecord?: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
64
+ /** Create the fragment `vec0` virtual table with the established dimension. */
65
+ private _createTable;
66
+ /** Prepare the statements the index reuses. Requires the table to exist. */
67
+ private _prepare;
68
+ /**
69
+ * Recover the established dimension of an existing fragment `vec0` table from its
70
+ * stored `CREATE VIRTUAL TABLE` SQL (`float[<n>]`). Returns `undefined` when the
71
+ * table does not exist yet (a fresh database — dimension is set by the first add).
72
+ */
73
+ private static _readExistingDimension;
74
+ /** Pack a `Float32Array` as the little-endian byte blob `vec0` stores. Copies, so the caller may reuse its buffer. */
75
+ private static _toBlob;
76
+ /**
77
+ * Reverse `edgeTargetKey` — the canonical key is `scope\0id` with NUL excluded
78
+ * from both components, so the first NUL splits it unambiguously. A key with no
79
+ * NUL cannot have been written by `edgeTargetKey`; rather than fabricate a wrong
80
+ * `(scope, id)` from corrupt / externally-edited table data, throw so the query
81
+ * surfaces it as a loud `Failure`.
82
+ */
83
+ private static _parseKey;
84
+ /**
85
+ * Coerce a persisted locator offset to a plain `number`. `better-sqlite3` returns
86
+ * integer columns as `bigint` under safe-integer mode, so an offset can arrive as
87
+ * either; both narrow to `number` here. A value outside the safe-integer range
88
+ * (only reachable via corrupt / externally-edited data — the index only ever
89
+ * writes in-document offsets) throws rather than silently losing precision, so the
90
+ * query surfaces it as a loud `Failure`.
91
+ */
92
+ private static _toOffset;
93
+ }
94
+ //# sourceMappingURL=sqliteVecFragmentIndex.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"sqliteVecFragmentIndex.d.ts","sourceRoot":"","sources":["../../../src/packlets/sqlite-vec-index/sqliteVecFragmentIndex.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,MAAM,EAAgC,MAAM,eAAe,CAAC;AACrE,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,oBAAoB,EACpB,eAAe,EAIhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAE,mCAAmC,EAAE,MAAM,SAAS,CAAC;AAsB9D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,qBAAa,sBAAuB,YAAW,oBAAoB;IACjE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAyB;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,yFAAyF;IACzF,OAAO,CAAC,UAAU,CAAqB;IACvC,qFAAqF;IACrF,OAAO,CAAC,MAAM,CAAkC;IAEhD,OAAO;IAOP,yGAAyG;IACzG,IAAW,WAAW,IAAI,MAAM,CAO/B;IAED,kGAAkG;IAClG,IAAW,aAAa,IAAI,MAAM,CAKjC;IAED;;;;;;;;;OASG;WACW,MAAM,CAAC,MAAM,EAAE,mCAAmC,GAAG,OAAO,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;IAmB1G,sDAAsD;IAC/C,YAAY,CACjB,MAAM,EAAE,WAAW,EACnB,SAAS,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAC1C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAyD1B,gDAAgD;IACzC,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAahE,+CAA+C;IACxC,KAAK,CACV,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,MAAM,EACZ,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC;IAyDlD,+EAA+E;IAC/E,OAAO,CAAC,YAAY;IAQpB,4EAA4E;IAC5E,OAAO,CAAC,QAAQ;IAoChB;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAWrC,sHAAsH;IACtH,OAAO,CAAC,MAAM,CAAC,OAAO;IAItB;;;;;;OAMG;IACH,OAAO,CAAC,MAAM,CAAC,SAAS;IAWxB;;;;;;;OAOG;IACH,OAAO,CAAC,MAAM,CAAC,SAAS;CASzB"}
@@ -0,0 +1,281 @@
1
+ "use strict";
2
+ /*
3
+ * Copyright (c) 2026 Erik Fortune
4
+ * SPDX-License-Identifier: MIT
5
+ */
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ exports.SqliteVecFragmentIndex = void 0;
8
+ const sqlite_vec_1 = require("sqlite-vec");
9
+ const ts_utils_1 = require("@fgv/ts-utils");
10
+ const ts_agent_memory_1 = require("@fgv/ts-agent-memory");
11
+ /** Default name for the fragment `vec0` virtual table. */
12
+ const DEFAULT_TABLE_NAME = 'memory_fragments';
13
+ /** A simple SQL identifier — the only shape allowed for the table name (it is interpolated into DDL). */
14
+ const IDENTIFIER_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
15
+ /**
16
+ * A persistent, `sqlite-vec`-backed `IFragmentVectorIndex` (from
17
+ * `@fgv/ts-agent-memory`) — the fragment-granular sibling of
18
+ * {@link SqliteVecVectorIndex}, and the **durable** counterpart to the in-memory
19
+ * `InMemoryFragmentCosineIndex`.
20
+ *
21
+ * @remarks
22
+ * Where {@link SqliteVecVectorIndex} keys one vector per record on a
23
+ * `target_key` primary key, this index holds **many** vectors per record — one per
24
+ * in-record `[start, end)` span — so it keys the `vec0` table on `target_key` as a
25
+ * **`PARTITION KEY`** (many rows may share it) and stores each fragment's locator
26
+ * offsets in two auxiliary columns (`+start_off`, `+end_off`) that ride alongside
27
+ * the vector and are returned on query but never filtered. A query is a brute-force
28
+ * `vec0` KNN scan across all partitions returning per-fragment hits, each carrying
29
+ * its record `target` and the matched `locator`.
30
+ *
31
+ * Semantics match `InMemoryFragmentCosineIndex` exactly: `addFragments` is
32
+ * whole-record-replace (a single transaction deletes every prior fragment of the
33
+ * target, then inserts the new set), `remove` drops every fragment of a target,
34
+ * and `query` applies the optional `maxPerRecord` cap **during selection, before
35
+ * the topK cut** — so one long document cannot crowd others out. The dimension is
36
+ * established by the first `addFragments` (the `vec0` column is fixed-width) and
37
+ * recovered from the table schema when a persistent file is reopened; similarity is
38
+ * cosine (`score = 1 - cosineDistance`), byte-identical to the in-memory index.
39
+ * Large-N ANN indexing is explicitly out of scope, same regime as the record index.
40
+ *
41
+ * The `better-sqlite3` `Database` is consumer-owned (bring-your-own): this index
42
+ * loads the `sqlite-vec` extension onto it and reads/writes the table, but never
43
+ * opens or closes the connection.
44
+ * @public
45
+ */
46
+ class SqliteVecFragmentIndex {
47
+ constructor(db, table, dimension) {
48
+ this._db = db;
49
+ this._table = table;
50
+ this._dimension = dimension;
51
+ this._stmts = dimension === undefined ? undefined : this._prepare();
52
+ }
53
+ /** The number of records that currently have at least one stored fragment. Zero before the first add. */
54
+ get recordCount() {
55
+ if (this._stmts === undefined) {
56
+ return 0;
57
+ }
58
+ // `Number(...)` narrows the count in case the consumer enabled better-sqlite3
59
+ // safe-integer mode (which returns `count(*)` as a `bigint`).
60
+ return Number(this._stmts.recordCount.get().c);
61
+ }
62
+ /** The total number of fragments currently held across all records. Zero before the first add. */
63
+ get fragmentCount() {
64
+ if (this._stmts === undefined) {
65
+ return 0;
66
+ }
67
+ return Number(this._stmts.fragmentCount.get().c);
68
+ }
69
+ /**
70
+ * Family-convention factory. Loads the `sqlite-vec` extension onto the supplied
71
+ * `better-sqlite3` connection and, if the fragment table already exists (a
72
+ * reopened persistent file), recovers its established dimension so no
73
+ * re-embedding is needed on open.
74
+ *
75
+ * @param params - See {@link ISqliteVecFragmentIndexCreateParams}.
76
+ * @returns `Success` with the index, or `Failure` if the table name is not a
77
+ * simple identifier or the extension fails to load.
78
+ */
79
+ static create(params) {
80
+ var _a;
81
+ const table = (_a = params.tableName) !== null && _a !== void 0 ? _a : DEFAULT_TABLE_NAME;
82
+ if (!IDENTIFIER_RE.test(table)) {
83
+ return Promise.resolve((0, ts_utils_1.fail)(`sqlite-vec fragment index: table name '${table}' is not a simple SQL identifier`));
84
+ }
85
+ return Promise.resolve((0, ts_utils_1.captureResult)(() => {
86
+ (0, sqlite_vec_1.load)(params.database);
87
+ const dimension = SqliteVecFragmentIndex._readExistingDimension(params.database, table);
88
+ return new SqliteVecFragmentIndex(params.database, table, dimension);
89
+ }).withErrorFormat((e) => `sqlite-vec fragment index: failed to initialize: ${e}`));
90
+ }
91
+ /** {@inheritDoc IFragmentVectorIndex.addFragments} */
92
+ addFragments(target, fragments) {
93
+ const key = (0, ts_agent_memory_1.edgeTargetKey)(target);
94
+ // Validate every fragment before touching the database, so a bad fragment never
95
+ // leaves the record half-replaced or the dimension half-established (whole-record
96
+ // replace is all-or-nothing). The effective dimension is the established one, or —
97
+ // on a still-dimensionless index — the first fragment's length; it is committed
98
+ // (via table creation) only once the whole batch validates.
99
+ let dimension = this._dimension;
100
+ for (const fragment of fragments) {
101
+ if (fragment.vector.length === 0) {
102
+ return Promise.resolve((0, ts_utils_1.fail)(`fragment index: cannot add '${key}': empty fragment vector`));
103
+ }
104
+ // Locator offsets are persisted as SQLite integers (bound via BigInt). Reject a
105
+ // non-safe-integer offset up front with a clear message, rather than letting
106
+ // `BigInt(nonInteger)` throw cryptically inside the write transaction OR storing
107
+ // a value the read-side `_toOffset` guard would later reject on every query.
108
+ if (!Number.isSafeInteger(fragment.locator.start) || !Number.isSafeInteger(fragment.locator.end)) {
109
+ return Promise.resolve((0, ts_utils_1.fail)(`fragment index: cannot add '${key}': locator [${fragment.locator.start}, ${fragment.locator.end}) offsets must be safe integers`));
110
+ }
111
+ if (dimension === undefined) {
112
+ dimension = fragment.vector.length;
113
+ }
114
+ else if (fragment.vector.length !== dimension) {
115
+ return Promise.resolve((0, ts_utils_1.fail)(`fragment index: cannot add '${key}': fragment dimension ${fragment.vector.length} does not match index dimension ${dimension}`));
116
+ }
117
+ }
118
+ return Promise.resolve((0, ts_utils_1.captureResult)(() => {
119
+ // A same-target re-author (or an empty batch) still needs the table to exist
120
+ // to delete prior fragments; create it lazily on the first non-empty add.
121
+ if (this._stmts === undefined) {
122
+ if (fragments.length === 0) {
123
+ // Nothing stored yet and nothing to store: no table, no work.
124
+ return 0;
125
+ }
126
+ // `fragments` is non-empty here (the empty case returned above), so the
127
+ // validation loop proved every fragment shares `fragments[0]`'s length —
128
+ // which IS the dimension to establish. Read it straight from the first
129
+ // fragment: no cast, no invariant-dependent narrowing.
130
+ const established = fragments[0].vector.length;
131
+ this._createTable(established);
132
+ this._dimension = established;
133
+ this._stmts = this._prepare();
134
+ }
135
+ this._stmts.replace(key, fragments);
136
+ return fragments.length;
137
+ }).withErrorFormat((e) => `fragment index: cannot add '${key}': ${e}`));
138
+ }
139
+ /** {@inheritDoc IFragmentVectorIndex.remove} */
140
+ remove(target) {
141
+ return Promise.resolve((0, ts_utils_1.captureResult)(() => {
142
+ // Idempotent: removing a target with no fragments (or before any add created
143
+ // the table) still succeeds.
144
+ if (this._stmts !== undefined) {
145
+ this._stmts.deleteByTarget.run((0, ts_agent_memory_1.edgeTargetKey)(target));
146
+ }
147
+ return target;
148
+ }).withErrorFormat((e) => `fragment index: cannot remove '${(0, ts_agent_memory_1.edgeTargetKey)(target)}': ${e}`));
149
+ }
150
+ /** {@inheritDoc IFragmentVectorIndex.query} */
151
+ query(vector, topK, maxPerRecord) {
152
+ if (topK <= 0 || this._stmts === undefined) {
153
+ return Promise.resolve((0, ts_utils_1.succeed)([]));
154
+ }
155
+ if (vector.length !== this._dimension) {
156
+ return Promise.resolve((0, ts_utils_1.fail)(`fragment index: query dimension ${vector.length} does not match index dimension ${this._dimension}`));
157
+ }
158
+ const stmts = this._stmts;
159
+ return Promise.resolve((0, ts_utils_1.captureResult)(() => {
160
+ var _a;
161
+ // With a per-record cap the topK winners may lie past the first topK rows (a
162
+ // capped record's later fragments are skipped), so fetch the full ranked set
163
+ // and apply the cap + topK cut here — exactly as the in-memory index does.
164
+ // Uncapped, KNN's own `k = topK` is already the answer.
165
+ const fetchK = maxPerRecord === undefined ? topK : Number(stmts.fragmentCount.get().c);
166
+ if (fetchK <= 0) {
167
+ return [];
168
+ }
169
+ const rows = stmts.query.all(SqliteVecFragmentIndex._toBlob(vector), fetchK);
170
+ // sqlite-vec returns rows ascending by distance (nearest first); score is
171
+ // `1 - cosineDistance`, so this order is already descending score.
172
+ const hits = [];
173
+ const perRecord = new Map();
174
+ for (const row of rows) {
175
+ if (hits.length >= topK) {
176
+ break;
177
+ }
178
+ if (maxPerRecord !== undefined) {
179
+ const used = (_a = perRecord.get(row.target_key)) !== null && _a !== void 0 ? _a : 0;
180
+ if (used >= maxPerRecord) {
181
+ continue;
182
+ }
183
+ perRecord.set(row.target_key, used + 1);
184
+ }
185
+ const key = row.target_key;
186
+ hits.push({
187
+ target: SqliteVecFragmentIndex._parseKey(key),
188
+ score: 1 - row.distance,
189
+ locator: {
190
+ start: SqliteVecFragmentIndex._toOffset(row.start_off, key),
191
+ end: SqliteVecFragmentIndex._toOffset(row.end_off, key)
192
+ }
193
+ });
194
+ }
195
+ return hits;
196
+ }).withErrorFormat((e) => `fragment index: query failed: ${e}`));
197
+ }
198
+ /** Create the fragment `vec0` virtual table with the established dimension. */
199
+ _createTable(dimension) {
200
+ this._db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS "${this._table}" USING vec0(` +
201
+ `target_key TEXT PARTITION KEY, embedding float[${dimension}] distance_metric=cosine, ` +
202
+ `+start_off integer, +end_off integer)`);
203
+ }
204
+ /** Prepare the statements the index reuses. Requires the table to exist. */
205
+ _prepare() {
206
+ const del = this._db.prepare(`DELETE FROM "${this._table}" WHERE target_key = ?`);
207
+ const ins = this._db.prepare(`INSERT INTO "${this._table}"(target_key, embedding, start_off, end_off) VALUES (?, ?, ?, ?)`);
208
+ // Whole-record replace: drop every prior fragment of the target, then insert the
209
+ // new set, atomically. An empty set collapses to a pure delete.
210
+ const replaceTxn = this._db.transaction((key, fragments) => {
211
+ del.run(key);
212
+ for (const fragment of fragments) {
213
+ ins.run(key, SqliteVecFragmentIndex._toBlob(fragment.vector),
214
+ // vec0 typed columns reject a JS float; bind the offsets as integers.
215
+ BigInt(fragment.locator.start), BigInt(fragment.locator.end));
216
+ }
217
+ });
218
+ return {
219
+ deleteByTarget: del,
220
+ replace: (key, fragments) => {
221
+ replaceTxn(key, fragments);
222
+ },
223
+ query: this._db.prepare(`SELECT target_key, start_off, end_off, distance FROM "${this._table}" WHERE embedding MATCH ? AND k = ?`),
224
+ fragmentCount: this._db.prepare(`SELECT count(*) AS c FROM "${this._table}"`),
225
+ recordCount: this._db.prepare(`SELECT count(DISTINCT target_key) AS c FROM "${this._table}"`)
226
+ };
227
+ }
228
+ /**
229
+ * Recover the established dimension of an existing fragment `vec0` table from its
230
+ * stored `CREATE VIRTUAL TABLE` SQL (`float[<n>]`). Returns `undefined` when the
231
+ * table does not exist yet (a fresh database — dimension is set by the first add).
232
+ */
233
+ static _readExistingDimension(db, table) {
234
+ const row = db
235
+ .prepare("SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?")
236
+ .get(table);
237
+ if (row === undefined) {
238
+ return undefined;
239
+ }
240
+ const match = row.sql.match(/float\[(\d+)\]/);
241
+ return match === null ? undefined : Number(match[1]);
242
+ }
243
+ /** Pack a `Float32Array` as the little-endian byte blob `vec0` stores. Copies, so the caller may reuse its buffer. */
244
+ static _toBlob(vector) {
245
+ return new Uint8Array(Float32Array.from(vector).buffer);
246
+ }
247
+ /**
248
+ * Reverse `edgeTargetKey` — the canonical key is `scope\0id` with NUL excluded
249
+ * from both components, so the first NUL splits it unambiguously. A key with no
250
+ * NUL cannot have been written by `edgeTargetKey`; rather than fabricate a wrong
251
+ * `(scope, id)` from corrupt / externally-edited table data, throw so the query
252
+ * surfaces it as a loud `Failure`.
253
+ */
254
+ static _parseKey(key) {
255
+ const nul = key.indexOf('\0');
256
+ if (nul < 0) {
257
+ throw new Error(`malformed target key '${key}': missing scope/id separator (corrupt persisted data)`);
258
+ }
259
+ return {
260
+ scope: key.slice(0, nul),
261
+ id: key.slice(nul + 1)
262
+ };
263
+ }
264
+ /**
265
+ * Coerce a persisted locator offset to a plain `number`. `better-sqlite3` returns
266
+ * integer columns as `bigint` under safe-integer mode, so an offset can arrive as
267
+ * either; both narrow to `number` here. A value outside the safe-integer range
268
+ * (only reachable via corrupt / externally-edited data — the index only ever
269
+ * writes in-document offsets) throws rather than silently losing precision, so the
270
+ * query surfaces it as a loud `Failure`.
271
+ */
272
+ static _toOffset(value, key) {
273
+ const n = Number(value);
274
+ if (!Number.isSafeInteger(n)) {
275
+ throw new Error(`fragment '${key}': locator offset ${String(value)} is not a safe integer (corrupt persisted data)`);
276
+ }
277
+ return n;
278
+ }
279
+ }
280
+ exports.SqliteVecFragmentIndex = SqliteVecFragmentIndex;
281
+ //# sourceMappingURL=sqliteVecFragmentIndex.js.map