@fgv/ts-agent-memory 5.1.0-41 → 5.1.0-43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.rush/temp/{cbbdbe09515171b4eba8f2592be2dace1e4e8142.tar.log → 285f03271c27ef724d49e730c0db58d9e4ac1a44.tar.log} +38 -2
- package/.rush/temp/chunked-rush-logs/ts-agent-memory.build.chunks.jsonl +2 -2
- package/.rush/temp/operation/build/all.log +2 -2
- package/.rush/temp/operation/build/log-chunks.jsonl +2 -2
- package/.rush/temp/operation/build/state.json +1 -1
- package/dist/packlets/retrieve/fragmentSemanticRetriever.js +78 -0
- package/dist/packlets/retrieve/fragmentSemanticRetriever.js.map +1 -0
- package/dist/packlets/retrieve/index.js +1 -0
- package/dist/packlets/retrieve/index.js.map +1 -1
- package/dist/packlets/store/fileTreeMemoryStore.js +105 -14
- package/dist/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js +200 -0
- package/dist/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -0
- package/dist/packlets/vector/index.js +1 -0
- package/dist/packlets/vector/index.js.map +1 -1
- package/dist/packlets/vector/vectorIndex.js.map +1 -1
- package/dist/test/unit/retrieve/fragmentSemanticRetriever.test.js +116 -0
- package/dist/test/unit/retrieve/fragmentSemanticRetriever.test.js.map +1 -0
- package/dist/test/unit/store/fragmentEmbedOnWrite.test.js +255 -0
- package/dist/test/unit/store/fragmentEmbedOnWrite.test.js.map +1 -0
- package/dist/test/unit/store/lenientOpen.test.js +248 -0
- package/dist/test/unit/store/lenientOpen.test.js.map +1 -0
- package/dist/test/unit/vector/inMemoryFragmentCosineIndex.test.js +297 -0
- package/dist/test/unit/vector/inMemoryFragmentCosineIndex.test.js.map +1 -0
- package/dist/ts-agent-memory.d.ts +367 -4
- package/etc/ts-agent-memory.api.md +78 -0
- package/lib/packlets/retrieve/fragmentSemanticRetriever.d.ts +90 -0
- package/lib/packlets/retrieve/fragmentSemanticRetriever.d.ts.map +1 -0
- package/lib/packlets/retrieve/fragmentSemanticRetriever.js +82 -0
- package/lib/packlets/retrieve/fragmentSemanticRetriever.js.map +1 -0
- package/lib/packlets/retrieve/index.d.ts +1 -0
- package/lib/packlets/retrieve/index.d.ts.map +1 -1
- package/lib/packlets/retrieve/index.js +1 -0
- package/lib/packlets/retrieve/index.js.map +1 -1
- package/lib/packlets/store/fileTreeMemoryStore.d.ts +115 -1
- package/lib/packlets/store/fileTreeMemoryStore.d.ts.map +1 -1
- package/lib/packlets/store/fileTreeMemoryStore.js +104 -13
- package/lib/packlets/store/fileTreeMemoryStore.js.map +1 -1
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts +74 -0
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.d.ts.map +1 -0
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js +204 -0
- package/lib/packlets/vector/inMemoryFragmentCosineIndex.js.map +1 -0
- package/lib/packlets/vector/index.d.ts +1 -0
- package/lib/packlets/vector/index.d.ts.map +1 -1
- package/lib/packlets/vector/index.js +1 -0
- package/lib/packlets/vector/index.js.map +1 -1
- package/lib/packlets/vector/vectorIndex.d.ts +85 -4
- package/lib/packlets/vector/vectorIndex.d.ts.map +1 -1
- package/lib/packlets/vector/vectorIndex.js.map +1 -1
- package/lib/test/unit/retrieve/fragmentSemanticRetriever.test.d.ts +2 -0
- package/lib/test/unit/retrieve/fragmentSemanticRetriever.test.d.ts.map +1 -0
- package/lib/test/unit/retrieve/fragmentSemanticRetriever.test.js +118 -0
- package/lib/test/unit/retrieve/fragmentSemanticRetriever.test.js.map +1 -0
- package/lib/test/unit/store/fragmentEmbedOnWrite.test.d.ts +2 -0
- package/lib/test/unit/store/fragmentEmbedOnWrite.test.d.ts.map +1 -0
- package/lib/test/unit/store/fragmentEmbedOnWrite.test.js +257 -0
- package/lib/test/unit/store/fragmentEmbedOnWrite.test.js.map +1 -0
- package/lib/test/unit/store/lenientOpen.test.d.ts +2 -0
- package/lib/test/unit/store/lenientOpen.test.d.ts.map +1 -0
- package/lib/test/unit/store/lenientOpen.test.js +250 -0
- package/lib/test/unit/store/lenientOpen.test.js.map +1 -0
- package/lib/test/unit/vector/inMemoryFragmentCosineIndex.test.d.ts +2 -0
- package/lib/test/unit/vector/inMemoryFragmentCosineIndex.test.d.ts.map +1 -0
- package/lib/test/unit/vector/inMemoryFragmentCosineIndex.test.js +299 -0
- package/lib/test/unit/vector/inMemoryFragmentCosineIndex.test.js.map +1 -0
- package/package.json +7 -7
- package/rush-logs/ts-agent-memory.build.cache.log +1 -1
- package/rush-logs/ts-agent-memory.build.log +2 -2
- package/src/packlets/retrieve/fragmentSemanticRetriever.ts +135 -0
- package/src/packlets/retrieve/index.ts +1 -0
- package/src/packlets/store/fileTreeMemoryStore.ts +208 -16
- package/src/packlets/vector/inMemoryFragmentCosineIndex.ts +262 -0
- package/src/packlets/vector/index.ts +1 -0
- package/src/packlets/vector/vectorIndex.ts +97 -4
- package/src/test/unit/retrieve/fragmentSemanticRetriever.test.ts +163 -0
- package/src/test/unit/store/fragmentEmbedOnWrite.test.ts +349 -0
- package/src/test/unit/store/lenientOpen.test.ts +292 -0
- package/src/test/unit/vector/inMemoryFragmentCosineIndex.test.ts +389 -0
- package/temp/build/lint/_eslint-5eVG3S6w.json +29 -5
- package/temp/build/typescript/ts_8nwakTlr.json +1 -1
- package/temp/ts-agent-memory.api.json +4584 -2859
- package/temp/ts-agent-memory.api.md +78 -0
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { Result } from '@fgv/ts-utils';
|
|
2
|
+
import { IEdgeTarget } from '../types';
|
|
3
|
+
import { FragmentEmbedder, IEmbeddedFragment, IFragmentVectorIndex, IMemoryRecordSource, IVectorQueryHit } from './vectorIndex';
|
|
4
|
+
/**
|
|
5
|
+
* The brute-force, in-memory cosine {@link IFragmentVectorIndex} — the
|
|
6
|
+
* fragment-granular sibling of {@link InMemoryCosineIndex}. Stores many
|
|
7
|
+
* `Float32Array`s per record (one per in-record {@link IFragmentLocator | span})
|
|
8
|
+
* and answers a query by computing cosine similarity against every stored
|
|
9
|
+
* fragment, returning the top-k fragment hits by descending score.
|
|
10
|
+
*
|
|
11
|
+
* @remarks
|
|
12
|
+
* Same regime and same non-goals as {@link InMemoryCosineIndex}: no external
|
|
13
|
+
* dependency, no ANN structure, a linear scan over the stored fragments — the seam
|
|
14
|
+
* ({@link IFragmentVectorIndex}) stays open for a consumer to swap a persistent /
|
|
15
|
+
* ANN backend once N grows. `addFragments` is whole-record-replace, so re-authoring
|
|
16
|
+
* a document never leaves stale fragments behind. The index has a single dimension
|
|
17
|
+
* established by the first fragment added; every subsequent fragment and every
|
|
18
|
+
* `query` vector must match it or fail loudly — a mismatched dimension is an
|
|
19
|
+
* embedder-wiring bug, never a silent zero-similarity result.
|
|
20
|
+
* {@link InMemoryFragmentCosineIndex.rebuild | rebuild} clears the index (and the
|
|
21
|
+
* established dimension), so a re-embed with a different model is supported.
|
|
22
|
+
*
|
|
23
|
+
* The optional `maxPerRecord` cap on `query` is applied **during selection**, before
|
|
24
|
+
* the `topK` cut, so one long document with many strong fragments cannot crowd every
|
|
25
|
+
* other record out of the result.
|
|
26
|
+
* @public
|
|
27
|
+
*/
|
|
28
|
+
export declare class InMemoryFragmentCosineIndex implements IFragmentVectorIndex {
|
|
29
|
+
/**
|
|
30
|
+
* Stored fragments keyed by the canonical {@link edgeTargetKey} of the record's
|
|
31
|
+
* scope-qualified address, so two records that share a filename stem across
|
|
32
|
+
* scopes occupy distinct entries and never overwrite each other's fragments.
|
|
33
|
+
*/
|
|
34
|
+
private readonly _records;
|
|
35
|
+
/** The dimension of every stored fragment vector; `undefined` until the first `add`. */
|
|
36
|
+
private _dimension;
|
|
37
|
+
private constructor();
|
|
38
|
+
/** The number of records that currently have at least one stored fragment. */
|
|
39
|
+
get recordCount(): number;
|
|
40
|
+
/** The total number of fragments currently held across all records. */
|
|
41
|
+
get fragmentCount(): number;
|
|
42
|
+
/** Family-convention factory. */
|
|
43
|
+
static create(): Result<InMemoryFragmentCosineIndex>;
|
|
44
|
+
/** {@inheritDoc IFragmentVectorIndex.addFragments} */
|
|
45
|
+
addFragments(target: IEdgeTarget, fragments: ReadonlyArray<IEmbeddedFragment>): Promise<Result<number>>;
|
|
46
|
+
/** {@inheritDoc IFragmentVectorIndex.remove} */
|
|
47
|
+
remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
|
|
48
|
+
/** {@inheritDoc IFragmentVectorIndex.query} */
|
|
49
|
+
query(vector: Float32Array, topK: number, maxPerRecord?: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
|
|
50
|
+
/**
|
|
51
|
+
* Re-embed every record from `source` and rebuild the fragment index from
|
|
52
|
+
* scratch. Clears the current contents (and the established dimension) first, so
|
|
53
|
+
* a re-embed with a different model is supported. Returns the total number of
|
|
54
|
+
* fragments indexed.
|
|
55
|
+
*
|
|
56
|
+
* On any failure (list, embed, or add) the index is rolled back to empty rather
|
|
57
|
+
* than left in a partially-rebuilt state.
|
|
58
|
+
*
|
|
59
|
+
* @param source - The scope-qualified record source to re-embed.
|
|
60
|
+
* @param embed - The fragment embedder applied to each record.
|
|
61
|
+
*/
|
|
62
|
+
rebuild(source: IMemoryRecordSource, embed: FragmentEmbedder): Promise<Result<number>>;
|
|
63
|
+
/** Empty the index and forget the established dimension. */
|
|
64
|
+
private _reset;
|
|
65
|
+
/** The Euclidean magnitude (L2 norm) of a vector. */
|
|
66
|
+
private static _magnitude;
|
|
67
|
+
/**
|
|
68
|
+
* Cosine similarity between the query (whose magnitude is precomputed once and
|
|
69
|
+
* reused across the scan) and a stored fragment vector. A zero-magnitude vector
|
|
70
|
+
* on either side yields `0` rather than `NaN`.
|
|
71
|
+
*/
|
|
72
|
+
private static _cosine;
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=inMemoryFragmentCosineIndex.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inMemoryFragmentCosineIndex.d.ts","sourceRoot":"","sources":["../../../src/packlets/vector/inMemoryFragmentCosineIndex.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAiB,MAAM,eAAe,CAAC;AACtD,OAAO,EAAE,WAAW,EAAiB,MAAM,UAAU,CAAC;AACtD,OAAO,EACL,gBAAgB,EAChB,iBAAiB,EAEjB,oBAAoB,EACpB,mBAAmB,EAEnB,eAAe,EAChB,MAAM,eAAe,CAAC;AAoBvB;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,qBAAa,2BAA4B,YAAW,oBAAoB;IACtE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsC;IAC/D,wFAAwF;IACxF,OAAO,CAAC,UAAU,CAAqB;IAEvC,OAAO;IAKP,8EAA8E;IAC9E,IAAW,WAAW,IAAI,MAAM,CAE/B;IAED,uEAAuE;IACvE,IAAW,aAAa,IAAI,MAAM,CAMjC;IAED,iCAAiC;WACnB,MAAM,IAAI,MAAM,CAAC,2BAA2B,CAAC;IAI3D,sDAAsD;IAC/C,YAAY,CACjB,MAAM,EAAE,WAAW,EACnB,SAAS,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAC1C,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAqC1B,gDAAgD;IACzC,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAKhE,+CAA+C;IACxC,KAAK,CACV,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,MAAM,EACZ,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC;IAkDlD;;;;;;;;;;;OAWG;IACU,OAAO,CAAC,MAAM,EAAE,mBAAmB,EAAE,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAuBnG,4DAA4D;IAC5D,OAAO,CAAC,MAAM;IAKd,qDAAqD;IACrD,OAAO,CAAC,MAAM,CAAC,UAAU;IAQzB;;;;OAIG;IACH,OAAO,CAAC,MAAM,CAAC,OAAO;CAWvB"}
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright (c) 2026 Erik Fortune
|
|
4
|
+
* SPDX-License-Identifier: MIT
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
exports.InMemoryFragmentCosineIndex = void 0;
|
|
8
|
+
const ts_utils_1 = require("@fgv/ts-utils");
|
|
9
|
+
const types_1 = require("../types");
|
|
10
|
+
/**
|
|
11
|
+
* The brute-force, in-memory cosine {@link IFragmentVectorIndex} — the
|
|
12
|
+
* fragment-granular sibling of {@link InMemoryCosineIndex}. Stores many
|
|
13
|
+
* `Float32Array`s per record (one per in-record {@link IFragmentLocator | span})
|
|
14
|
+
* and answers a query by computing cosine similarity against every stored
|
|
15
|
+
* fragment, returning the top-k fragment hits by descending score.
|
|
16
|
+
*
|
|
17
|
+
* @remarks
|
|
18
|
+
* Same regime and same non-goals as {@link InMemoryCosineIndex}: no external
|
|
19
|
+
* dependency, no ANN structure, a linear scan over the stored fragments — the seam
|
|
20
|
+
* ({@link IFragmentVectorIndex}) stays open for a consumer to swap a persistent /
|
|
21
|
+
* ANN backend once N grows. `addFragments` is whole-record-replace, so re-authoring
|
|
22
|
+
* a document never leaves stale fragments behind. The index has a single dimension
|
|
23
|
+
* established by the first fragment added; every subsequent fragment and every
|
|
24
|
+
* `query` vector must match it or fail loudly — a mismatched dimension is an
|
|
25
|
+
* embedder-wiring bug, never a silent zero-similarity result.
|
|
26
|
+
* {@link InMemoryFragmentCosineIndex.rebuild | rebuild} clears the index (and the
|
|
27
|
+
* established dimension), so a re-embed with a different model is supported.
|
|
28
|
+
*
|
|
29
|
+
* The optional `maxPerRecord` cap on `query` is applied **during selection**, before
|
|
30
|
+
* the `topK` cut, so one long document with many strong fragments cannot crowd every
|
|
31
|
+
* other record out of the result.
|
|
32
|
+
* @public
|
|
33
|
+
*/
|
|
34
|
+
class InMemoryFragmentCosineIndex {
|
|
35
|
+
constructor() {
|
|
36
|
+
this._records = new Map();
|
|
37
|
+
this._dimension = undefined;
|
|
38
|
+
}
|
|
39
|
+
/** The number of records that currently have at least one stored fragment. */
|
|
40
|
+
get recordCount() {
|
|
41
|
+
return this._records.size;
|
|
42
|
+
}
|
|
43
|
+
/** The total number of fragments currently held across all records. */
|
|
44
|
+
get fragmentCount() {
|
|
45
|
+
let total = 0;
|
|
46
|
+
for (const record of this._records.values()) {
|
|
47
|
+
total += record.fragments.length;
|
|
48
|
+
}
|
|
49
|
+
return total;
|
|
50
|
+
}
|
|
51
|
+
/** Family-convention factory. */
|
|
52
|
+
static create() {
|
|
53
|
+
return (0, ts_utils_1.succeed)(new InMemoryFragmentCosineIndex());
|
|
54
|
+
}
|
|
55
|
+
/** {@inheritDoc IFragmentVectorIndex.addFragments} */
|
|
56
|
+
addFragments(target, fragments) {
|
|
57
|
+
const key = (0, types_1.edgeTargetKey)(target);
|
|
58
|
+
// Validate every fragment before mutating any state, so a bad fragment never
|
|
59
|
+
// leaves the record half-replaced OR the index dimension half-established
|
|
60
|
+
// (whole-record-replace must be all-or-nothing). The effective dimension is the
|
|
61
|
+
// established one, or — on a still-dimensionless index — the first fragment's
|
|
62
|
+
// length; it is only committed to `this._dimension` once the whole batch passes.
|
|
63
|
+
const stored = [];
|
|
64
|
+
let dimension = this._dimension;
|
|
65
|
+
for (const fragment of fragments) {
|
|
66
|
+
if (fragment.vector.length === 0) {
|
|
67
|
+
return Promise.resolve((0, ts_utils_1.fail)(`fragment index: cannot add '${key}': empty fragment vector`));
|
|
68
|
+
}
|
|
69
|
+
if (dimension === undefined) {
|
|
70
|
+
dimension = fragment.vector.length;
|
|
71
|
+
}
|
|
72
|
+
else if (fragment.vector.length !== dimension) {
|
|
73
|
+
return Promise.resolve((0, ts_utils_1.fail)(`fragment index: cannot add '${key}': fragment dimension ${fragment.vector.length} does not match index dimension ${dimension}`));
|
|
74
|
+
}
|
|
75
|
+
// Defensive copy: the caller may reuse or mutate the buffer after `addFragments`.
|
|
76
|
+
stored.push({ locator: fragment.locator, vector: Float32Array.from(fragment.vector) });
|
|
77
|
+
}
|
|
78
|
+
// Whole-record replace: an empty `fragments` array drops the record entirely
|
|
79
|
+
// rather than leaving an empty shell behind. Commit the (possibly newly-derived)
|
|
80
|
+
// dimension only alongside a successful, non-empty store.
|
|
81
|
+
if (stored.length === 0) {
|
|
82
|
+
this._records.delete(key);
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
this._dimension = dimension;
|
|
86
|
+
this._records.set(key, { target, fragments: stored });
|
|
87
|
+
}
|
|
88
|
+
return Promise.resolve((0, ts_utils_1.succeed)(stored.length));
|
|
89
|
+
}
|
|
90
|
+
/** {@inheritDoc IFragmentVectorIndex.remove} */
|
|
91
|
+
remove(target) {
|
|
92
|
+
this._records.delete((0, types_1.edgeTargetKey)(target));
|
|
93
|
+
return Promise.resolve((0, ts_utils_1.succeed)(target));
|
|
94
|
+
}
|
|
95
|
+
/** {@inheritDoc IFragmentVectorIndex.query} */
|
|
96
|
+
query(vector, topK, maxPerRecord) {
|
|
97
|
+
var _a;
|
|
98
|
+
if (topK <= 0 || this._records.size === 0) {
|
|
99
|
+
return Promise.resolve((0, ts_utils_1.succeed)([]));
|
|
100
|
+
}
|
|
101
|
+
if (vector.length !== this._dimension) {
|
|
102
|
+
return Promise.resolve((0, ts_utils_1.fail)(`fragment index: query dimension ${vector.length} does not match index dimension ${this._dimension}`));
|
|
103
|
+
}
|
|
104
|
+
const queryMagnitude = InMemoryFragmentCosineIndex._magnitude(vector);
|
|
105
|
+
const scored = [];
|
|
106
|
+
for (const record of this._records.values()) {
|
|
107
|
+
for (const fragment of record.fragments) {
|
|
108
|
+
scored.push({
|
|
109
|
+
key: (0, types_1.edgeTargetKey)(record.target),
|
|
110
|
+
hit: {
|
|
111
|
+
target: record.target,
|
|
112
|
+
score: InMemoryFragmentCosineIndex._cosine(vector, queryMagnitude, fragment.vector),
|
|
113
|
+
locator: fragment.locator
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
// Descending by score; the caller re-resolves each `(target, locator)` hit.
|
|
119
|
+
scored.sort((a, b) => b.hit.score - a.hit.score);
|
|
120
|
+
const hits = [];
|
|
121
|
+
// Apply the per-record cap during selection (before the topK cut) so a single
|
|
122
|
+
// long document cannot monopolize the result. `undefined` maxPerRecord means
|
|
123
|
+
// uncapped; the counter map is always allocated (tiny) so the guard narrows
|
|
124
|
+
// `maxPerRecord` directly without a non-null assertion.
|
|
125
|
+
const perRecord = new Map();
|
|
126
|
+
for (const candidate of scored) {
|
|
127
|
+
if (hits.length >= topK) {
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
if (maxPerRecord !== undefined) {
|
|
131
|
+
const used = (_a = perRecord.get(candidate.key)) !== null && _a !== void 0 ? _a : 0;
|
|
132
|
+
if (used >= maxPerRecord) {
|
|
133
|
+
continue;
|
|
134
|
+
}
|
|
135
|
+
perRecord.set(candidate.key, used + 1);
|
|
136
|
+
}
|
|
137
|
+
hits.push(candidate.hit);
|
|
138
|
+
}
|
|
139
|
+
return Promise.resolve((0, ts_utils_1.succeed)(hits));
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Re-embed every record from `source` and rebuild the fragment index from
|
|
143
|
+
* scratch. Clears the current contents (and the established dimension) first, so
|
|
144
|
+
* a re-embed with a different model is supported. Returns the total number of
|
|
145
|
+
* fragments indexed.
|
|
146
|
+
*
|
|
147
|
+
* On any failure (list, embed, or add) the index is rolled back to empty rather
|
|
148
|
+
* than left in a partially-rebuilt state.
|
|
149
|
+
*
|
|
150
|
+
* @param source - The scope-qualified record source to re-embed.
|
|
151
|
+
* @param embed - The fragment embedder applied to each record.
|
|
152
|
+
*/
|
|
153
|
+
async rebuild(source, embed) {
|
|
154
|
+
this._reset();
|
|
155
|
+
const listed = await source.list();
|
|
156
|
+
if (listed.isFailure()) {
|
|
157
|
+
return (0, ts_utils_1.fail)(`fragment index rebuild: failed to list records: ${listed.message}`);
|
|
158
|
+
}
|
|
159
|
+
for (const scoped of listed.value) {
|
|
160
|
+
const embedded = await embed(scoped.record);
|
|
161
|
+
if (embedded.isFailure()) {
|
|
162
|
+
this._reset();
|
|
163
|
+
return (0, ts_utils_1.fail)(`fragment index rebuild: embedding '${(0, types_1.edgeTargetKey)(scoped.target)}' failed: ${embedded.message}`);
|
|
164
|
+
}
|
|
165
|
+
const added = await this.addFragments(scoped.target, embedded.value);
|
|
166
|
+
if (added.isFailure()) {
|
|
167
|
+
this._reset();
|
|
168
|
+
return (0, ts_utils_1.fail)(`fragment index rebuild: ${added.message}`);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
return (0, ts_utils_1.succeed)(this.fragmentCount);
|
|
172
|
+
}
|
|
173
|
+
/** Empty the index and forget the established dimension. */
|
|
174
|
+
_reset() {
|
|
175
|
+
this._records.clear();
|
|
176
|
+
this._dimension = undefined;
|
|
177
|
+
}
|
|
178
|
+
/** The Euclidean magnitude (L2 norm) of a vector. */
|
|
179
|
+
static _magnitude(vector) {
|
|
180
|
+
let sum = 0;
|
|
181
|
+
for (let i = 0; i < vector.length; i++) {
|
|
182
|
+
sum += vector[i] * vector[i];
|
|
183
|
+
}
|
|
184
|
+
return Math.sqrt(sum);
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Cosine similarity between the query (whose magnitude is precomputed once and
|
|
188
|
+
* reused across the scan) and a stored fragment vector. A zero-magnitude vector
|
|
189
|
+
* on either side yields `0` rather than `NaN`.
|
|
190
|
+
*/
|
|
191
|
+
static _cosine(query, queryMagnitude, stored) {
|
|
192
|
+
const storedMagnitude = InMemoryFragmentCosineIndex._magnitude(stored);
|
|
193
|
+
if (queryMagnitude === 0 || storedMagnitude === 0) {
|
|
194
|
+
return 0;
|
|
195
|
+
}
|
|
196
|
+
let dot = 0;
|
|
197
|
+
for (let i = 0; i < query.length; i++) {
|
|
198
|
+
dot += query[i] * stored[i];
|
|
199
|
+
}
|
|
200
|
+
return dot / (queryMagnitude * storedMagnitude);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
exports.InMemoryFragmentCosineIndex = InMemoryFragmentCosineIndex;
|
|
204
|
+
//# sourceMappingURL=inMemoryFragmentCosineIndex.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inMemoryFragmentCosineIndex.js","sourceRoot":"","sources":["../../../src/packlets/vector/inMemoryFragmentCosineIndex.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,4CAAsD;AACtD,oCAAsD;AA6BtD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAa,2BAA2B;IAUtC;QACE,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,EAAkC,CAAC;QAC1D,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,8EAA8E;IAC9E,IAAW,WAAW;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,uEAAuE;IACvE,IAAW,aAAa;QACtB,IAAI,KAAK,GAAW,CAAC,CAAC;QACtB,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;QACnC,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,iCAAiC;IAC1B,MAAM,CAAC,MAAM;QAClB,OAAO,IAAA,kBAAO,EAAC,IAAI,2BAA2B,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,sDAAsD;IAC/C,YAAY,CACjB,MAAmB,EACnB,SAA2C;QAE3C,MAAM,GAAG,GAAW,IAAA,qBAAa,EAAC,MAAM,CAAC,CAAC;QAC1C,6EAA6E;QAC7E,0EAA0E;QAC1E,gFAAgF;QAChF,8EAA8E;QAC9E,iFAAiF;QACjF,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,IAAI,SAAS,GAAuB,IAAI,CAAC,UAAU,CAAC;QACpD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACjC,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,eAAI,EAAC,+BAA+B,GAAG,0BAA0B,CAAC,CAAC,CAAC;YAC7F,CAAC;YACD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;gBAC5B,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;YACrC,CAAC;iBAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChD,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,eAAI,EACF,+BAA+B,GAAG,yBAAyB,QAAQ,CAAC,MAAM,CAAC,MAAM,mCAAmC,SAAS,EAAE,CAChI,CACF,CAAC;YACJ,CAAC;YACD,kFAAkF;YAClF,MAAM,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,6EAA6E;QAC7E,iFAAiF;QACjF,0DAA0D;QAC1D,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;YAC5B,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,kBAAO,EAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACjD,CAAC;IAED,gDAAgD;IACzC,MAAM,CAAC,MAAmB;QAC/B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAA,qBAAa,EAAC,MAAM,CAAC,CAAC,CAAC;QAC5C,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,kBAAO,EAAC,MAAM,CAAC,CAAC,CAAC;IAC1C,CAAC;IAED,+CAA+C;IACxC,KAAK,CACV,MAAoB,EACpB,IAAY,EACZ,YAAqB;;QAErB,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;YAC1C,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,kBAAO,EAAC,EAAE,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACtC,OAAO,OAAO,CAAC,OAAO,CACpB,IAAA,eAAI,EACF,mCAAmC,MAAM,CAAC,MAAM,mCAAmC,IAAI,CAAC,UAAU,EAAE,CACrG,CACF,CAAC;QACJ,CAAC;QACD,MAAM,cAAc,GAAW,2BAA2B,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAsB,EAAE,CAAC;QACrC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,GAAG,EAAE,IAAA,qBAAa,EAAC,MAAM,CAAC,MAAM,CAAC;oBACjC,GAAG,EAAE;wBACH,MAAM,EAAE,MAAM,CAAC,MAAM;wBACrB,KAAK,EAAE,2BAA2B,CAAC,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,CAAC,MAAM,CAAC;wBACnF,OAAO,EAAE,QAAQ,CAAC,OAAO;qBAC1B;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,4EAA4E;QAC5E,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAEjD,MAAM,IAAI,GAAsB,EAAE,CAAC;QACnC,8EAA8E;QAC9E,6EAA6E;QAC7E,4EAA4E;QAC5E,wDAAwD;QACxD,MAAM,SAAS,GAAwB,IAAI,GAAG,EAAkB,CAAC;QACjE,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM;YACR,CAAC;YACD,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;gBAC/B,MAAM,IAAI,GAAW,MAAA,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,mCAAI,CAAC,CAAC;gBACvD,IAAI,IAAI,IAAI,YAAY,EAAE,CAAC;oBACzB,SAAS;gBACX,CAAC;gBACD,SAAS,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC;YACzC,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,OAAO,CAAC,OAAO,CAAC,IAAA,kBAAO,EAAC,IAAI,CAAC,CAAC,CAAC;IACxC,CAAC;IAED;;;;;;;;;;;OAWG;IACI,KAAK,CAAC,OAAO,CAAC,MAA2B,EAAE,KAAuB;QACvE,IAAI,CAAC,MAAM,EAAE,CAAC;QACd,MAAM,MAAM,GAA+C,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC/E,IAAI,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC;YACvB,OAAO,IAAA,eAAI,EAAC,mDAAmD,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QACnF,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAClC,MAAM,QAAQ,GAA6C,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACtF,IAAI,QAAQ,CAAC,SAAS,EAAE,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,IAAA,eAAI,EACT,sCAAsC,IAAA,qBAAa,EAAC,MAAM,CAAC,MAAM,CAAC,aAAa,QAAQ,CAAC,OAAO,EAAE,CAClG,CAAC;YACJ,CAAC;YACD,MAAM,KAAK,GAAmB,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;YACrF,IAAI,KAAK,CAAC,SAAS,EAAE,EAAE,CAAC;gBACtB,IAAI,CAAC,MAAM,EAAE,CAAC;gBACd,OAAO,IAAA,eAAI,EAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QACD,OAAO,IAAA,kBAAO,EAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACrC,CAAC;IAED,4DAA4D;IACpD,MAAM;QACZ,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;IAC9B,CAAC;IAED,qDAAqD;IAC7C,MAAM,CAAC,UAAU,CAAC,MAAoB;QAC5C,IAAI,GAAG,GAAW,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,OAAO,CAAC,KAAmB,EAAE,cAAsB,EAAE,MAAoB;QACtF,MAAM,eAAe,GAAW,2BAA2B,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC/E,IAAI,cAAc,KAAK,CAAC,IAAI,eAAe,KAAK,CAAC,EAAE,CAAC;YAClD,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,GAAG,GAAW,CAAC,CAAC;QACpB,KAAK,IAAI,CAAC,GAAW,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9C,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9B,CAAC;QACD,OAAO,GAAG,GAAG,CAAC,cAAc,GAAG,eAAe,CAAC,CAAC;IAClD,CAAC;CACF;AA1MD,kEA0MC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result, fail, succeed } from '@fgv/ts-utils';\nimport { IEdgeTarget, edgeTargetKey } from '../types';\nimport {\n FragmentEmbedder,\n IEmbeddedFragment,\n IFragmentLocator,\n IFragmentVectorIndex,\n IMemoryRecordSource,\n IScopedMemoryRecord,\n IVectorQueryHit\n} from './vectorIndex';\n\n/** One stored fragment: its in-record locator plus the vector for that span. */\ninterface IStoredFragment {\n readonly locator: IFragmentLocator;\n readonly vector: Float32Array;\n}\n\n/** Every stored fragment for one record, tagged with the record's scoped address. */\ninterface IStoredRecordFragments {\n readonly target: IEdgeTarget;\n readonly fragments: IStoredFragment[];\n}\n\n/** A candidate hit carried through selection: the fragment's key, hit, and score. */\ninterface IScoredFragment {\n readonly key: string;\n readonly hit: IVectorQueryHit;\n}\n\n/**\n * The brute-force, in-memory cosine {@link IFragmentVectorIndex} — the\n * fragment-granular sibling of {@link InMemoryCosineIndex}. Stores many\n * `Float32Array`s per record (one per in-record {@link IFragmentLocator | span})\n * and answers a query by computing cosine similarity against every stored\n * fragment, returning the top-k fragment hits by descending score.\n *\n * @remarks\n * Same regime and same non-goals as {@link InMemoryCosineIndex}: no external\n * dependency, no ANN structure, a linear scan over the stored fragments — the seam\n * ({@link IFragmentVectorIndex}) stays open for a consumer to swap a persistent /\n * ANN backend once N grows. `addFragments` is whole-record-replace, so re-authoring\n * a document never leaves stale fragments behind. The index has a single dimension\n * established by the first fragment added; every subsequent fragment and every\n * `query` vector must match it or fail loudly — a mismatched dimension is an\n * embedder-wiring bug, never a silent zero-similarity result.\n * {@link InMemoryFragmentCosineIndex.rebuild | rebuild} clears the index (and the\n * established dimension), so a re-embed with a different model is supported.\n *\n * The optional `maxPerRecord` cap on `query` is applied **during selection**, before\n * the `topK` cut, so one long document with many strong fragments cannot crowd every\n * other record out of the result.\n * @public\n */\nexport class InMemoryFragmentCosineIndex implements IFragmentVectorIndex {\n /**\n * Stored fragments keyed by the canonical {@link edgeTargetKey} of the record's\n * scope-qualified address, so two records that share a filename stem across\n * scopes occupy distinct entries and never overwrite each other's fragments.\n */\n private readonly _records: Map<string, IStoredRecordFragments>;\n /** The dimension of every stored fragment vector; `undefined` until the first `add`. */\n private _dimension: number | undefined;\n\n private constructor() {\n this._records = new Map<string, IStoredRecordFragments>();\n this._dimension = undefined;\n }\n\n /** The number of records that currently have at least one stored fragment. */\n public get recordCount(): number {\n return this._records.size;\n }\n\n /** The total number of fragments currently held across all records. */\n public get fragmentCount(): number {\n let total: number = 0;\n for (const record of this._records.values()) {\n total += record.fragments.length;\n }\n return total;\n }\n\n /** Family-convention factory. */\n public static create(): Result<InMemoryFragmentCosineIndex> {\n return succeed(new InMemoryFragmentCosineIndex());\n }\n\n /** {@inheritDoc IFragmentVectorIndex.addFragments} */\n public addFragments(\n target: IEdgeTarget,\n fragments: ReadonlyArray<IEmbeddedFragment>\n ): Promise<Result<number>> {\n const key: string = edgeTargetKey(target);\n // Validate every fragment before mutating any state, so a bad fragment never\n // leaves the record half-replaced OR the index dimension half-established\n // (whole-record-replace must be all-or-nothing). The effective dimension is the\n // established one, or — on a still-dimensionless index — the first fragment's\n // length; it is only committed to `this._dimension` once the whole batch passes.\n const stored: IStoredFragment[] = [];\n let dimension: number | undefined = this._dimension;\n for (const fragment of fragments) {\n if (fragment.vector.length === 0) {\n return Promise.resolve(fail(`fragment index: cannot add '${key}': empty fragment vector`));\n }\n if (dimension === undefined) {\n dimension = fragment.vector.length;\n } else if (fragment.vector.length !== dimension) {\n return Promise.resolve(\n fail(\n `fragment index: cannot add '${key}': fragment dimension ${fragment.vector.length} does not match index dimension ${dimension}`\n )\n );\n }\n // Defensive copy: the caller may reuse or mutate the buffer after `addFragments`.\n stored.push({ locator: fragment.locator, vector: Float32Array.from(fragment.vector) });\n }\n // Whole-record replace: an empty `fragments` array drops the record entirely\n // rather than leaving an empty shell behind. Commit the (possibly newly-derived)\n // dimension only alongside a successful, non-empty store.\n if (stored.length === 0) {\n this._records.delete(key);\n } else {\n this._dimension = dimension;\n this._records.set(key, { target, fragments: stored });\n }\n return Promise.resolve(succeed(stored.length));\n }\n\n /** {@inheritDoc IFragmentVectorIndex.remove} */\n public remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>> {\n this._records.delete(edgeTargetKey(target));\n return Promise.resolve(succeed(target));\n }\n\n /** {@inheritDoc IFragmentVectorIndex.query} */\n public query(\n vector: Float32Array,\n topK: number,\n maxPerRecord?: number\n ): Promise<Result<ReadonlyArray<IVectorQueryHit>>> {\n if (topK <= 0 || this._records.size === 0) {\n return Promise.resolve(succeed([]));\n }\n if (vector.length !== this._dimension) {\n return Promise.resolve(\n fail(\n `fragment index: query dimension ${vector.length} does not match index dimension ${this._dimension}`\n )\n );\n }\n const queryMagnitude: number = InMemoryFragmentCosineIndex._magnitude(vector);\n const scored: IScoredFragment[] = [];\n for (const record of this._records.values()) {\n for (const fragment of record.fragments) {\n scored.push({\n key: edgeTargetKey(record.target),\n hit: {\n target: record.target,\n score: InMemoryFragmentCosineIndex._cosine(vector, queryMagnitude, fragment.vector),\n locator: fragment.locator\n }\n });\n }\n }\n // Descending by score; the caller re-resolves each `(target, locator)` hit.\n scored.sort((a, b) => b.hit.score - a.hit.score);\n\n const hits: IVectorQueryHit[] = [];\n // Apply the per-record cap during selection (before the topK cut) so a single\n // long document cannot monopolize the result. `undefined` maxPerRecord means\n // uncapped; the counter map is always allocated (tiny) so the guard narrows\n // `maxPerRecord` directly without a non-null assertion.\n const perRecord: Map<string, number> = new Map<string, number>();\n for (const candidate of scored) {\n if (hits.length >= topK) {\n break;\n }\n if (maxPerRecord !== undefined) {\n const used: number = perRecord.get(candidate.key) ?? 0;\n if (used >= maxPerRecord) {\n continue;\n }\n perRecord.set(candidate.key, used + 1);\n }\n hits.push(candidate.hit);\n }\n return Promise.resolve(succeed(hits));\n }\n\n /**\n * Re-embed every record from `source` and rebuild the fragment index from\n * scratch. Clears the current contents (and the established dimension) first, so\n * a re-embed with a different model is supported. Returns the total number of\n * fragments indexed.\n *\n * On any failure (list, embed, or add) the index is rolled back to empty rather\n * than left in a partially-rebuilt state.\n *\n * @param source - The scope-qualified record source to re-embed.\n * @param embed - The fragment embedder applied to each record.\n */\n public async rebuild(source: IMemoryRecordSource, embed: FragmentEmbedder): Promise<Result<number>> {\n this._reset();\n const listed: Result<ReadonlyArray<IScopedMemoryRecord>> = await source.list();\n if (listed.isFailure()) {\n return fail(`fragment index rebuild: failed to list records: ${listed.message}`);\n }\n for (const scoped of listed.value) {\n const embedded: Result<ReadonlyArray<IEmbeddedFragment>> = await embed(scoped.record);\n if (embedded.isFailure()) {\n this._reset();\n return fail(\n `fragment index rebuild: embedding '${edgeTargetKey(scoped.target)}' failed: ${embedded.message}`\n );\n }\n const added: Result<number> = await this.addFragments(scoped.target, embedded.value);\n if (added.isFailure()) {\n this._reset();\n return fail(`fragment index rebuild: ${added.message}`);\n }\n }\n return succeed(this.fragmentCount);\n }\n\n /** Empty the index and forget the established dimension. */\n private _reset(): void {\n this._records.clear();\n this._dimension = undefined;\n }\n\n /** The Euclidean magnitude (L2 norm) of a vector. */\n private static _magnitude(vector: Float32Array): number {\n let sum: number = 0;\n for (let i: number = 0; i < vector.length; i++) {\n sum += vector[i] * vector[i];\n }\n return Math.sqrt(sum);\n }\n\n /**\n * Cosine similarity between the query (whose magnitude is precomputed once and\n * reused across the scan) and a stored fragment vector. A zero-magnitude vector\n * on either side yields `0` rather than `NaN`.\n */\n private static _cosine(query: Float32Array, queryMagnitude: number, stored: Float32Array): number {\n const storedMagnitude: number = InMemoryFragmentCosineIndex._magnitude(stored);\n if (queryMagnitude === 0 || storedMagnitude === 0) {\n return 0;\n }\n let dot: number = 0;\n for (let i: number = 0; i < query.length; i++) {\n dot += query[i] * stored[i];\n }\n return dot / (queryMagnitude * storedMagnitude);\n }\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/packlets/vector/index.ts"],"names":[],"mappings":"AAKA,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/packlets/vector/index.ts"],"names":[],"mappings":"AAKA,cAAc,eAAe,CAAC;AAC9B,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC"}
|
|
@@ -20,4 +20,5 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
20
20
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
21
|
__exportStar(require("./vectorIndex"), exports);
|
|
22
22
|
__exportStar(require("./inMemoryCosineIndex"), exports);
|
|
23
|
+
__exportStar(require("./inMemoryFragmentCosineIndex"), exports);
|
|
23
24
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/packlets/vector/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;AAEH,gDAA8B;AAC9B,wDAAsC","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nexport * from './vectorIndex';\nexport * from './inMemoryCosineIndex';\n"]}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/packlets/vector/index.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;AAEH,gDAA8B;AAC9B,wDAAsC;AACtC,gEAA8C","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nexport * from './vectorIndex';\nexport * from './inMemoryCosineIndex';\nexport * from './inMemoryFragmentCosineIndex';\n"]}
|
|
@@ -1,10 +1,25 @@
|
|
|
1
1
|
import { Result } from '@fgv/ts-utils';
|
|
2
2
|
import { IEdgeTarget, IMemoryRecord } from '../types';
|
|
3
3
|
/**
|
|
4
|
-
* A
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* A half-open `[start, end)` span into a record's body — the in-record locator a
|
|
5
|
+
* {@link IFragmentVectorIndex} carries on each fragment hit. `start` is inclusive,
|
|
6
|
+
* `end` exclusive. The unit (character / byte / token offsets) is the consumer's
|
|
7
|
+
* choice: the index stores the two integers opaquely and never interprets them,
|
|
8
|
+
* so they line up with whatever locator the consumer's own read side uses.
|
|
9
|
+
* @public
|
|
10
|
+
*/
|
|
11
|
+
export interface IFragmentLocator {
|
|
12
|
+
/** Inclusive start offset into the record body. */
|
|
13
|
+
readonly start: number;
|
|
14
|
+
/** Exclusive end offset into the record body. */
|
|
15
|
+
readonly end: number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* A single hit returned by {@link IVectorIndex.query} (or
|
|
19
|
+
* {@link IFragmentVectorIndex.query}): the matched record's scope-qualified
|
|
20
|
+
* {@link IEdgeTarget | address} and the backend's similarity score (higher = more
|
|
21
|
+
* similar; the exact scale is backend-defined). Hits are returned in descending
|
|
22
|
+
* score order.
|
|
8
23
|
*
|
|
9
24
|
* @remarks
|
|
10
25
|
* The address is a `(scope, id)` pair, NOT a bare {@link MemoryId} — per-scope
|
|
@@ -12,6 +27,10 @@ import { IEdgeTarget, IMemoryRecord } from '../types';
|
|
|
12
27
|
* stem under different scopes, so a bare id could not disambiguate two records
|
|
13
28
|
* that share a stem. The caller re-resolves the hit against the record index by
|
|
14
29
|
* the same scoped address.
|
|
30
|
+
*
|
|
31
|
+
* `locator` is present only on hits from a {@link IFragmentVectorIndex} — it
|
|
32
|
+
* identifies WHICH fragment of the record matched. Record-granular
|
|
33
|
+
* {@link IVectorIndex} hits omit it.
|
|
15
34
|
* @public
|
|
16
35
|
*/
|
|
17
36
|
export interface IVectorQueryHit {
|
|
@@ -19,6 +38,8 @@ export interface IVectorQueryHit {
|
|
|
19
38
|
readonly target: IEdgeTarget;
|
|
20
39
|
/** Backend similarity score; higher is more similar. */
|
|
21
40
|
readonly score: number;
|
|
41
|
+
/** The matched fragment's in-record span; present only for fragment-index hits. */
|
|
42
|
+
readonly locator?: IFragmentLocator;
|
|
22
43
|
}
|
|
23
44
|
/**
|
|
24
45
|
* The vector-index seam an embedding backend implements to make
|
|
@@ -55,6 +76,57 @@ export interface IVectorIndex {
|
|
|
55
76
|
*/
|
|
56
77
|
query(vector: Float32Array, topK: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
|
|
57
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* One embedded fragment of a record: its in-record {@link IFragmentLocator | span}
|
|
81
|
+
* and the vector for that span. Produced by a {@link FragmentEmbedder} and stored
|
|
82
|
+
* via {@link IFragmentVectorIndex.addFragments}.
|
|
83
|
+
* @public
|
|
84
|
+
*/
|
|
85
|
+
export interface IEmbeddedFragment {
|
|
86
|
+
/** The fragment's in-record span. */
|
|
87
|
+
readonly locator: IFragmentLocator;
|
|
88
|
+
/** The embedding vector for that span. */
|
|
89
|
+
readonly vector: Float32Array;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* The fragment-granular sibling of {@link IVectorIndex}: instead of one vector per
|
|
93
|
+
* record it holds many vectors per record, each tagged with an in-record
|
|
94
|
+
* {@link IFragmentLocator}, and its `query` returns per-fragment hits carrying that
|
|
95
|
+
* locator. This is the seam behind sub-document semantic search — the "discovery"
|
|
96
|
+
* half of a search-then-read contract, where a hit's `(target, locator)` tells the
|
|
97
|
+
* consumer which record AND which span to read.
|
|
98
|
+
*
|
|
99
|
+
* @remarks
|
|
100
|
+
* Deliberately NOT `extends IVectorIndex`: an index keyed by `(target, locator)`
|
|
101
|
+
* has no well-defined single-vector `add(target, vector)`. It is a parallel
|
|
102
|
+
* contract with three operations — `addFragments`, `remove`, `query` — reusing
|
|
103
|
+
* {@link IVectorQueryHit} (whose `locator` is always populated here). Kept distinct
|
|
104
|
+
* from the record-granular index per the consumer contract: memory recall stays
|
|
105
|
+
* record-granular; sub-document knowledge uses a separate fragment index.
|
|
106
|
+
* @public
|
|
107
|
+
*/
|
|
108
|
+
export interface IFragmentVectorIndex {
|
|
109
|
+
/**
|
|
110
|
+
* Add (or replace) all fragments for the scope-qualified `target`. Whole-record
|
|
111
|
+
* semantics: every fragment previously held for `target` is dropped and replaced
|
|
112
|
+
* by `fragments`, so a re-authored document never leaves stale fragments behind.
|
|
113
|
+
* Returns the number of fragments now held for the record.
|
|
114
|
+
*/
|
|
115
|
+
addFragments(target: IEdgeTarget, fragments: ReadonlyArray<IEmbeddedFragment>): Promise<Result<number>>;
|
|
116
|
+
/**
|
|
117
|
+
* Remove every fragment for the scope-qualified `target`. Returns the removed
|
|
118
|
+
* target. Idempotent — removing a target with no fragments still succeeds.
|
|
119
|
+
*/
|
|
120
|
+
remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;
|
|
121
|
+
/**
|
|
122
|
+
* Return the `topK` nearest fragments to `vector`, in descending score order,
|
|
123
|
+
* each hit carrying its record `target` and fragment `locator`. When
|
|
124
|
+
* `maxPerRecord` is supplied, no more than that many fragments of any single
|
|
125
|
+
* record appear in the result — the cap is applied during selection (before the
|
|
126
|
+
* `topK` cut) so one long document cannot crowd out others.
|
|
127
|
+
*/
|
|
128
|
+
query(vector: Float32Array, topK: number, maxPerRecord?: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;
|
|
129
|
+
}
|
|
58
130
|
/**
|
|
59
131
|
* Embeds a complete record into a vector for the store's embed-on-write hook.
|
|
60
132
|
* Async and `Result`-returning, since a real embedder does a network call (cloud
|
|
@@ -63,6 +135,15 @@ export interface IVectorIndex {
|
|
|
63
135
|
* @public
|
|
64
136
|
*/
|
|
65
137
|
export type MemoryEmbedder = (record: IMemoryRecord<unknown>) => Promise<Result<Float32Array>>;
|
|
138
|
+
/**
|
|
139
|
+
* The fragment-granular sibling of {@link MemoryEmbedder}: chunks a record's body
|
|
140
|
+
* and embeds each chunk, returning one {@link IEmbeddedFragment} per chunk. The
|
|
141
|
+
* chunking policy (window size, overlap) lives entirely in the consumer's embedder
|
|
142
|
+
* — the core stays chunking-agnostic, exactly as it stays embedder-agnostic for
|
|
143
|
+
* the record-granular path. Used by the store's fragment-embed-on-write hook.
|
|
144
|
+
* @public
|
|
145
|
+
*/
|
|
146
|
+
export type FragmentEmbedder = (record: IMemoryRecord<unknown>) => Promise<Result<ReadonlyArray<IEmbeddedFragment>>>;
|
|
66
147
|
/**
|
|
67
148
|
* A record paired with its scope-qualified {@link IEdgeTarget | address}, as
|
|
68
149
|
* yielded by {@link IMemoryRecordSource.list}. The address is required because
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vectorIndex.d.ts","sourceRoot":"","sources":["../../../src/packlets/vector/vectorIndex.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEtD
|
|
1
|
+
{"version":3,"file":"vectorIndex.d.ts","sourceRoot":"","sources":["../../../src/packlets/vector/vectorIndex.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEtD;;;;;;;GAOG;AACH,MAAM,WAAW,gBAAgB;IAC/B,mDAAmD;IACnD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,iDAAiD;IACjD,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,eAAe;IAC9B,yDAAyD;IACzD,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,wDAAwD;IACxD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,mFAAmF;IACnF,QAAQ,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,YAAY;IAC3B;;;;;;OAMG;IACH,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAExE;;;;OAIG;IACH,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;IAE1D;;OAEG;IACH,KAAK,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;CAC5F;AAED;;;;;GAKG;AACH,MAAM,WAAW,iBAAiB;IAChC,qCAAqC;IACrC,QAAQ,CAAC,OAAO,EAAE,gBAAgB,CAAC;IACnC,0CAA0C;IAC1C,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,WAAW,oBAAoB;IACnC;;;;;OAKG;IACH,YAAY,CAAC,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,CAAC,iBAAiB,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IAExG;;;OAGG;IACH,MAAM,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC;IAE1D;;;;;;OAMG;IACH,KAAK,CACH,MAAM,EAAE,YAAY,EACpB,IAAI,EAAE,MAAM,EACZ,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;CACpD;AAED;;;;;;GAMG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC;AAE/F;;;;;;;GAOG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAC7B,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,KAC3B,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;AAEvD;;;;;;;GAOG;AACH,MAAM,WAAW,mBAAmB;IAClC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,iDAAiD;IACjD,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC;CACzC;AAED;;;;;;;;;GASG;AACH,MAAM,WAAW,mBAAmB;IAClC,2EAA2E;IAC3E,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;CAC7D"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vectorIndex.js","sourceRoot":"","sources":["../../../src/packlets/vector/vectorIndex.ts"],"names":[],"mappings":";AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result } from '@fgv/ts-utils';\nimport { IEdgeTarget, IMemoryRecord } from '../types';\n\n/**\n * A single hit returned by {@link IVectorIndex.query}: the matched record's\n *
|
|
1
|
+
{"version":3,"file":"vectorIndex.js","sourceRoot":"","sources":["../../../src/packlets/vector/vectorIndex.ts"],"names":[],"mappings":";AAAA;;;GAGG","sourcesContent":["/*\n * Copyright (c) 2026 Erik Fortune\n * SPDX-License-Identifier: MIT\n */\n\nimport { Result } from '@fgv/ts-utils';\nimport { IEdgeTarget, IMemoryRecord } from '../types';\n\n/**\n * A half-open `[start, end)` span into a record's body — the in-record locator a\n * {@link IFragmentVectorIndex} carries on each fragment hit. `start` is inclusive,\n * `end` exclusive. The unit (character / byte / token offsets) is the consumer's\n * choice: the index stores the two integers opaquely and never interprets them,\n * so they line up with whatever locator the consumer's own read side uses.\n * @public\n */\nexport interface IFragmentLocator {\n /** Inclusive start offset into the record body. */\n readonly start: number;\n /** Exclusive end offset into the record body. */\n readonly end: number;\n}\n\n/**\n * A single hit returned by {@link IVectorIndex.query} (or\n * {@link IFragmentVectorIndex.query}): the matched record's scope-qualified\n * {@link IEdgeTarget | address} and the backend's similarity score (higher = more\n * similar; the exact scale is backend-defined). Hits are returned in descending\n * score order.\n *\n * @remarks\n * The address is a `(scope, id)` pair, NOT a bare {@link MemoryId} — per-scope\n * codecs (e.g. the medium-term codec's `turn-<n>` stems) legally mint the same\n * stem under different scopes, so a bare id could not disambiguate two records\n * that share a stem. The caller re-resolves the hit against the record index by\n * the same scoped address.\n *\n * `locator` is present only on hits from a {@link IFragmentVectorIndex} — it\n * identifies WHICH fragment of the record matched. Record-granular\n * {@link IVectorIndex} hits omit it.\n * @public\n */\nexport interface IVectorQueryHit {\n /** The scope-qualified address of the matched record. */\n readonly target: IEdgeTarget;\n /** Backend similarity score; higher is more similar. */\n readonly score: number;\n /** The matched fragment's in-record span; present only for fragment-index hits. */\n readonly locator?: IFragmentLocator;\n}\n\n/**\n * The vector-index seam an embedding backend implements to make\n * {@link SemanticRetriever | semantic recall} operational.\n *\n * @remarks\n * Vectors cross this seam as `Float32Array` (the in-memory representation an\n * embedding model produces); `number[]` is reserved for the JSON-wire edges\n * (e.g. a provider's embedding response). The in-package brute-force cosine\n * implementation is {@link InMemoryCosineIndex}; a consumer can swap an external\n * ANN backend behind the same seam once N grows beyond the in-memory regime.\n *\n * Every operation returns a `Result` (async, since a real backend does I/O) so\n * failure is explicit and never throws across the seam.\n * @public\n */\nexport interface IVectorIndex {\n /**\n * Add (or replace) the embedding for the scope-qualified `target`. Returns the\n * opaque {@link IMemoryEnvelope.embeddingRef | embeddingRef} the store stamps\n * onto the envelope so a later read knows the record is embedded. Keying on the\n * `(scope, id)` address (not a bare id) is load-bearing: two records that share\n * a filename stem across scopes must not clobber each other's embedding.\n */\n add(target: IEdgeTarget, vector: Float32Array): Promise<Result<string>>;\n\n /**\n * Remove the embedding for the scope-qualified `target`. Returns the removed\n * target. Idempotent — removing a target with no embedding still succeeds\n * (returns the target).\n */\n remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;\n\n /**\n * Return the `topK` nearest records to `vector`, in descending score order.\n */\n query(vector: Float32Array, topK: number): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;\n}\n\n/**\n * One embedded fragment of a record: its in-record {@link IFragmentLocator | span}\n * and the vector for that span. Produced by a {@link FragmentEmbedder} and stored\n * via {@link IFragmentVectorIndex.addFragments}.\n * @public\n */\nexport interface IEmbeddedFragment {\n /** The fragment's in-record span. */\n readonly locator: IFragmentLocator;\n /** The embedding vector for that span. */\n readonly vector: Float32Array;\n}\n\n/**\n * The fragment-granular sibling of {@link IVectorIndex}: instead of one vector per\n * record it holds many vectors per record, each tagged with an in-record\n * {@link IFragmentLocator}, and its `query` returns per-fragment hits carrying that\n * locator. This is the seam behind sub-document semantic search — the \"discovery\"\n * half of a search-then-read contract, where a hit's `(target, locator)` tells the\n * consumer which record AND which span to read.\n *\n * @remarks\n * Deliberately NOT `extends IVectorIndex`: an index keyed by `(target, locator)`\n * has no well-defined single-vector `add(target, vector)`. It is a parallel\n * contract with three operations — `addFragments`, `remove`, `query` — reusing\n * {@link IVectorQueryHit} (whose `locator` is always populated here). Kept distinct\n * from the record-granular index per the consumer contract: memory recall stays\n * record-granular; sub-document knowledge uses a separate fragment index.\n * @public\n */\nexport interface IFragmentVectorIndex {\n /**\n * Add (or replace) all fragments for the scope-qualified `target`. Whole-record\n * semantics: every fragment previously held for `target` is dropped and replaced\n * by `fragments`, so a re-authored document never leaves stale fragments behind.\n * Returns the number of fragments now held for the record.\n */\n addFragments(target: IEdgeTarget, fragments: ReadonlyArray<IEmbeddedFragment>): Promise<Result<number>>;\n\n /**\n * Remove every fragment for the scope-qualified `target`. Returns the removed\n * target. Idempotent — removing a target with no fragments still succeeds.\n */\n remove(target: IEdgeTarget): Promise<Result<IEdgeTarget>>;\n\n /**\n * Return the `topK` nearest fragments to `vector`, in descending score order,\n * each hit carrying its record `target` and fragment `locator`. When\n * `maxPerRecord` is supplied, no more than that many fragments of any single\n * record appear in the result — the cap is applied during selection (before the\n * `topK` cut) so one long document cannot crowd out others.\n */\n query(\n vector: Float32Array,\n topK: number,\n maxPerRecord?: number\n ): Promise<Result<ReadonlyArray<IVectorQueryHit>>>;\n}\n\n/**\n * Embeds a complete record into a vector for the store's embed-on-write hook.\n * Async and `Result`-returning, since a real embedder does a network call (cloud\n * provider) or in-process model inference. The consumer wires this — the core\n * package never calls an embedding provider directly, staying embedder-agnostic.\n * @public\n */\nexport type MemoryEmbedder = (record: IMemoryRecord<unknown>) => Promise<Result<Float32Array>>;\n\n/**\n * The fragment-granular sibling of {@link MemoryEmbedder}: chunks a record's body\n * and embeds each chunk, returning one {@link IEmbeddedFragment} per chunk. The\n * chunking policy (window size, overlap) lives entirely in the consumer's embedder\n * — the core stays chunking-agnostic, exactly as it stays embedder-agnostic for\n * the record-granular path. Used by the store's fragment-embed-on-write hook.\n * @public\n */\nexport type FragmentEmbedder = (\n record: IMemoryRecord<unknown>\n) => Promise<Result<ReadonlyArray<IEmbeddedFragment>>>;\n\n/**\n * A record paired with its scope-qualified {@link IEdgeTarget | address}, as\n * yielded by {@link IMemoryRecordSource.list}. The address is required because\n * {@link InMemoryCosineIndex.rebuild} keys each re-embedded entry on the\n * scope-qualified target, not a bare {@link MemoryId} — two records that share a\n * filename stem across scopes must not collide when the whole vault is re-indexed.\n * @public\n */\nexport interface IScopedMemoryRecord {\n /** The record's scope-qualified `(scope, id)` address. */\n readonly target: IEdgeTarget;\n /** The record itself, passed to the embedder. */\n readonly record: IMemoryRecord<unknown>;\n}\n\n/**\n * The minimal record-source surface {@link InMemoryCosineIndex.rebuild} reads to\n * re-embed an entire vault. Each entry carries the record's scope-qualified\n * address (see {@link IScopedMemoryRecord}) so the rebuild keys the vector index\n * exactly as the incremental embed-on-write path does. A consumer backs this with\n * the store's scoped index — the vector packlet does not import the store packlet\n * (which depends on the vector packlet for {@link IVectorIndex}, so the reverse\n * import would be a cycle).\n * @public\n */\nexport interface IMemoryRecordSource {\n /** List every record in the vault, each paired with its scoped address. */\n list(): Promise<Result<ReadonlyArray<IScopedMemoryRecord>>>;\n}\n"]}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fragmentSemanticRetriever.test.d.ts","sourceRoot":"","sources":["../../../../src/test/unit/retrieve/fragmentSemanticRetriever.test.ts"],"names":[],"mappings":"AAKA,OAAO,oBAAoB,CAAC"}
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/*
|
|
3
|
+
* Copyright (c) 2026 Erik Fortune
|
|
4
|
+
* SPDX-License-Identifier: MIT
|
|
5
|
+
*/
|
|
6
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
|
+
require("@fgv/ts-utils-jest");
|
|
8
|
+
const ts_utils_1 = require("@fgv/ts-utils");
|
|
9
|
+
const index_1 = require("../../../index");
|
|
10
|
+
function target(scope, id) {
|
|
11
|
+
return { scope: scope, id: id };
|
|
12
|
+
}
|
|
13
|
+
function loc(start, end) {
|
|
14
|
+
return { start, end };
|
|
15
|
+
}
|
|
16
|
+
function hit(scope, id, start, end, score) {
|
|
17
|
+
return { target: target(scope, id), score, locator: loc(start, end) };
|
|
18
|
+
}
|
|
19
|
+
const okEmbed = () => Promise.resolve((0, ts_utils_1.succeed)(Float32Array.from([0.1, 0.2])));
|
|
20
|
+
/** A scripted fragment index that records the args it was queried with. */
|
|
21
|
+
class FakeFragmentIndex {
|
|
22
|
+
constructor(hits, shouldFail = false) {
|
|
23
|
+
this._hits = hits;
|
|
24
|
+
this._fail = shouldFail;
|
|
25
|
+
}
|
|
26
|
+
addFragments(__t, fragments) {
|
|
27
|
+
return Promise.resolve((0, ts_utils_1.succeed)(fragments.length));
|
|
28
|
+
}
|
|
29
|
+
remove(t) {
|
|
30
|
+
return Promise.resolve((0, ts_utils_1.succeed)(t));
|
|
31
|
+
}
|
|
32
|
+
query(__vector, topK, maxPerRecord) {
|
|
33
|
+
this.lastTopK = topK;
|
|
34
|
+
this.lastMaxPerRecord = maxPerRecord;
|
|
35
|
+
return Promise.resolve(this._fail ? (0, ts_utils_1.fail)('fragment backend down') : (0, ts_utils_1.succeed)(this._hits));
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
describe('FragmentSemanticRetriever', () => {
|
|
39
|
+
test('reports supportsFragmentRecall=false when no backend is wired', () => {
|
|
40
|
+
const r = index_1.FragmentSemanticRetriever.create({}).orThrow();
|
|
41
|
+
expect(r.capabilities.supportsFragmentRecall).toBe(false);
|
|
42
|
+
});
|
|
43
|
+
test('reports supportsFragmentRecall=true when a backend is wired', () => {
|
|
44
|
+
const r = index_1.FragmentSemanticRetriever.create({
|
|
45
|
+
backend: { fragmentIndex: new FakeFragmentIndex([]), embedQuery: okEmbed }
|
|
46
|
+
}).orThrow();
|
|
47
|
+
expect(r.capabilities.supportsFragmentRecall).toBe(true);
|
|
48
|
+
});
|
|
49
|
+
test('degrades loudly when no backend is wired — never a silent empty', async () => {
|
|
50
|
+
const r = index_1.FragmentSemanticRetriever.create({}).orThrow();
|
|
51
|
+
expect(await r.retrieve({ semantic: 'hi' })).toFailWith(index_1.FRAGMENT_SEMANTIC_UNWIRED_MESSAGE);
|
|
52
|
+
});
|
|
53
|
+
test('returns the per-fragment hits (target + locator + score) in backend order', async () => {
|
|
54
|
+
const hits = [hit('knowledge', 'doc-a', 0, 5, 0.9), hit('knowledge', 'doc-a', 20, 25, 0.4)];
|
|
55
|
+
const r = index_1.FragmentSemanticRetriever.create({
|
|
56
|
+
backend: { fragmentIndex: new FakeFragmentIndex(hits), embedQuery: okEmbed }
|
|
57
|
+
}).orThrow();
|
|
58
|
+
expect(await r.retrieve({ semantic: 'q' })).toSucceedAndSatisfy((result) => {
|
|
59
|
+
expect(result).toEqual(hits);
|
|
60
|
+
expect(result[0].locator).toEqual(loc(0, 5));
|
|
61
|
+
expect(result[1].locator).toEqual(loc(20, 25));
|
|
62
|
+
});
|
|
63
|
+
});
|
|
64
|
+
test('forwards topK and maxPerRecord to the fragment index', async () => {
|
|
65
|
+
const fragmentIndex = new FakeFragmentIndex([hit('knowledge', 'doc-a', 0, 5, 0.9)]);
|
|
66
|
+
const r = index_1.FragmentSemanticRetriever.create({
|
|
67
|
+
backend: { fragmentIndex, embedQuery: okEmbed }
|
|
68
|
+
}).orThrow();
|
|
69
|
+
expect(await r.retrieve({ semantic: 'q', topK: 3, maxPerRecord: 1 })).toSucceed();
|
|
70
|
+
expect(fragmentIndex.lastTopK).toBe(3);
|
|
71
|
+
expect(fragmentIndex.lastMaxPerRecord).toBe(1);
|
|
72
|
+
});
|
|
73
|
+
test('defaults topK to 10 and leaves maxPerRecord undefined when the query omits them', async () => {
|
|
74
|
+
const fragmentIndex = new FakeFragmentIndex([hit('knowledge', 'doc-a', 0, 5, 0.9)]);
|
|
75
|
+
const r = index_1.FragmentSemanticRetriever.create({
|
|
76
|
+
backend: { fragmentIndex, embedQuery: okEmbed }
|
|
77
|
+
}).orThrow();
|
|
78
|
+
expect(await r.retrieve({ semantic: 'q' })).toSucceed();
|
|
79
|
+
expect(fragmentIndex.lastTopK).toBe(10);
|
|
80
|
+
expect(fragmentIndex.lastMaxPerRecord).toBeUndefined();
|
|
81
|
+
});
|
|
82
|
+
test('fails loudly when the embedder fails', async () => {
|
|
83
|
+
const r = index_1.FragmentSemanticRetriever.create({
|
|
84
|
+
backend: {
|
|
85
|
+
fragmentIndex: new FakeFragmentIndex([]),
|
|
86
|
+
embedQuery: () => Promise.resolve((0, ts_utils_1.fail)('no embed model'))
|
|
87
|
+
}
|
|
88
|
+
}).orThrow();
|
|
89
|
+
expect(await r.retrieve({ semantic: 'q' })).toFailWith(/fragment recall: query embedding failed: no embed model/i);
|
|
90
|
+
});
|
|
91
|
+
test('fails loudly when the fragment backend fails', async () => {
|
|
92
|
+
const r = index_1.FragmentSemanticRetriever.create({
|
|
93
|
+
backend: { fragmentIndex: new FakeFragmentIndex([], true), embedQuery: okEmbed }
|
|
94
|
+
}).orThrow();
|
|
95
|
+
expect(await r.retrieve({ semantic: 'q' })).toFailWith(/fragment recall: fragment query failed: fragment backend down/i);
|
|
96
|
+
});
|
|
97
|
+
test('normalizes a rejecting embedder into a Failure (never escapes as a rejection)', async () => {
|
|
98
|
+
const r = index_1.FragmentSemanticRetriever.create({
|
|
99
|
+
backend: {
|
|
100
|
+
fragmentIndex: new FakeFragmentIndex([]),
|
|
101
|
+
embedQuery: () => Promise.reject(new Error('embedder blew up'))
|
|
102
|
+
}
|
|
103
|
+
}).orThrow();
|
|
104
|
+
expect(await r.retrieve({ semantic: 'q' })).toFailWith(/fragment recall: query embedding failed: .*embedder blew up/i);
|
|
105
|
+
});
|
|
106
|
+
test('normalizes a rejecting fragment backend into a Failure', async () => {
|
|
107
|
+
const rejectingIndex = {
|
|
108
|
+
addFragments: (__t, f) => Promise.resolve((0, ts_utils_1.succeed)(f.length)),
|
|
109
|
+
remove: (t) => Promise.resolve((0, ts_utils_1.succeed)(t)),
|
|
110
|
+
query: () => Promise.reject(new Error('socket hangup'))
|
|
111
|
+
};
|
|
112
|
+
const r = index_1.FragmentSemanticRetriever.create({
|
|
113
|
+
backend: { fragmentIndex: rejectingIndex, embedQuery: okEmbed }
|
|
114
|
+
}).orThrow();
|
|
115
|
+
expect(await r.retrieve({ semantic: 'q' })).toFailWith(/fragment recall: fragment query failed: .*socket hangup/i);
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
//# sourceMappingURL=fragmentSemanticRetriever.test.js.map
|