@seanhogg/builderforce-memory 2026.10.27 → 2026.10.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ingest/IngestionPipeline.d.ts +1 -8
- package/dist/ingest/IngestionPipeline.d.ts.map +1 -1
- package/dist/ingest/IngestionPipeline.js +3 -15
- package/dist/ingest/IngestionPipeline.js.map +1 -1
- package/dist/ingest/docsExtract.d.ts +30 -0
- package/dist/ingest/docsExtract.d.ts.map +1 -0
- package/dist/ingest/docsExtract.js +221 -0
- package/dist/ingest/docsExtract.js.map +1 -0
- package/dist/ingest/hash.d.ts +15 -0
- package/dist/ingest/hash.d.ts.map +1 -0
- package/dist/ingest/hash.js +22 -0
- package/dist/ingest/hash.js.map +1 -0
- package/dist/ingest/parsers.d.ts.map +1 -1
- package/dist/ingest/parsers.js +4 -0
- package/dist/ingest/parsers.js.map +1 -1
- package/dist/ingest/types.d.ts +7 -0
- package/dist/ingest/types.d.ts.map +1 -1
- package/dist/ingest/types.js.map +1 -1
- package/dist/knowledge/build.d.ts +24 -0
- package/dist/knowledge/build.d.ts.map +1 -0
- package/dist/knowledge/build.js +263 -0
- package/dist/knowledge/build.js.map +1 -0
- package/dist/knowledge/cards.d.ts +17 -0
- package/dist/knowledge/cards.d.ts.map +1 -0
- package/dist/knowledge/cards.js +118 -0
- package/dist/knowledge/cards.js.map +1 -0
- package/dist/knowledge/findings.d.ts +24 -0
- package/dist/knowledge/findings.d.ts.map +1 -0
- package/dist/knowledge/findings.js +152 -0
- package/dist/knowledge/findings.js.map +1 -0
- package/dist/knowledge/importance.d.ts +21 -0
- package/dist/knowledge/importance.d.ts.map +1 -0
- package/dist/knowledge/importance.js +70 -0
- package/dist/knowledge/importance.js.map +1 -0
- package/dist/knowledge/index.d.ts +31 -0
- package/dist/knowledge/index.d.ts.map +1 -0
- package/dist/knowledge/index.js +26 -0
- package/dist/knowledge/index.js.map +1 -0
- package/dist/knowledge/judge.d.ts +86 -0
- package/dist/knowledge/judge.d.ts.map +1 -0
- package/dist/knowledge/judge.js +213 -0
- package/dist/knowledge/judge.js.map +1 -0
- package/dist/knowledge/model.d.ts +333 -0
- package/dist/knowledge/model.d.ts.map +1 -0
- package/dist/knowledge/model.js +13 -0
- package/dist/knowledge/model.js.map +1 -0
- package/dist/knowledge/packs/docs.d.ts +17 -0
- package/dist/knowledge/packs/docs.d.ts.map +1 -0
- package/dist/knowledge/packs/docs.js +89 -0
- package/dist/knowledge/packs/docs.js.map +1 -0
- package/dist/knowledge/partitions.d.ts +31 -0
- package/dist/knowledge/partitions.d.ts.map +1 -0
- package/dist/knowledge/partitions.js +135 -0
- package/dist/knowledge/partitions.js.map +1 -0
- package/dist/knowledge/plan.d.ts +33 -0
- package/dist/knowledge/plan.d.ts.map +1 -0
- package/dist/knowledge/plan.js +89 -0
- package/dist/knowledge/plan.js.map +1 -0
- package/dist/knowledge/refs.d.ts +40 -0
- package/dist/knowledge/refs.d.ts.map +1 -0
- package/dist/knowledge/refs.js +102 -0
- package/dist/knowledge/refs.js.map +1 -0
- package/dist/knowledge/resolve.d.ts +35 -0
- package/dist/knowledge/resolve.d.ts.map +1 -0
- package/dist/knowledge/resolve.js +67 -0
- package/dist/knowledge/resolve.js.map +1 -0
- package/dist/knowledge/review.d.ts +23 -0
- package/dist/knowledge/review.d.ts.map +1 -0
- package/dist/knowledge/review.js +233 -0
- package/dist/knowledge/review.js.map +1 -0
- package/dist/knowledge/sources.d.ts +47 -0
- package/dist/knowledge/sources.d.ts.map +1 -0
- package/dist/knowledge/sources.js +136 -0
- package/dist/knowledge/sources.js.map +1 -0
- package/dist/knowledge/stores/localSnapshotStore.d.ts +50 -0
- package/dist/knowledge/stores/localSnapshotStore.d.ts.map +1 -0
- package/dist/knowledge/stores/localSnapshotStore.js +125 -0
- package/dist/knowledge/stores/localSnapshotStore.js.map +1 -0
- package/dist/knowledge/trace.d.ts +32 -0
- package/dist/knowledge/trace.d.ts.map +1 -0
- package/dist/knowledge/trace.js +102 -0
- package/dist/knowledge/trace.js.map +1 -0
- package/dist/knowledge/truncate.d.ts +24 -0
- package/dist/knowledge/truncate.d.ts.map +1 -0
- package/dist/knowledge/truncate.js +128 -0
- package/dist/knowledge/truncate.js.map +1 -0
- package/dist/memory/SemanticRecall.d.ts.map +1 -1
- package/dist/memory/SemanticRecall.js +3 -1
- package/dist/memory/SemanticRecall.js.map +1 -1
- package/dist/retrieval/HybridRetriever.d.ts.map +1 -1
- package/dist/retrieval/HybridRetriever.js +3 -1
- package/dist/retrieval/HybridRetriever.js.map +1 -1
- package/dist/retrieval/bm25.d.ts +3 -2
- package/dist/retrieval/bm25.d.ts.map +1 -1
- package/dist/retrieval/bm25.js +5 -3
- package/dist/retrieval/bm25.js.map +1 -1
- package/dist/retrieval/fusion.d.ts +2 -1
- package/dist/retrieval/fusion.d.ts.map +1 -1
- package/dist/retrieval/fusion.js +4 -2
- package/dist/retrieval/fusion.js.map +1 -1
- package/dist/retrieval/hnsw.d.ts +49 -11
- package/dist/retrieval/hnsw.d.ts.map +1 -1
- package/dist/retrieval/hnsw.js +57 -29
- package/dist/retrieval/hnsw.js.map +1 -1
- package/dist/retrieval/index.d.ts +7 -3
- package/dist/retrieval/index.d.ts.map +1 -1
- package/dist/retrieval/index.js +5 -2
- package/dist/retrieval/index.js.map +1 -1
- package/dist/retrieval/knowledge-index/bytes.d.ts +43 -0
- package/dist/retrieval/knowledge-index/bytes.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/bytes.js +151 -0
- package/dist/retrieval/knowledge-index/bytes.js.map +1 -0
- package/dist/retrieval/knowledge-index/cascade.d.ts +37 -0
- package/dist/retrieval/knowledge-index/cascade.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/cascade.js +111 -0
- package/dist/retrieval/knowledge-index/cascade.js.map +1 -0
- package/dist/retrieval/knowledge-index/compact.d.ts +18 -0
- package/dist/retrieval/knowledge-index/compact.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/compact.js +64 -0
- package/dist/retrieval/knowledge-index/compact.js.map +1 -0
- package/dist/retrieval/knowledge-index/csr.d.ts +41 -0
- package/dist/retrieval/knowledge-index/csr.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/csr.js +76 -0
- package/dist/retrieval/knowledge-index/csr.js.map +1 -0
- package/dist/retrieval/knowledge-index/digest.d.ts +21 -0
- package/dist/retrieval/knowledge-index/digest.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/digest.js +50 -0
- package/dist/retrieval/knowledge-index/digest.js.map +1 -0
- package/dist/retrieval/knowledge-index/embeddings.d.ts +32 -0
- package/dist/retrieval/knowledge-index/embeddings.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/embeddings.js +78 -0
- package/dist/retrieval/knowledge-index/embeddings.js.map +1 -0
- package/dist/retrieval/knowledge-index/expand.d.ts +19 -0
- package/dist/retrieval/knowledge-index/expand.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/expand.js +49 -0
- package/dist/retrieval/knowledge-index/expand.js.map +1 -0
- package/dist/retrieval/knowledge-index/index.d.ts +40 -0
- package/dist/retrieval/knowledge-index/index.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/index.js +134 -0
- package/dist/retrieval/knowledge-index/index.js.map +1 -0
- package/dist/retrieval/knowledge-index/lexical.d.ts +26 -0
- package/dist/retrieval/knowledge-index/lexical.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/lexical.js +76 -0
- package/dist/retrieval/knowledge-index/lexical.js.map +1 -0
- package/dist/retrieval/knowledge-index/pairs.d.ts +22 -0
- package/dist/retrieval/knowledge-index/pairs.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/pairs.js +51 -0
- package/dist/retrieval/knowledge-index/pairs.js.map +1 -0
- package/dist/retrieval/knowledge-index/prune.d.ts +21 -0
- package/dist/retrieval/knowledge-index/prune.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/prune.js +43 -0
- package/dist/retrieval/knowledge-index/prune.js.map +1 -0
- package/dist/retrieval/knowledge-index/quantize.d.ts +37 -0
- package/dist/retrieval/knowledge-index/quantize.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/quantize.js +77 -0
- package/dist/retrieval/knowledge-index/quantize.js.map +1 -0
- package/dist/retrieval/knowledge-index/serialize.d.ts +18 -0
- package/dist/retrieval/knowledge-index/serialize.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/serialize.js +123 -0
- package/dist/retrieval/knowledge-index/serialize.js.map +1 -0
- package/dist/retrieval/knowledge-index/sha256.d.ts +30 -0
- package/dist/retrieval/knowledge-index/sha256.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/sha256.js +130 -0
- package/dist/retrieval/knowledge-index/sha256.js.map +1 -0
- package/dist/retrieval/knowledge-index/state.d.ts +55 -0
- package/dist/retrieval/knowledge-index/state.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/state.js +67 -0
- package/dist/retrieval/knowledge-index/state.js.map +1 -0
- package/dist/retrieval/knowledge-index/tokenize.d.ts +20 -0
- package/dist/retrieval/knowledge-index/tokenize.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/tokenize.js +42 -0
- package/dist/retrieval/knowledge-index/tokenize.js.map +1 -0
- package/dist/retrieval/knowledge-index/traverse.d.ts +26 -0
- package/dist/retrieval/knowledge-index/traverse.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/traverse.js +90 -0
- package/dist/retrieval/knowledge-index/traverse.js.map +1 -0
- package/dist/retrieval/knowledge-index/types.d.ts +85 -0
- package/dist/retrieval/knowledge-index/types.d.ts.map +1 -0
- package/dist/retrieval/knowledge-index/types.js +9 -0
- package/dist/retrieval/knowledge-index/types.js.map +1 -0
- package/dist/retrieval/mmr.d.ts +11 -4
- package/dist/retrieval/mmr.d.ts.map +1 -1
- package/dist/retrieval/mmr.js +28 -8
- package/dist/retrieval/mmr.js.map +1 -1
- package/dist/retrieval/order.d.ts +41 -0
- package/dist/retrieval/order.d.ts.map +1 -0
- package/dist/retrieval/order.js +54 -0
- package/dist/retrieval/order.js.map +1 -0
- package/dist/retrieval/weightedFusion.d.ts +2 -1
- package/dist/retrieval/weightedFusion.d.ts.map +1 -1
- package/dist/retrieval/weightedFusion.js +7 -5
- package/dist/retrieval/weightedFusion.js.map +1 -1
- package/dist/vectorstore/MemoryVectorStore.d.ts.map +1 -1
- package/dist/vectorstore/MemoryVectorStore.js +2 -1
- package/dist/vectorstore/MemoryVectorStore.js.map +1 -1
- package/package.json +16 -5
- package/src/ingest/IngestionPipeline.ts +3 -15
- package/src/ingest/docsExtract.ts +219 -0
- package/src/ingest/hash.ts +22 -0
- package/src/ingest/parsers.ts +4 -0
- package/src/ingest/types.ts +7 -0
- package/src/knowledge/build.test.ts +150 -0
- package/src/knowledge/build.ts +266 -0
- package/src/knowledge/cards.ts +130 -0
- package/src/knowledge/docs.test.ts +118 -0
- package/src/knowledge/findings.ts +159 -0
- package/src/knowledge/importance.ts +72 -0
- package/src/knowledge/index.ts +53 -0
- package/src/knowledge/judge.ts +231 -0
- package/src/knowledge/model.ts +312 -0
- package/src/knowledge/packs/docs.ts +91 -0
- package/src/knowledge/partitions.test.ts +98 -0
- package/src/knowledge/partitions.ts +119 -0
- package/src/knowledge/plan.ts +115 -0
- package/src/knowledge/refs.ts +108 -0
- package/src/knowledge/resolve.ts +78 -0
- package/src/knowledge/review.test.ts +180 -0
- package/src/knowledge/review.ts +235 -0
- package/src/knowledge/sources.ts +146 -0
- package/src/knowledge/stores/localSnapshotStore.test.ts +82 -0
- package/src/knowledge/stores/localSnapshotStore.ts +159 -0
- package/src/knowledge/testkit.test-util.ts +92 -0
- package/src/knowledge/trace.test.ts +91 -0
- package/src/knowledge/trace.ts +111 -0
- package/src/knowledge/truncate.ts +118 -0
- package/src/memory/SemanticRecall.ts +3 -1
- package/src/retrieval/HybridRetriever.ts +3 -1
- package/src/retrieval/bm25.ts +5 -3
- package/src/retrieval/fusion.ts +5 -2
- package/src/retrieval/hnsw.ts +82 -35
- package/src/retrieval/index.ts +17 -3
- package/src/retrieval/knowledge-index/bytes.ts +166 -0
- package/src/retrieval/knowledge-index/cascade.ts +125 -0
- package/src/retrieval/knowledge-index/compact.ts +68 -0
- package/src/retrieval/knowledge-index/csr.ts +102 -0
- package/src/retrieval/knowledge-index/digest.ts +54 -0
- package/src/retrieval/knowledge-index/embeddings.ts +85 -0
- package/src/retrieval/knowledge-index/expand.ts +55 -0
- package/src/retrieval/knowledge-index/index.ts +162 -0
- package/src/retrieval/knowledge-index/lexical.ts +78 -0
- package/src/retrieval/knowledge-index/pairs.ts +57 -0
- package/src/retrieval/knowledge-index/prune.ts +48 -0
- package/src/retrieval/knowledge-index/quantize.ts +78 -0
- package/src/retrieval/knowledge-index/serialize.ts +124 -0
- package/src/retrieval/knowledge-index/sha256.ts +122 -0
- package/src/retrieval/knowledge-index/state.ts +108 -0
- package/src/retrieval/knowledge-index/tokenize.ts +41 -0
- package/src/retrieval/knowledge-index/traverse.ts +95 -0
- package/src/retrieval/knowledge-index/types.ts +78 -0
- package/src/retrieval/mmr.ts +35 -10
- package/src/retrieval/order.ts +68 -0
- package/src/retrieval/weightedFusion.ts +8 -5
- package/src/vectorstore/MemoryVectorStore.ts +2 -1
package/dist/retrieval/hnsw.d.ts
CHANGED
|
@@ -10,8 +10,14 @@
|
|
|
10
10
|
* above a threshold.
|
|
11
11
|
*
|
|
12
12
|
* Distance is cosine similarity (higher = closer); the graph stores `1 - cos`
|
|
13
|
-
* as the metric so "nearer" is "smaller".
|
|
14
|
-
*
|
|
13
|
+
* as the metric so "nearer" is "smaller".
|
|
14
|
+
*
|
|
15
|
+
* Determinism: a node's level is {@link levelOf} its id — a geometric draw from
|
|
16
|
+
* the id's sha-256, never a sequential RNG — and every ordering (candidates,
|
|
17
|
+
* pruning, results) is {@link compareScoreThenId}: score, then id. So the graph
|
|
18
|
+
* depends only on the vectors and the insertion order, and
|
|
19
|
+
* {@link HnswIndex.canonicalBuild} (id-sorted inserts) removes the latter: the
|
|
20
|
+
* same set of (id, vector) gives the same graph whatever order it arrived in.
|
|
15
21
|
*/
|
|
16
22
|
export interface HnswOptions {
|
|
17
23
|
/** Max neighbours per node on layers > 0. Default 16. */
|
|
@@ -20,7 +26,11 @@ export interface HnswOptions {
|
|
|
20
26
|
efConstruction?: number;
|
|
21
27
|
/** Default candidate-list size during search. Default 50. */
|
|
22
28
|
efSearch?: number;
|
|
23
|
-
/**
|
|
29
|
+
/**
|
|
30
|
+
* Retained for API compatibility. It no longer affects anything: levels come
|
|
31
|
+
* from each id's hash ({@link levelOf}), so a build does not depend on a seed
|
|
32
|
+
* or on how many inserts preceded a node.
|
|
33
|
+
*/
|
|
24
34
|
seed?: number;
|
|
25
35
|
}
|
|
26
36
|
export interface SearchHit {
|
|
@@ -28,26 +38,53 @@ export interface SearchHit {
|
|
|
28
38
|
/** Cosine similarity to the query (higher = closer). */
|
|
29
39
|
score: number;
|
|
30
40
|
}
|
|
41
|
+
/** Any numeric vector: a float embedding, or int8 codes (cosine over codes is exact). */
|
|
42
|
+
export type HnswVector = ArrayLike<number>;
|
|
43
|
+
/** A read-only view of the built graph, for a consumer that stores it elsewhere (CSR). */
|
|
44
|
+
export interface HnswGraphView {
|
|
45
|
+
/** Node ids, by node index (insertion order). */
|
|
46
|
+
ids: readonly string[];
|
|
47
|
+
/** neighbours[node][layer] = node indices. */
|
|
48
|
+
neighbours: ReadonlyArray<ReadonlyArray<readonly number[]>>;
|
|
49
|
+
entry: number;
|
|
50
|
+
maxLayer: number;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The HNSW level of `id`: a geometric draw with P(level ≥ L) = M^-L, taken from the
|
|
54
|
+
* eight 32-bit words of sha-256(id) — each word below 2^32 / M is one more level.
|
|
55
|
+
* Integer-only, so it is the same on every platform and independent of insertion.
|
|
56
|
+
*/
|
|
57
|
+
export declare function levelOf(id: string, M: number): number;
|
|
31
58
|
export declare class HnswIndex {
|
|
32
59
|
private readonly _M;
|
|
33
60
|
private readonly _Mmax0;
|
|
34
61
|
private readonly _efC;
|
|
35
62
|
private readonly _efS;
|
|
36
|
-
private readonly _mL;
|
|
37
|
-
private readonly _rng;
|
|
38
63
|
private readonly _nodes;
|
|
39
64
|
private readonly _idToIdx;
|
|
40
65
|
private _entry;
|
|
41
66
|
private _maxLayer;
|
|
42
67
|
constructor(opts?: HnswOptions);
|
|
68
|
+
/**
|
|
69
|
+
* Builds an index by inserting `items` in id order — the canonical build. The
|
|
70
|
+
* same (id, vector) set yields the same graph regardless of the input order.
|
|
71
|
+
* Ids are unique; for a duplicated id the first in input order wins.
|
|
72
|
+
*/
|
|
73
|
+
static canonicalBuild(items: ReadonlyArray<{
|
|
74
|
+
id: string;
|
|
75
|
+
vector: HnswVector;
|
|
76
|
+
}>, opts?: HnswOptions): HnswIndex;
|
|
43
77
|
get size(): number;
|
|
44
78
|
has(id: string): boolean;
|
|
79
|
+
/** The built graph, by node index — what a CSR store and pruning read. */
|
|
80
|
+
graph(): HnswGraphView;
|
|
45
81
|
private _dist;
|
|
46
|
-
|
|
82
|
+
/** Nearest first; equal (quantized) distances by id ascending. */
|
|
83
|
+
private readonly _byDist;
|
|
47
84
|
/** Insert a vector. Ids are unique — re-adding a known id is ignored. */
|
|
48
|
-
add(id: string, vector:
|
|
49
|
-
/** Top-k nearest by cosine similarity (descending). */
|
|
50
|
-
search(query:
|
|
85
|
+
add(id: string, vector: HnswVector): void;
|
|
86
|
+
/** Top-k nearest by cosine similarity (descending; equal scores by id). */
|
|
87
|
+
search(query: HnswVector, k: number, ef?: number): SearchHit[];
|
|
51
88
|
clear(): void;
|
|
52
89
|
/** Greedy hill-climb to the single closest node to `q` on layer `lc`. */
|
|
53
90
|
private _greedyClosest;
|
|
@@ -61,8 +98,9 @@ export declare class HnswIndex {
|
|
|
61
98
|
}
|
|
62
99
|
/**
|
|
63
100
|
* Dense KNN with an exact-scan fast path: below `threshold` candidates a linear
|
|
64
|
-
* cosine scan is simpler and faster; at/above it, build an HNSW index
|
|
65
|
-
*
|
|
101
|
+
* cosine scan is simpler and faster; at/above it, build an HNSW index (the
|
|
102
|
+
* canonical, id-ordered build) and query it. Returned hits are in
|
|
103
|
+
* {@link compareDenseHits} order. Pure given its inputs, and independent of their order.
|
|
66
104
|
*/
|
|
67
105
|
export declare function denseSearch(query: Float32Array, items: Array<{
|
|
68
106
|
id: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hnsw.d.ts","sourceRoot":"","sources":["../../src/retrieval/hnsw.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"hnsw.d.ts","sourceRoot":"","sources":["../../src/retrieval/hnsw.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAMH,MAAM,WAAW,WAAW;IACxB,yDAAyD;IACzD,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,4DAA4D;IAC5D,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,SAAS;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,wDAAwD;IACxD,KAAK,EAAE,MAAM,CAAC;CACjB;AAED,yFAAyF;AACzF,MAAM,MAAM,UAAU,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;AAS3C,0FAA0F;AAC1F,MAAM,WAAW,aAAa;IAC1B,iDAAiD;IACjD,GAAG,EAAE,SAAS,MAAM,EAAE,CAAC;IACvB,8CAA8C;IAC9C,UAAU,EAAE,aAAa,CAAC,aAAa,CAAC,SAAS,MAAM,EAAE,CAAC,CAAC,CAAC;IAC5D,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAOrD;AAED,qBAAa,SAAS;IAClB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAS;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAE9B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAc;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA6B;IACtD,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,SAAS,CAAM;gBAEX,IAAI,GAAE,WAAgB;IAOlC;;;;OAIG;IACH,MAAM,CAAC,cAAc,CAAC,KAAK,EAAE,aAAa,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,UAAU,CAAA;KAAE,CAAC,EAAE,IAAI,CAAC,EAAE,WAAW,GAAG,SAAS;IAQ9G,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO;IAIxB,0EAA0E;IAC1E,KAAK,IAAI,aAAa;IAStB,OAAO,CAAC,KAAK;IAIb,kEAAkE;IAClE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAC4F;IAEpH,yEAAyE;IACzE,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG,IAAI;IA0CzC,2EAA2E;IAC3E,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE,MAAM,EAAE,EAAE,SAAY,GAAG,SAAS,EAAE;IAUjE,KAAK,IAAI,IAAI;IASb,yEAAyE;IACzE,OAAO,CAAC,cAAc;IAkBtB;;;OAGG;IACH,OAAO,CAAC,YAAY;IAoCpB,uEAAuE;IACvE,OAAO,CAAC,MAAM;CAIjB;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CACvB,KAAK,EAAE,YAAY,EACnB,KAAK,EAAE,KAAK,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,YAAY,CAAA;CAAE,CAAC,EAClD,CAAC,EAAE,MAAM,EACT,SAAS,SAAM,EACf,IAAI,CAAC,EAAE,WAAW,GACnB,SAAS,EAAE,CASb"}
|
package/dist/retrieval/hnsw.js
CHANGED
|
@@ -10,25 +10,37 @@
|
|
|
10
10
|
* above a threshold.
|
|
11
11
|
*
|
|
12
12
|
* Distance is cosine similarity (higher = closer); the graph stores `1 - cos`
|
|
13
|
-
* as the metric so "nearer" is "smaller".
|
|
14
|
-
*
|
|
13
|
+
* as the metric so "nearer" is "smaller".
|
|
14
|
+
*
|
|
15
|
+
* Determinism: a node's level is {@link levelOf} its id — a geometric draw from
|
|
16
|
+
* the id's sha-256, never a sequential RNG — and every ordering (candidates,
|
|
17
|
+
* pruning, results) is {@link compareScoreThenId}: score, then id. So the graph
|
|
18
|
+
* depends only on the vectors and the insertion order, and
|
|
19
|
+
* {@link HnswIndex.canonicalBuild} (id-sorted inserts) removes the latter: the
|
|
20
|
+
* same set of (id, vector) gives the same graph whatever order it arrived in.
|
|
15
21
|
*/
|
|
16
22
|
import { cosineSimilarity } from '../similarity/index.js';
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
23
|
+
import { compareScoreThenId, compareDenseHits, compareIds, DENSE_SCORE_QUANTUM } from './order.js';
|
|
24
|
+
import { sha256 } from './knowledge-index/sha256.js';
|
|
25
|
+
/**
|
|
26
|
+
* The HNSW level of `id`: a geometric draw with P(level ≥ L) = M^-L, taken from the
|
|
27
|
+
* eight 32-bit words of sha-256(id) — each word below 2^32 / M is one more level.
|
|
28
|
+
* Integer-only, so it is the same on every platform and independent of insertion.
|
|
29
|
+
*/
|
|
30
|
+
export function levelOf(id, M) {
|
|
31
|
+
const h = sha256(id);
|
|
32
|
+
const view = new DataView(h.buffer, h.byteOffset, h.byteLength);
|
|
33
|
+
const threshold = 0x1_0000_0000 / M;
|
|
34
|
+
let level = 0;
|
|
35
|
+
while (level < 8 && view.getUint32(level * 4) < threshold)
|
|
36
|
+
level++;
|
|
37
|
+
return level;
|
|
24
38
|
}
|
|
25
39
|
export class HnswIndex {
|
|
26
40
|
_M;
|
|
27
41
|
_Mmax0;
|
|
28
42
|
_efC;
|
|
29
43
|
_efS;
|
|
30
|
-
_mL;
|
|
31
|
-
_rng;
|
|
32
44
|
_nodes = [];
|
|
33
45
|
_idToIdx = new Map();
|
|
34
46
|
_entry = -1; // index of the entry-point node
|
|
@@ -38,8 +50,19 @@ export class HnswIndex {
|
|
|
38
50
|
this._Mmax0 = this._M * 2;
|
|
39
51
|
this._efC = Math.max(this._M, opts.efConstruction ?? 200);
|
|
40
52
|
this._efS = Math.max(1, opts.efSearch ?? 50);
|
|
41
|
-
|
|
42
|
-
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Builds an index by inserting `items` in id order — the canonical build. The
|
|
56
|
+
* same (id, vector) set yields the same graph regardless of the input order.
|
|
57
|
+
* Ids are unique; for a duplicated id the first in input order wins.
|
|
58
|
+
*/
|
|
59
|
+
static canonicalBuild(items, opts) {
|
|
60
|
+
const index = new HnswIndex(opts);
|
|
61
|
+
const sorted = items.map((it, i) => ({ it, i }))
|
|
62
|
+
.sort((a, b) => compareIds(a.it.id, b.it.id) || a.i - b.i);
|
|
63
|
+
for (const { it } of sorted)
|
|
64
|
+
index.add(it.id, it.vector);
|
|
65
|
+
return index;
|
|
43
66
|
}
|
|
44
67
|
get size() {
|
|
45
68
|
return this._nodes.length;
|
|
@@ -47,17 +70,25 @@ export class HnswIndex {
|
|
|
47
70
|
has(id) {
|
|
48
71
|
return this._idToIdx.has(id);
|
|
49
72
|
}
|
|
73
|
+
/** The built graph, by node index — what a CSR store and pruning read. */
|
|
74
|
+
graph() {
|
|
75
|
+
return {
|
|
76
|
+
ids: this._nodes.map((n) => n.id),
|
|
77
|
+
neighbours: this._nodes.map((n) => n.neighbours),
|
|
78
|
+
entry: this._entry,
|
|
79
|
+
maxLayer: this._maxLayer,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
50
82
|
_dist(a, b) {
|
|
51
83
|
return 1 - cosineSimilarity(a, b);
|
|
52
84
|
}
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
}
|
|
85
|
+
/** Nearest first; equal (quantized) distances by id ascending. */
|
|
86
|
+
_byDist = (a, b) => compareScoreThenId(1 - a.dist, this._nodes[a.idx].id, 1 - b.dist, this._nodes[b.idx].id, DENSE_SCORE_QUANTUM);
|
|
56
87
|
/** Insert a vector. Ids are unique — re-adding a known id is ignored. */
|
|
57
88
|
add(id, vector) {
|
|
58
89
|
if (this._idToIdx.has(id))
|
|
59
90
|
return;
|
|
60
|
-
const layer = this.
|
|
91
|
+
const layer = levelOf(id, this._M);
|
|
61
92
|
const node = { id, vector, neighbours: Array.from({ length: layer + 1 }, () => []) };
|
|
62
93
|
const idx = this._nodes.length;
|
|
63
94
|
this._nodes.push(node);
|
|
@@ -93,7 +124,7 @@ export class HnswIndex {
|
|
|
93
124
|
this._entry = idx;
|
|
94
125
|
}
|
|
95
126
|
}
|
|
96
|
-
/** Top-k nearest by cosine similarity (descending). */
|
|
127
|
+
/** Top-k nearest by cosine similarity (descending; equal scores by id). */
|
|
97
128
|
search(query, k, ef = this._efS) {
|
|
98
129
|
if (this._entry === -1 || k <= 0)
|
|
99
130
|
return [];
|
|
@@ -170,19 +201,19 @@ export class HnswIndex {
|
|
|
170
201
|
}
|
|
171
202
|
}
|
|
172
203
|
}
|
|
173
|
-
return results.sort(
|
|
204
|
+
return results.sort(this._byDist);
|
|
174
205
|
}
|
|
175
206
|
/** Keep only the `Mmax` closest neighbours of `node` on layer `lc`. */
|
|
176
207
|
_prune(node, lc, Mmax) {
|
|
177
|
-
const
|
|
178
|
-
|
|
179
|
-
node.neighbours[lc] = list.slice(0, Mmax);
|
|
208
|
+
const ranked = node.neighbours[lc].map((idx) => ({ idx, dist: this._dist(node.vector, this._nodes[idx].vector) }));
|
|
209
|
+
node.neighbours[lc] = ranked.sort(this._byDist).slice(0, Mmax).map((r) => r.idx);
|
|
180
210
|
}
|
|
181
211
|
}
|
|
182
212
|
/**
|
|
183
213
|
* Dense KNN with an exact-scan fast path: below `threshold` candidates a linear
|
|
184
|
-
* cosine scan is simpler and faster; at/above it, build an HNSW index
|
|
185
|
-
*
|
|
214
|
+
* cosine scan is simpler and faster; at/above it, build an HNSW index (the
|
|
215
|
+
* canonical, id-ordered build) and query it. Returned hits are in
|
|
216
|
+
* {@link compareDenseHits} order. Pure given its inputs, and independent of their order.
|
|
186
217
|
*/
|
|
187
218
|
export function denseSearch(query, items, k, threshold = 256, hnsw) {
|
|
188
219
|
if (items.length === 0 || k <= 0)
|
|
@@ -190,12 +221,9 @@ export function denseSearch(query, items, k, threshold = 256, hnsw) {
|
|
|
190
221
|
if (items.length < threshold) {
|
|
191
222
|
return items
|
|
192
223
|
.map((it) => ({ id: it.id, score: cosineSimilarity(query, it.vector) }))
|
|
193
|
-
.sort(
|
|
224
|
+
.sort(compareDenseHits)
|
|
194
225
|
.slice(0, k);
|
|
195
226
|
}
|
|
196
|
-
|
|
197
|
-
for (const it of items)
|
|
198
|
-
index.add(it.id, it.vector);
|
|
199
|
-
return index.search(query, k);
|
|
227
|
+
return HnswIndex.canonicalBuild(items, hnsw).search(query, k);
|
|
200
228
|
}
|
|
201
229
|
//# sourceMappingURL=hnsw.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hnsw.js","sourceRoot":"","sources":["../../src/retrieval/hnsw.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"hnsw.js","sourceRoot":"","sources":["../../src/retrieval/hnsw.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,UAAU,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACnG,OAAO,EAAE,MAAM,EAAE,MAAM,6BAA6B,CAAC;AA2CrD;;;;GAIG;AACH,MAAM,UAAU,OAAO,CAAC,EAAU,EAAE,CAAS;IACzC,MAAM,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC;IACrB,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;IAChE,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,CAAC;IACpC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,SAAS;QAAE,KAAK,EAAE,CAAC;IACnE,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,MAAM,OAAO,SAAS;IACD,EAAE,CAAS;IACX,MAAM,CAAS;IACf,IAAI,CAAS;IACb,IAAI,CAAS;IAEb,MAAM,GAAW,EAAE,CAAC;IACpB,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,gCAAgC;IAC7C,SAAS,GAAG,CAAC,CAAC,CAAC;IAEvB,YAAY,OAAoB,EAAE;QAC9B,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACpC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1B,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,CAAC;QAC1D,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IACjD,CAAC;IAED;;;;OAIG;IACH,MAAM,CAAC,cAAc,CAAC,KAAwD,EAAE,IAAkB;QAC9F,MAAM,KAAK,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC;QAClC,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,KAAK,MAAM,EAAE,EAAE,EAAE,IAAI,MAAM;YAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QACzD,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,IAAI,IAAI;QACJ,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC9B,CAAC;IAED,GAAG,CAAC,EAAU;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IACjC,CAAC;IAED,0EAA0E;IAC1E,KAAK;QACD,OAAO;YACH,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;YAChD,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,QAAQ,EAAE,IAAI,CAAC,SAAS;SAC3B,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,CAAa,EAAE,CAAa;QACtC,OAAO,CAAC,GAAG,gBAAgB,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAED,kEAAkE;IACjD,OAAO,GAAG,CAAC,CAAgC,EAAE,CAAgC,EAAU,EAAE,CACtG,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,EAAE,EAAE,mBAAmB,CAAC,CAAC;IAEpH,yEAAyE;IACzE,GAAG,CAAC,EAAU,EAAE,MAAkB;QAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,OAAO;QAClC,MAAM,KAAK,GAAG,OAAO,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QACnC,MAAM,IAAI,GAAS,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;QAC3F,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;QAE3B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;YAClB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;YACvB,OAAO;QACX,CAAC;QAED,+DAA+D;QAC/D,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACrB,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC;YAC7C,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAC7C,CAAC;QAED,iEAAiE;QACjE,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC;YAC3D,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YAChE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;YACvC,0DAA0D;YAC1D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;gBAC1B,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAE,CAAC;gBAC9B,MAAM,IAAI,GAAG,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;gBAC3D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACf,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI;oBAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;YACtD,CAAC;YACD,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,EAAE,CAAC;QAClC,CAAC;QAED,IAAI,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;YACzB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;YACvB,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC;QACtB,CAAC;IACL,CAAC;IAED,2EAA2E;IAC3E,MAAM,CAAC,KAAiB,EAAE,CAAS,EAAE,EAAE,GAAG,IAAI,CAAC,IAAI;QAC/C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,CAAC;QAC5C,IAAI,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QACrB,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC;YACzC,EAAE,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAC5C,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/D,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7F,CAAC;IAED,KAAK;QACD,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACjB,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC;IACxB,CAAC;IAED,8EAA8E;IAE9E,yEAAyE;IACjE,cAAc,CAAC,CAAa,EAAE,KAAa,EAAE,EAAU;QAC3D,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,IAAI,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAE,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,QAAQ,GAAG,IAAI,CAAC;QACpB,OAAO,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,KAAK,CAAC;YACjB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;gBACzD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAE,CAAC,MAAM,CAAC,CAAC;gBACnD,IAAI,CAAC,GAAG,QAAQ,EAAE,CAAC;oBACf,QAAQ,GAAG,CAAC,CAAC;oBACb,IAAI,GAAG,IAAI,CAAC;oBACZ,QAAQ,GAAG,IAAI,CAAC;gBACpB,CAAC;YACL,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC;IAChB,CAAC;IAED;;;OAGG;IACK,YAAY,CAAC,CAAa,EAAE,KAAa,EAAE,EAAU,EAAE,EAAU;QACrE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,CAAC,KAAK,CAAC,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAE,CAAC,MAAM,CAAC,CAAC;QAC5D,wEAAwE;QACxE,MAAM,UAAU,GAAyC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QAC3F,MAAM,OAAO,GAAyC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;QAExF,OAAO,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,yBAAyB;YACzB,IAAI,EAAE,GAAG,CAAC,CAAC;YACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE;gBAAE,IAAI,UAAU,CAAC,CAAC,CAAE,CAAC,IAAI,GAAG,UAAU,CAAC,EAAE,CAAE,CAAC,IAAI;oBAAE,EAAE,GAAG,CAAC,CAAC;YACnG,MAAM,GAAG,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC;YACzC,uBAAuB;YACvB,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;YAC/E,IAAI,GAAG,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,IAAI,EAAE;gBAAE,MAAM;YAEzD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC;gBAC5D,IAAI,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAChC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAClB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAE,CAAC,MAAM,CAAC,CAAC;gBACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAE,CAAC,CAAC;gBAClF,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;oBAC3C,UAAU,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;oBACxC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC;oBACrC,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;wBACtB,6BAA6B;wBAC7B,IAAI,EAAE,GAAG,CAAC,CAAC;wBACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE;4BAAE,IAAI,OAAO,CAAC,CAAC,CAAE,CAAC,IAAI,GAAG,OAAO,CAAC,EAAE,CAAE,CAAC,IAAI;gCAAE,EAAE,GAAG,CAAC,CAAC;wBAC1F,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;oBAC1B,CAAC;gBACL,CAAC;YACL,CAAC;QACL,CAAC;QACD,OAAO,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACtC,CAAC;IAED,uEAAuE;IAC/D,MAAM,CAAC,IAAU,EAAE,EAAU,EAAE,IAAY;QAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,CAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACrH,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACrF,CAAC;CACJ;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CACvB,KAAmB,EACnB,KAAkD,EAClD,CAAS,EACT,SAAS,GAAG,GAAG,EACf,IAAkB;IAElB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAC5C,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,OAAO,KAAK;aACP,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;aACvE,IAAI,CAAC,gBAAgB,CAAC;aACtB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACrB,CAAC;IACD,OAAO,SAAS,CAAC,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAClE,CAAC"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Retrieval layer — chunking, BM25, rank and score fusion, MMR, recency decay, the
|
|
3
|
-
* similarity primitives they rest on,
|
|
3
|
+
* similarity primitives they rest on, the HybridRetriever, the ONE total order every
|
|
4
|
+
* ranker returns (`compareHits`), and the deterministic knowledge index.
|
|
4
5
|
*
|
|
5
6
|
* Zero-dependency and engine-free: this is the `@seanhogg/builderforce-memory/retrieval`
|
|
6
7
|
* subpath, which a Cloudflare Worker, Node and the browser can all import without
|
|
@@ -20,6 +21,9 @@ export type { HybridWeights, HybridArmScores } from './weightedFusion.js';
|
|
|
20
21
|
export { toDecayLambda, calculateTemporalDecayMultiplier, applyTemporalDecayToScore } from './decay.js';
|
|
21
22
|
export { hybridRetrieve } from './HybridRetriever.js';
|
|
22
23
|
export type { RetrievalCandidate, HybridQuery, HybridRetrieveOptions, HybridHit, } from './HybridRetriever.js';
|
|
23
|
-
export { HnswIndex, denseSearch } from './hnsw.js';
|
|
24
|
-
export type { HnswOptions, SearchHit } from './hnsw.js';
|
|
24
|
+
export { HnswIndex, denseSearch, levelOf } from './hnsw.js';
|
|
25
|
+
export type { HnswOptions, SearchHit, HnswVector, HnswGraphView } from './hnsw.js';
|
|
26
|
+
export { compareHits, compareDenseHits, compareScoreThenId, compareIds, quantizeScore, DENSE_SCORE_QUANTUM, LEXICAL_SCORE_QUANTUM, } from './order.js';
|
|
27
|
+
export type { ScoredId } from './order.js';
|
|
28
|
+
export * from './knowledge-index/index.js';
|
|
25
29
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/retrieval/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/retrieval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAEvF,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,YAAY,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEtD,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAChH,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE9E,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAExD,OAAO,EAAE,SAAS,EAAE,wBAAwB,EAAE,4BAA4B,EAAE,MAAM,UAAU,CAAC;AAC7F,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAE5E,OAAO,EACH,4BAA4B,EAC5B,0BAA0B,EAC1B,sBAAsB,EACtB,WAAW,EACX,eAAe,EACf,cAAc,GACjB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAE1E,OAAO,EAAE,aAAa,EAAE,gCAAgC,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAExG,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EACR,kBAAkB,EAClB,WAAW,EACX,qBAAqB,EACrB,SAAS,GACZ,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC5D,YAAY,EAAE,WAAW,EAAE,SAAS,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAEnF,OAAO,EACH,WAAW,EACX,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,aAAa,EACb,mBAAmB,EACnB,qBAAqB,GACxB,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,cAAc,4BAA4B,CAAC"}
|
package/dist/retrieval/index.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Retrieval layer — chunking, BM25, rank and score fusion, MMR, recency decay, the
|
|
3
|
-
* similarity primitives they rest on,
|
|
3
|
+
* similarity primitives they rest on, the HybridRetriever, the ONE total order every
|
|
4
|
+
* ranker returns (`compareHits`), and the deterministic knowledge index.
|
|
4
5
|
*
|
|
5
6
|
* Zero-dependency and engine-free: this is the `@seanhogg/builderforce-memory/retrieval`
|
|
6
7
|
* subpath, which a Cloudflare Worker, Node and the browser can all import without
|
|
@@ -14,5 +15,7 @@ export { mmrSelect, maximalMarginalRelevance, textMaximalMarginalRelevance } fro
|
|
|
14
15
|
export { DEFAULT_HYBRID_VECTOR_WEIGHT, DEFAULT_HYBRID_TEXT_WEIGHT, normalizeHybridWeights, hybridScore, bm25RankToScore, fuseHybridArms, } from './weightedFusion.js';
|
|
15
16
|
export { toDecayLambda, calculateTemporalDecayMultiplier, applyTemporalDecayToScore } from './decay.js';
|
|
16
17
|
export { hybridRetrieve } from './HybridRetriever.js';
|
|
17
|
-
export { HnswIndex, denseSearch } from './hnsw.js';
|
|
18
|
+
export { HnswIndex, denseSearch, levelOf } from './hnsw.js';
|
|
19
|
+
export { compareHits, compareDenseHits, compareScoreThenId, compareIds, quantizeScore, DENSE_SCORE_QUANTUM, LEXICAL_SCORE_QUANTUM, } from './order.js';
|
|
20
|
+
export * from './knowledge-index/index.js';
|
|
18
21
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/retrieval/index.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/retrieval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAEvF,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAGvC,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAGhH,OAAO,EAAE,oBAAoB,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG3E,OAAO,EAAE,SAAS,EAAE,wBAAwB,EAAE,4BAA4B,EAAE,MAAM,UAAU,CAAC;AAG7F,OAAO,EACH,4BAA4B,EAC5B,0BAA0B,EAC1B,sBAAsB,EACtB,WAAW,EACX,eAAe,EACf,cAAc,GACjB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,aAAa,EAAE,gCAAgC,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AAExG,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAQtD,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAG5D,OAAO,EACH,WAAW,EACX,gBAAgB,EAChB,kBAAkB,EAClB,UAAU,EACV,aAAa,EACb,mBAAmB,EACnB,qBAAqB,GACxB,MAAM,YAAY,CAAC;AAGpB,cAAc,4BAA4B,CAAC"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* bytes.ts — the little-endian byte writer/reader that `serialize` and `digest`
|
|
3
|
+
* share, so the persisted format and the hashed canonical state encode every
|
|
4
|
+
* field the same way. DataView only: no Node `Buffer`, no host endianness.
|
|
5
|
+
*/
|
|
6
|
+
export declare class ByteWriter {
|
|
7
|
+
private _buf;
|
|
8
|
+
private _view;
|
|
9
|
+
private _len;
|
|
10
|
+
private _reserve;
|
|
11
|
+
u8(v: number): this;
|
|
12
|
+
u32(v: number): this;
|
|
13
|
+
i32(v: number): this;
|
|
14
|
+
f64(v: number): this;
|
|
15
|
+
/** Raw bytes, no length prefix. */
|
|
16
|
+
raw(bytes: Uint8Array): this;
|
|
17
|
+
/** Length-prefixed UTF-8. */
|
|
18
|
+
str(s: string): this;
|
|
19
|
+
/** Count-prefixed u32 values. */
|
|
20
|
+
u32s(arr: ArrayLike<number>): this;
|
|
21
|
+
/** Count-prefixed f32 values. */
|
|
22
|
+
f32s(arr: ArrayLike<number>): this;
|
|
23
|
+
/** Count-prefixed bytes (an Int8Array's bytes are written as-is). */
|
|
24
|
+
bytes(arr: Uint8Array | Int8Array): this;
|
|
25
|
+
finish(): Uint8Array;
|
|
26
|
+
}
|
|
27
|
+
export declare class ByteReader {
|
|
28
|
+
private readonly _buf;
|
|
29
|
+
private readonly _view;
|
|
30
|
+
private _pos;
|
|
31
|
+
constructor(_buf: Uint8Array);
|
|
32
|
+
private _need;
|
|
33
|
+
u8(): number;
|
|
34
|
+
u32(): number;
|
|
35
|
+
i32(): number;
|
|
36
|
+
f64(): number;
|
|
37
|
+
raw(n: number): Uint8Array;
|
|
38
|
+
str(): string;
|
|
39
|
+
u32s(): Uint32Array;
|
|
40
|
+
f32s(): Float32Array;
|
|
41
|
+
bytes(): Uint8Array;
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=bytes.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bytes.d.ts","sourceRoot":"","sources":["../../../src/retrieval/knowledge-index/bytes.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,qBAAa,UAAU;IACnB,OAAO,CAAC,IAAI,CAAwB;IACpC,OAAO,CAAC,KAAK,CAAkC;IAC/C,OAAO,CAAC,IAAI,CAAK;IAEjB,OAAO,CAAC,QAAQ;IAUhB,EAAE,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAOnB,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAOpB,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAOpB,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAOpB,mCAAmC;IACnC,GAAG,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAO5B,6BAA6B;IAC7B,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKpB,iCAAiC;IACjC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI;IAQlC,iCAAiC;IACjC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI;IAQlC,qEAAqE;IACrE,KAAK,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,GAAG,IAAI;IAKxC,MAAM,IAAI,UAAU;CAGvB;AAED,qBAAa,UAAU;IAIP,OAAO,CAAC,QAAQ,CAAC,IAAI;IAHjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAW;IACjC,OAAO,CAAC,IAAI,CAAK;gBAEY,IAAI,EAAE,UAAU;IAI7C,OAAO,CAAC,KAAK;IAIb,EAAE,IAAI,MAAM;IAKZ,GAAG,IAAI,MAAM;IAOb,GAAG,IAAI,MAAM;IAOb,GAAG,IAAI,MAAM;IAOb,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,UAAU;IAO1B,GAAG,IAAI,MAAM;IAIb,IAAI,IAAI,WAAW;IASnB,IAAI,IAAI,YAAY;IASpB,KAAK,IAAI,UAAU;CAGtB"}
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* bytes.ts — the little-endian byte writer/reader that `serialize` and `digest`
|
|
3
|
+
* share, so the persisted format and the hashed canonical state encode every
|
|
4
|
+
* field the same way. DataView only: no Node `Buffer`, no host endianness.
|
|
5
|
+
*/
|
|
6
|
+
const encoder = new TextEncoder();
|
|
7
|
+
const decoder = new TextDecoder();
|
|
8
|
+
export class ByteWriter {
|
|
9
|
+
_buf = new Uint8Array(1024);
|
|
10
|
+
_view = new DataView(this._buf.buffer);
|
|
11
|
+
_len = 0;
|
|
12
|
+
_reserve(n) {
|
|
13
|
+
if (this._len + n <= this._buf.length)
|
|
14
|
+
return;
|
|
15
|
+
let size = this._buf.length * 2;
|
|
16
|
+
while (size < this._len + n)
|
|
17
|
+
size *= 2;
|
|
18
|
+
const next = new Uint8Array(size);
|
|
19
|
+
next.set(this._buf.subarray(0, this._len));
|
|
20
|
+
this._buf = next;
|
|
21
|
+
this._view = new DataView(next.buffer);
|
|
22
|
+
}
|
|
23
|
+
u8(v) {
|
|
24
|
+
this._reserve(1);
|
|
25
|
+
this._view.setUint8(this._len, v);
|
|
26
|
+
this._len += 1;
|
|
27
|
+
return this;
|
|
28
|
+
}
|
|
29
|
+
u32(v) {
|
|
30
|
+
this._reserve(4);
|
|
31
|
+
this._view.setUint32(this._len, v >>> 0, true);
|
|
32
|
+
this._len += 4;
|
|
33
|
+
return this;
|
|
34
|
+
}
|
|
35
|
+
i32(v) {
|
|
36
|
+
this._reserve(4);
|
|
37
|
+
this._view.setInt32(this._len, v, true);
|
|
38
|
+
this._len += 4;
|
|
39
|
+
return this;
|
|
40
|
+
}
|
|
41
|
+
f64(v) {
|
|
42
|
+
this._reserve(8);
|
|
43
|
+
this._view.setFloat64(this._len, v, true);
|
|
44
|
+
this._len += 8;
|
|
45
|
+
return this;
|
|
46
|
+
}
|
|
47
|
+
/** Raw bytes, no length prefix. */
|
|
48
|
+
raw(bytes) {
|
|
49
|
+
this._reserve(bytes.length);
|
|
50
|
+
this._buf.set(bytes, this._len);
|
|
51
|
+
this._len += bytes.length;
|
|
52
|
+
return this;
|
|
53
|
+
}
|
|
54
|
+
/** Length-prefixed UTF-8. */
|
|
55
|
+
str(s) {
|
|
56
|
+
const b = encoder.encode(s);
|
|
57
|
+
return this.u32(b.length).raw(b);
|
|
58
|
+
}
|
|
59
|
+
/** Count-prefixed u32 values. */
|
|
60
|
+
u32s(arr) {
|
|
61
|
+
this.u32(arr.length);
|
|
62
|
+
this._reserve(arr.length * 4);
|
|
63
|
+
for (let i = 0; i < arr.length; i++)
|
|
64
|
+
this._view.setUint32(this._len + i * 4, arr[i] >>> 0, true);
|
|
65
|
+
this._len += arr.length * 4;
|
|
66
|
+
return this;
|
|
67
|
+
}
|
|
68
|
+
/** Count-prefixed f32 values. */
|
|
69
|
+
f32s(arr) {
|
|
70
|
+
this.u32(arr.length);
|
|
71
|
+
this._reserve(arr.length * 4);
|
|
72
|
+
for (let i = 0; i < arr.length; i++)
|
|
73
|
+
this._view.setFloat32(this._len + i * 4, arr[i], true);
|
|
74
|
+
this._len += arr.length * 4;
|
|
75
|
+
return this;
|
|
76
|
+
}
|
|
77
|
+
/** Count-prefixed bytes (an Int8Array's bytes are written as-is). */
|
|
78
|
+
bytes(arr) {
|
|
79
|
+
this.u32(arr.length);
|
|
80
|
+
return this.raw(new Uint8Array(arr.buffer, arr.byteOffset, arr.byteLength));
|
|
81
|
+
}
|
|
82
|
+
finish() {
|
|
83
|
+
return this._buf.slice(0, this._len);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
export class ByteReader {
|
|
87
|
+
_buf;
|
|
88
|
+
_view;
|
|
89
|
+
_pos = 0;
|
|
90
|
+
constructor(_buf) {
|
|
91
|
+
this._buf = _buf;
|
|
92
|
+
this._view = new DataView(_buf.buffer, _buf.byteOffset, _buf.byteLength);
|
|
93
|
+
}
|
|
94
|
+
_need(n) {
|
|
95
|
+
if (this._pos + n > this._buf.length)
|
|
96
|
+
throw new Error('knowledge-index: truncated bytes');
|
|
97
|
+
}
|
|
98
|
+
u8() {
|
|
99
|
+
this._need(1);
|
|
100
|
+
return this._view.getUint8(this._pos++);
|
|
101
|
+
}
|
|
102
|
+
u32() {
|
|
103
|
+
this._need(4);
|
|
104
|
+
const v = this._view.getUint32(this._pos, true);
|
|
105
|
+
this._pos += 4;
|
|
106
|
+
return v;
|
|
107
|
+
}
|
|
108
|
+
i32() {
|
|
109
|
+
this._need(4);
|
|
110
|
+
const v = this._view.getInt32(this._pos, true);
|
|
111
|
+
this._pos += 4;
|
|
112
|
+
return v;
|
|
113
|
+
}
|
|
114
|
+
f64() {
|
|
115
|
+
this._need(8);
|
|
116
|
+
const v = this._view.getFloat64(this._pos, true);
|
|
117
|
+
this._pos += 8;
|
|
118
|
+
return v;
|
|
119
|
+
}
|
|
120
|
+
raw(n) {
|
|
121
|
+
this._need(n);
|
|
122
|
+
const out = this._buf.slice(this._pos, this._pos + n);
|
|
123
|
+
this._pos += n;
|
|
124
|
+
return out;
|
|
125
|
+
}
|
|
126
|
+
str() {
|
|
127
|
+
return decoder.decode(this.raw(this.u32()));
|
|
128
|
+
}
|
|
129
|
+
u32s() {
|
|
130
|
+
const n = this.u32();
|
|
131
|
+
this._need(n * 4);
|
|
132
|
+
const out = new Uint32Array(n);
|
|
133
|
+
for (let i = 0; i < n; i++)
|
|
134
|
+
out[i] = this._view.getUint32(this._pos + i * 4, true);
|
|
135
|
+
this._pos += n * 4;
|
|
136
|
+
return out;
|
|
137
|
+
}
|
|
138
|
+
f32s() {
|
|
139
|
+
const n = this.u32();
|
|
140
|
+
this._need(n * 4);
|
|
141
|
+
const out = new Float32Array(n);
|
|
142
|
+
for (let i = 0; i < n; i++)
|
|
143
|
+
out[i] = this._view.getFloat32(this._pos + i * 4, true);
|
|
144
|
+
this._pos += n * 4;
|
|
145
|
+
return out;
|
|
146
|
+
}
|
|
147
|
+
bytes() {
|
|
148
|
+
return this.raw(this.u32());
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
//# sourceMappingURL=bytes.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bytes.js","sourceRoot":"","sources":["../../../src/retrieval/knowledge-index/bytes.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAClC,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAElC,MAAM,OAAO,UAAU;IACX,IAAI,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IAC5B,KAAK,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,GAAG,CAAC,CAAC;IAET,QAAQ,CAAC,CAAS;QACtB,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,OAAO;QAC9C,IAAI,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAChC,OAAO,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,CAAC;YAAE,IAAI,IAAI,CAAC,CAAC;QACvC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;QAClC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAC3C,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;QACjB,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,EAAE,CAAC,CAAS;QACR,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACjB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAClC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QACf,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,GAAG,CAAC,CAAS;QACT,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACjB,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QACf,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,GAAG,CAAC,CAAS;QACT,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACjB,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QACf,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,GAAG,CAAC,CAAS;QACT,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACjB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;QAC1C,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QACf,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,mCAAmC;IACnC,GAAG,CAAC,KAAiB;QACjB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QAC1B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,6BAA6B;IAC7B,GAAG,CAAC,CAAS;QACT,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACrC,CAAC;IAED,iCAAiC;IACjC,IAAI,CAAC,GAAsB;QACvB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE;YAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAE,KAAK,CAAC,EAAE,IAAI,CAAC,CAAC;QAClG,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,iCAAiC;IACjC,IAAI,CAAC,GAAsB;QACvB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE;YAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAE,EAAE,IAAI,CAAC,CAAC;QAC7F,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,qEAAqE;IACrE,KAAK,CAAC,GAA2B;QAC7B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC;IAChF,CAAC;IAED,MAAM;QACF,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;CACJ;AAED,MAAM,OAAO,UAAU;IAIU;IAHZ,KAAK,CAAW;IACzB,IAAI,GAAG,CAAC,CAAC;IAEjB,YAA6B,IAAgB;QAAhB,SAAI,GAAJ,IAAI,CAAY;QACzC,IAAI,CAAC,KAAK,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC7E,CAAC;IAEO,KAAK,CAAC,CAAS;QACnB,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAC9F,CAAC;IAED,EAAE;QACE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACd,OAAO,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,CAAC;IAED,GAAG;QACC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACd,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAChD,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QACf,OAAO,CAAC,CAAC;IACb,CAAC;IAED,GAAG;QACC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACd,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC/C,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QACf,OAAO,CAAC,CAAC;IACb,CAAC;IAED,GAAG;QACC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACd,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QACf,OAAO,CAAC,CAAC;IACb,CAAC;IAED,GAAG,CAAC,CAAS;QACT,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;QACtD,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QACf,OAAO,GAAG,CAAC;IACf,CAAC;IAED,GAAG;QACC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IAChD,CAAC;IAED,IAAI;QACA,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC;QAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACnF,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,OAAO,GAAG,CAAC;IACf,CAAC;IAED,IAAI;QACA,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClB,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,CAAC;QACpF,IAAI,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,OAAO,GAAG,CAAC;IACf,CAAC;IAED,KAAK;QACD,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAChC,CAAC;CACJ"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* cascade.ts — the knowledge index's search: cheapest tier first.
|
|
3
|
+
*
|
|
4
|
+
* 1. exact — the folded `keys` map; a hit scores 1.0, and when the exact hits
|
|
5
|
+
* fill `k` nothing else runs;
|
|
6
|
+
* 2. lexical — BM25 over card text with the code-aware tokenizer;
|
|
7
|
+
* 3. dense — int8 graph search for `k × rerankFactor` candidates (plus the
|
|
8
|
+
* cards not yet compacted), then a float rerank over vectors
|
|
9
|
+
* recomputed through the hash-keyed LRU;
|
|
10
|
+
* 4. fuse — RRF over the lexical and dense lists; exact hits stay on top at
|
|
11
|
+
* 1.0; everything in {@link compareHits} order.
|
|
12
|
+
*
|
|
13
|
+
* Dense runs only with an embedder of the stored fingerprint — otherwise the tier
|
|
14
|
+
* is absent, never silently replaced.
|
|
15
|
+
*/
|
|
16
|
+
import type { EmbedderPort, IndexHit, IndexTier } from './types.js';
|
|
17
|
+
import { type IndexState } from './state.js';
|
|
18
|
+
import type { EmbeddingCache } from './embeddings.js';
|
|
19
|
+
export interface SearchContext {
|
|
20
|
+
state: IndexState;
|
|
21
|
+
embedder?: EmbedderPort;
|
|
22
|
+
/** The embedder's recompute cache; present whenever `embedder` is. */
|
|
23
|
+
cache?: EmbeddingCache;
|
|
24
|
+
}
|
|
25
|
+
export interface SearchOptions {
|
|
26
|
+
k?: number;
|
|
27
|
+
tiers?: IndexTier[];
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Dense candidates before the float rerank: the best `n` live graph nodes by
|
|
31
|
+
* integer dot product, plus the best `n` cards not yet in the graph. Ids, unordered
|
|
32
|
+
* between the two sources (the rerank orders them).
|
|
33
|
+
*/
|
|
34
|
+
export declare function denseCandidates(ctx: SearchContext, qv: Float32Array, n: number): Promise<string[]>;
|
|
35
|
+
/** Runs the cascade. A pure function of the state (its digest), `q` and `opts`. */
|
|
36
|
+
export declare function cascadeSearch(ctx: SearchContext, q: string, opts?: SearchOptions): Promise<IndexHit[]>;
|
|
37
|
+
//# sourceMappingURL=cascade.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cascade.d.ts","sourceRoot":"","sources":["../../../src/retrieval/knowledge-index/cascade.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAKH,OAAO,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACpE,OAAO,EAA4C,KAAK,UAAU,EAAE,MAAM,YAAY,CAAC;AACvF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAKtD,MAAM,WAAW,aAAa;IAC1B,KAAK,EAAE,UAAU,CAAC;IAClB,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB,sEAAsE;IACtE,KAAK,CAAC,EAAE,cAAc,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC1B,CAAC,CAAC,EAAE,MAAM,CAAC;IACX,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;CACvB;AAQD;;;;GAIG;AACH,wBAAsB,eAAe,CAAC,GAAG,EAAE,aAAa,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAgCxG;AAiBD,mFAAmF;AACnF,wBAAsB,aAAa,CAAC,GAAG,EAAE,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,IAAI,GAAE,aAAkB,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAyBhH"}
|