@modusensus/dsh-mneme 0.6.7 → 0.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +463 -463
- package/cordis.patch.yml +15 -15
- package/lib/api.js +783 -783
- package/lib/client.js +1757 -1757
- package/lib/commands.js +64 -64
- package/lib/config.js +288 -288
- package/lib/dream/clustering.js +118 -118
- package/lib/dream/decisions.js +439 -439
- package/lib/dream/sleep.js +561 -554
- package/lib/dream/tag-extractor.js +156 -156
- package/lib/dream.js +935 -929
- package/lib/embedding.js +154 -154
- package/lib/entities/extractor.js +242 -242
- package/lib/hot-memory.js +53 -53
- package/lib/index.js +361 -361
- package/lib/inject.js +208 -208
- package/lib/local-embedder.js +282 -282
- package/lib/mirror.js +170 -170
- package/lib/parser/tag.js +59 -59
- package/lib/parser/wiki-link.js +38 -38
- package/lib/quality-filter.js +123 -123
- package/lib/reranker.js +218 -218
- package/lib/search/adaptive.js +22 -22
- package/lib/search/bm25.js +96 -96
- package/lib/search/tag-boost.js +61 -61
- package/lib/service.js +1726 -1726
- package/lib/settings.js +172 -172
- package/lib/store.js +2238 -2238
- package/lib/summarize.js +236 -236
- package/lib/tools.js +290 -290
- package/lib/vector-index.js +116 -116
- package/package.json +80 -80
- package/scripts/benchmark-embed.js +201 -201
- package/scripts/benchmark-recall.js +133 -133
- package/scripts/benchmark-rerank.js +166 -166
- package/scripts/e2e-dsh.js +218 -218
- package/scripts/stress-dsh.js +255 -255
- package/scripts/sync-lib.js +52 -52
- package/src/api.js +783 -783
- package/src/commands.js +64 -64
- package/src/config.js +288 -288
- package/src/dream/clustering.js +118 -118
- package/src/dream/decisions.js +439 -439
- package/src/dream/sleep.js +561 -554
- package/src/dream/tag-extractor.js +156 -156
- package/src/dream.js +935 -929
- package/src/embedding.js +154 -154
- package/src/entities/extractor.js +242 -242
- package/src/hot-memory.js +53 -53
- package/src/index.js +361 -361
- package/src/inject.js +208 -208
- package/src/local-embedder.js +282 -282
- package/src/mirror.js +170 -170
- package/src/parser/tag.js +59 -59
- package/src/parser/wiki-link.js +38 -38
- package/src/quality-filter.js +123 -123
- package/src/reranker.js +218 -218
- package/src/search/adaptive.js +22 -22
- package/src/search/bm25.js +96 -96
- package/src/search/tag-boost.js +61 -61
- package/src/service.js +1726 -1726
- package/src/settings.js +172 -172
- package/src/store.js +2238 -2238
- package/src/summarize.js +236 -236
- package/src/tools.js +290 -290
- package/src/vector-index.js +116 -116
- package/test/api.test.js +594 -594
- package/test/audit.test.js +448 -448
- package/test/benchmark.test.js +35 -35
- package/test/boundary-v0625.test.js +82 -82
- package/test/client.test.js +368 -368
- package/test/clustering.test.js +100 -100
- package/test/commands.test.js +69 -69
- package/test/config.test.js +50 -50
- package/test/conflict-freeze.test.js +290 -290
- package/test/directory.test.js +134 -134
- package/test/dream.test.js +903 -901
- package/test/entities.test.js +522 -522
- package/test/epistemic.test.js +298 -298
- package/test/fnew-0112.test.js +311 -311
- package/test/fnew-03.test.js +422 -422
- package/test/graph-api.test.js +175 -175
- package/test/helpers/dream-mock.js +82 -82
- package/test/hot-memory.test.js +174 -174
- package/test/inject.test.js +103 -103
- package/test/llm-audit.test.js +279 -279
- package/test/local-embedder.test.js +227 -227
- package/test/mirror-dirty.test.js +424 -424
- package/test/mirror-edit-digest.test.js +187 -187
- package/test/mirror-generation.test.js +499 -499
- package/test/mirror.test.js +249 -249
- package/test/normalize-decisions.test.js +120 -120
- package/test/peer-blockers.test.js +190 -190
- package/test/policy-epoch.test.js +259 -259
- package/test/provenance.test.js +103 -103
- package/test/quality-filter.test.js +118 -118
- package/test/reasoning-effort.test.js +199 -199
- package/test/recall-evals.test.js +235 -235
- package/test/recall-layer.test.js +315 -315
- package/test/receipt-chain.test.js +451 -451
- package/test/reflection.test.js +226 -226
- package/test/reranker.test.js +240 -240
- package/test/search-fusion.test.js +90 -90
- package/test/semantic.test.js +124 -124
- package/test/service-search.test.js +199 -199
- package/test/service.test.js +435 -435
- package/test/settings.test.js +118 -118
- package/test/sleep.test.js +365 -365
- package/test/store.test.js +436 -436
- package/test/stress.test.js +209 -209
- package/test/summarize.test.js +191 -191
- package/test/tag-boost.test.js +125 -125
- package/test/tag.test.js +312 -312
- package/test/tools.test.js +285 -285
- package/test/vector-index.test.js +221 -221
- package/test/wiki-link.test.js +332 -332
package/lib/dream/clustering.js
CHANGED
|
@@ -1,118 +1,118 @@
|
|
|
1
|
-
// Cosine similarity of two equal-length numeric vectors. Returns 0 for empty
|
|
2
|
-
// or mismatched-length inputs, and for zero-norm vectors (numerically stable).
|
|
3
|
-
export function cosineSimilarity(a, b) {
|
|
4
|
-
if (!Array.isArray(a) || !Array.isArray(b) || a.length === 0 || a.length !== b.length) return 0;
|
|
5
|
-
let dot = 0, na = 0, nb = 0;
|
|
6
|
-
for (let i = 0; i < a.length; i++) {
|
|
7
|
-
dot += a[i] * b[i];
|
|
8
|
-
na += a[i] * a[i];
|
|
9
|
-
nb += b[i] * b[i];
|
|
10
|
-
}
|
|
11
|
-
if (na === 0 || nb === 0) return 0;
|
|
12
|
-
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
13
|
-
}
|
|
14
|
-
|
|
15
|
-
// K-Means clustering of numeric vectors. Returns one index array per cluster.
|
|
16
|
-
// Uses k-means++ seeding and repairs empty clusters mid-iteration.
|
|
17
|
-
export function kMeans(vectors, k, opts = {}) {
|
|
18
|
-
const n = vectors.length;
|
|
19
|
-
if (n === 0) return [];
|
|
20
|
-
const { maxIter = 100, tol = 1e-4 } = opts ?? {};
|
|
21
|
-
if (k <= 0 || k >= n) return [vectors.map((_, i) => i)];
|
|
22
|
-
|
|
23
|
-
const dim = vectors[0].length;
|
|
24
|
-
const randomIndex = () => Math.floor(Math.random() * n);
|
|
25
|
-
const squaredDist = (v, c) => {
|
|
26
|
-
let s = 0;
|
|
27
|
-
for (let d = 0; d < dim; d++) s += (v[d] - c[d]) * (v[d] - c[d]);
|
|
28
|
-
return s;
|
|
29
|
-
};
|
|
30
|
-
|
|
31
|
-
// k-means++ seeding: first centroid uniform-random, later centroids sampled
|
|
32
|
-
// with probability proportional to squared distance from nearest centroid.
|
|
33
|
-
const centroids = [[...vectors[randomIndex()]]];
|
|
34
|
-
while (centroids.length < k) {
|
|
35
|
-
const dist = new Array(n);
|
|
36
|
-
let sum = 0;
|
|
37
|
-
for (let i = 0; i < n; i++) {
|
|
38
|
-
let best = Infinity;
|
|
39
|
-
for (const c of centroids) best = Math.min(best, squaredDist(vectors[i], c));
|
|
40
|
-
dist[i] = best;
|
|
41
|
-
sum += best;
|
|
42
|
-
}
|
|
43
|
-
if (sum === 0) {
|
|
44
|
-
// All points coincide with a centroid; fall back to uniform random.
|
|
45
|
-
centroids.push([...vectors[randomIndex()]]);
|
|
46
|
-
} else {
|
|
47
|
-
let r = Math.random() * sum;
|
|
48
|
-
let pick = 0;
|
|
49
|
-
for (let i = 0; i < n; i++) {
|
|
50
|
-
r -= dist[i];
|
|
51
|
-
if (r <= 0) { pick = i; break; }
|
|
52
|
-
}
|
|
53
|
-
centroids.push([...vectors[pick]]);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
const assignment = new Array(n).fill(0);
|
|
58
|
-
for (let iter = 0; iter < maxIter; iter++) {
|
|
59
|
-
// Assign each point to its nearest centroid.
|
|
60
|
-
let changed = false;
|
|
61
|
-
for (let i = 0; i < n; i++) {
|
|
62
|
-
let best = 0, bestD = Infinity;
|
|
63
|
-
for (let j = 0; j < k; j++) {
|
|
64
|
-
const d = squaredDist(vectors[i], centroids[j]);
|
|
65
|
-
if (d < bestD) { bestD = d; best = j; }
|
|
66
|
-
}
|
|
67
|
-
if (assignment[i] !== best) { assignment[i] = best; changed = true; }
|
|
68
|
-
}
|
|
69
|
-
if (!changed) break; // stable assignment => converged.
|
|
70
|
-
|
|
71
|
-
// Recompute centroids as cluster means; reseed empty clusters randomly.
|
|
72
|
-
const sums = Array.from({ length: k }, () => new Array(dim).fill(0));
|
|
73
|
-
const counts = new Array(k).fill(0);
|
|
74
|
-
for (let i = 0; i < n; i++) {
|
|
75
|
-
counts[assignment[i]]++;
|
|
76
|
-
for (let d = 0; d < dim; d++) sums[assignment[i]][d] += vectors[i][d];
|
|
77
|
-
}
|
|
78
|
-
let maxMove = 0;
|
|
79
|
-
for (let j = 0; j < k; j++) {
|
|
80
|
-
if (counts[j] === 0) {
|
|
81
|
-
centroids[j] = [...vectors[randomIndex()]];
|
|
82
|
-
maxMove = Infinity; // never call this converged this round.
|
|
83
|
-
} else {
|
|
84
|
-
for (let d = 0; d < dim; d++) {
|
|
85
|
-
const mean = sums[j][d] / counts[j];
|
|
86
|
-
const move = mean - centroids[j][d];
|
|
87
|
-
if (move * move > maxMove) maxMove = move * move;
|
|
88
|
-
centroids[j][d] = mean;
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
if (Math.sqrt(maxMove) < tol) break; // centroids moved less than tol.
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
// Build clusters from final assignments.
|
|
96
|
-
const clusters = Array.from({ length: k }, () => []);
|
|
97
|
-
for (let i = 0; i < n; i++) clusters[assignment[i]].push(i);
|
|
98
|
-
return clusters;
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
// Group memories by vector clustering, preserving original object references.
|
|
102
|
-
export function clusterMemories(memories, vectors, k) {
|
|
103
|
-
return kMeans(vectors, k).map((indices) => indices.map((i) => memories[i]));
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
// Find memory pairs that are highly similar yet could contradict each other.
|
|
107
|
-
// Only same-type pairs count; each pair reported once, lower index first.
|
|
108
|
-
export function findPotentialConflicts(memories, vectors, threshold = 0.85) {
|
|
109
|
-
const pairs = [];
|
|
110
|
-
for (let i = 0; i < memories.length; i++) {
|
|
111
|
-
for (let j = i + 1; j < memories.length; j++) {
|
|
112
|
-
if (memories[i].type !== memories[j].type) continue;
|
|
113
|
-
const sim = cosineSimilarity(vectors[i], vectors[j]);
|
|
114
|
-
if (sim > threshold) pairs.push({ a: memories[i], b: memories[j], similarity: sim });
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
return pairs;
|
|
118
|
-
}
|
|
1
|
+
// Cosine similarity of two equal-length numeric vectors. Returns 0 for empty
|
|
2
|
+
// or mismatched-length inputs, and for zero-norm vectors (numerically stable).
|
|
3
|
+
export function cosineSimilarity(a, b) {
|
|
4
|
+
if (!Array.isArray(a) || !Array.isArray(b) || a.length === 0 || a.length !== b.length) return 0;
|
|
5
|
+
let dot = 0, na = 0, nb = 0;
|
|
6
|
+
for (let i = 0; i < a.length; i++) {
|
|
7
|
+
dot += a[i] * b[i];
|
|
8
|
+
na += a[i] * a[i];
|
|
9
|
+
nb += b[i] * b[i];
|
|
10
|
+
}
|
|
11
|
+
if (na === 0 || nb === 0) return 0;
|
|
12
|
+
return dot / (Math.sqrt(na) * Math.sqrt(nb));
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// K-Means clustering of numeric vectors. Returns one index array per cluster.
|
|
16
|
+
// Uses k-means++ seeding and repairs empty clusters mid-iteration.
|
|
17
|
+
export function kMeans(vectors, k, opts = {}) {
|
|
18
|
+
const n = vectors.length;
|
|
19
|
+
if (n === 0) return [];
|
|
20
|
+
const { maxIter = 100, tol = 1e-4 } = opts ?? {};
|
|
21
|
+
if (k <= 0 || k >= n) return [vectors.map((_, i) => i)];
|
|
22
|
+
|
|
23
|
+
const dim = vectors[0].length;
|
|
24
|
+
const randomIndex = () => Math.floor(Math.random() * n);
|
|
25
|
+
const squaredDist = (v, c) => {
|
|
26
|
+
let s = 0;
|
|
27
|
+
for (let d = 0; d < dim; d++) s += (v[d] - c[d]) * (v[d] - c[d]);
|
|
28
|
+
return s;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
// k-means++ seeding: first centroid uniform-random, later centroids sampled
|
|
32
|
+
// with probability proportional to squared distance from nearest centroid.
|
|
33
|
+
const centroids = [[...vectors[randomIndex()]]];
|
|
34
|
+
while (centroids.length < k) {
|
|
35
|
+
const dist = new Array(n);
|
|
36
|
+
let sum = 0;
|
|
37
|
+
for (let i = 0; i < n; i++) {
|
|
38
|
+
let best = Infinity;
|
|
39
|
+
for (const c of centroids) best = Math.min(best, squaredDist(vectors[i], c));
|
|
40
|
+
dist[i] = best;
|
|
41
|
+
sum += best;
|
|
42
|
+
}
|
|
43
|
+
if (sum === 0) {
|
|
44
|
+
// All points coincide with a centroid; fall back to uniform random.
|
|
45
|
+
centroids.push([...vectors[randomIndex()]]);
|
|
46
|
+
} else {
|
|
47
|
+
let r = Math.random() * sum;
|
|
48
|
+
let pick = 0;
|
|
49
|
+
for (let i = 0; i < n; i++) {
|
|
50
|
+
r -= dist[i];
|
|
51
|
+
if (r <= 0) { pick = i; break; }
|
|
52
|
+
}
|
|
53
|
+
centroids.push([...vectors[pick]]);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const assignment = new Array(n).fill(0);
|
|
58
|
+
for (let iter = 0; iter < maxIter; iter++) {
|
|
59
|
+
// Assign each point to its nearest centroid.
|
|
60
|
+
let changed = false;
|
|
61
|
+
for (let i = 0; i < n; i++) {
|
|
62
|
+
let best = 0, bestD = Infinity;
|
|
63
|
+
for (let j = 0; j < k; j++) {
|
|
64
|
+
const d = squaredDist(vectors[i], centroids[j]);
|
|
65
|
+
if (d < bestD) { bestD = d; best = j; }
|
|
66
|
+
}
|
|
67
|
+
if (assignment[i] !== best) { assignment[i] = best; changed = true; }
|
|
68
|
+
}
|
|
69
|
+
if (!changed) break; // stable assignment => converged.
|
|
70
|
+
|
|
71
|
+
// Recompute centroids as cluster means; reseed empty clusters randomly.
|
|
72
|
+
const sums = Array.from({ length: k }, () => new Array(dim).fill(0));
|
|
73
|
+
const counts = new Array(k).fill(0);
|
|
74
|
+
for (let i = 0; i < n; i++) {
|
|
75
|
+
counts[assignment[i]]++;
|
|
76
|
+
for (let d = 0; d < dim; d++) sums[assignment[i]][d] += vectors[i][d];
|
|
77
|
+
}
|
|
78
|
+
let maxMove = 0;
|
|
79
|
+
for (let j = 0; j < k; j++) {
|
|
80
|
+
if (counts[j] === 0) {
|
|
81
|
+
centroids[j] = [...vectors[randomIndex()]];
|
|
82
|
+
maxMove = Infinity; // never call this converged this round.
|
|
83
|
+
} else {
|
|
84
|
+
for (let d = 0; d < dim; d++) {
|
|
85
|
+
const mean = sums[j][d] / counts[j];
|
|
86
|
+
const move = mean - centroids[j][d];
|
|
87
|
+
if (move * move > maxMove) maxMove = move * move;
|
|
88
|
+
centroids[j][d] = mean;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (Math.sqrt(maxMove) < tol) break; // centroids moved less than tol.
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Build clusters from final assignments.
|
|
96
|
+
const clusters = Array.from({ length: k }, () => []);
|
|
97
|
+
for (let i = 0; i < n; i++) clusters[assignment[i]].push(i);
|
|
98
|
+
return clusters;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Group memories by vector clustering, preserving original object references.
|
|
102
|
+
export function clusterMemories(memories, vectors, k) {
|
|
103
|
+
return kMeans(vectors, k).map((indices) => indices.map((i) => memories[i]));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Find memory pairs that are highly similar yet could contradict each other.
|
|
107
|
+
// Only same-type pairs count; each pair reported once, lower index first.
|
|
108
|
+
export function findPotentialConflicts(memories, vectors, threshold = 0.85) {
|
|
109
|
+
const pairs = [];
|
|
110
|
+
for (let i = 0; i < memories.length; i++) {
|
|
111
|
+
for (let j = i + 1; j < memories.length; j++) {
|
|
112
|
+
if (memories[i].type !== memories[j].type) continue;
|
|
113
|
+
const sim = cosineSimilarity(vectors[i], vectors[j]);
|
|
114
|
+
if (sim > threshold) pairs.push({ a: memories[i], b: memories[j], similarity: sim });
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
return pairs;
|
|
118
|
+
}
|