akm-cli 0.10.261125 → 0.10.261126-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/indexer/indexer.js +13 -19
- package/dist/indexer/search/db-search.js +3 -3
- package/dist/indexer/search/search-source.js +22 -15
- package/dist/workspace/graph-duplicates.js +204 -0
- package/dist/workspace/graph.js +12 -134
- package/docs/architecture/internals/storage-locations.md +0 -1
- package/docs/reference/workspace.md +22 -35
- package/package.json +1 -1
- package/dist/workspace/graph-topics.js +0 -249
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
### Changed
|
|
10
|
+
|
|
11
|
+
- AKM Desktop's library map is a mind map: the library, its asset types, and the tags each type's assets carry most, every label drawn, with a bar under each tag for its size. Contents, Use and Problems recolour the same map; choosing a type or tag opens its figures. It replaces the topic circles of 0.10.261125, whose topic names were word pairs from paths, and drops the Relations view. The map and the replay no longer need WebGL.
|
|
12
|
+
- AKM Desktop's activity replay is a timeline: events per kind stacked over the period with a playhead you click or drag, the mind map of the assets the period touched counting the events on each type and tag, and a log of the latest events. A curate counts as a search, and its results as that search's hits. It replaces the point cloud with flying pulses and the per-agent lanes.
|
|
13
|
+
- `akm-cli/workspace`: `assets.graph()` returns each asset's own tags (`nodes.tags`, indexes into a new `tags` list, lower case); it no longer returns `meaning`, `topics`, `nodes.topic`, `nodes.title`, `nodes.searches`, `emptyBundles` or `unresolvedLinks`. `duplicates` stays.
|
|
14
|
+
|
|
15
|
+
### Fixed
|
|
16
|
+
|
|
17
|
+
- AKM Desktop's library map no longer shows "This view needs WebGL" after the pointer leaves an item, and the map and replay pages scroll as one page instead of a side panel scrolling inside a scrolling page.
|
|
18
|
+
- Matching indexed files to their bundle (curate, related, `search --from`, AKM Desktop's library map) no longer resolves real paths on disk for files outside the enabled bundles: the index and the sources both store real paths, so the spelling is compared. Since 0.10.261125 this added about 3.6 seconds to each library map read on a library of 20,000 indexed files.
|
|
19
|
+
- Each index build no longer applies a full day of decay to utility scores: a score now decays by the time since it was last written, so many builds in a day decay it about one day in total, and a new score starts at its select or feedback rate (itlackey/akm#1659).
|
|
20
|
+
|
|
9
21
|
## [0.10.261125] - 2026-11-14
|
|
10
22
|
|
|
11
23
|
### Added
|
package/dist/indexer/indexer.js
CHANGED
|
@@ -1383,7 +1383,7 @@ function readLookupEntry(db, id, fallbackConceptId, sourceRoot) {
|
|
|
1383
1383
|
* Recompute utility scores based on usage_events data: every entry by default,
|
|
1384
1384
|
* or just `onlyEntryId` (what `akm feedback` passes so the value it stores is
|
|
1385
1385
|
* the one `akm index` would compute, not a second formula). A one-entry call
|
|
1386
|
-
* leaves the retention purge
|
|
1386
|
+
* leaves the retention purge alone.
|
|
1387
1387
|
*
|
|
1388
1388
|
* For each entry:
|
|
1389
1389
|
* - Count search appearances (event_type = 'search')
|
|
@@ -1405,18 +1405,6 @@ export function recomputeUtilityScores(db, stateDb, onlyEntryId) {
|
|
|
1405
1405
|
// (Chunk-8 WI-8.3); its table is created by state migration 020.
|
|
1406
1406
|
if (!scoped)
|
|
1407
1407
|
purgeOldUsageEvents(stateDb, USAGE_EVENT_RETENTION_DAYS);
|
|
1408
|
-
// Decay is raised to the days since the last run, floored at one day. Every
|
|
1409
|
-
// run therefore applies at least one full round of EMA decay (so runs less
|
|
1410
|
-
// than a day apart each decay a full round), and a longer gap applies
|
|
1411
|
-
// proportionally more rounds. The first run counts as one day.
|
|
1412
|
-
const lastComputedAt = getMeta(db, "last_utility_computed_at");
|
|
1413
|
-
let elapsedDays = 1; // default for first run
|
|
1414
|
-
if (lastComputedAt) {
|
|
1415
|
-
const ms = Date.now() - new Date(lastComputedAt).getTime();
|
|
1416
|
-
elapsedDays = Math.max(1, ms / (1000 * 60 * 60 * 24));
|
|
1417
|
-
}
|
|
1418
|
-
const emaDecay = EMA_DECAY ** elapsedDays;
|
|
1419
|
-
const emaNew = 1 - emaDecay; // complement so weights still sum to 1
|
|
1420
1408
|
// Aggregate explicit user demand per `entry_ref` from state.db's usage_events
|
|
1421
1409
|
// (the stable identity every event carries; search and curate telemetry hold
|
|
1422
1410
|
// only the ref, not the index row id), then map each ref to its CURRENT entry
|
|
@@ -1461,11 +1449,12 @@ export function recomputeUtilityScores(db, stateDb, onlyEntryId) {
|
|
|
1461
1449
|
// Batch-load existing utility scores
|
|
1462
1450
|
const existingScores = new Map();
|
|
1463
1451
|
const scoreRows = db
|
|
1464
|
-
.prepare(`SELECT entry_id, utility,
|
|
1452
|
+
.prepare(`SELECT entry_id, utility, updated_at FROM utility_scores${scoped ? " WHERE entry_id = ?" : ""}`)
|
|
1465
1453
|
.all(...(scoped ? [onlyEntryId] : []));
|
|
1466
1454
|
for (const row of scoreRows) {
|
|
1467
|
-
existingScores.set(row.entry_id, { utility: row.utility,
|
|
1455
|
+
existingScores.set(row.entry_id, { utility: row.utility, updatedAt: row.updated_at });
|
|
1468
1456
|
}
|
|
1457
|
+
const now = Date.now();
|
|
1469
1458
|
const entryIds = new Set([...existingScores.keys(), ...usageByEntry.keys()]);
|
|
1470
1459
|
// One transaction for every row: per-row autocommits cost one WAL sync each (~43 s of a 45 s finalize on a
|
|
1471
1460
|
// 20,900-entry library). The reads above stay outside, so state.db is held no longer than before.
|
|
@@ -1484,8 +1473,15 @@ export function recomputeUtilityScores(db, stateDb, onlyEntryId) {
|
|
|
1484
1473
|
const feedbackRate = feedbackTotal > 0 ? Math.max(0, row.positive_feedback_count - row.negative_feedback_count) / feedbackTotal : 0;
|
|
1485
1474
|
const effectiveRate = Math.max(selectRate, feedbackRate);
|
|
1486
1475
|
const existing = existingScores.get(row.entry_id);
|
|
1487
|
-
|
|
1488
|
-
|
|
1476
|
+
// Decay by the time since this row was last written, so any number of builds
|
|
1477
|
+
// in a day apply about one day of decay in total. A new row starts at its rate.
|
|
1478
|
+
let utility = effectiveRate;
|
|
1479
|
+
if (existing) {
|
|
1480
|
+
const updatedMs = existing.updatedAt ? Date.parse(`${existing.updatedAt.replace(" ", "T")}Z`) : Number.NaN;
|
|
1481
|
+
const elapsedDays = Number.isFinite(updatedMs) ? Math.max(0, (now - updatedMs) / 86_400_000) : 1;
|
|
1482
|
+
const decay = EMA_DECAY ** elapsedDays;
|
|
1483
|
+
utility = existing.utility * decay + effectiveRate * (1 - decay);
|
|
1484
|
+
}
|
|
1489
1485
|
// `utility_scores.last_used_at` is consumed by salience as the timestamp of
|
|
1490
1486
|
// the most-recent retrieval. Preserve that meaning by carrying the event's
|
|
1491
1487
|
// timestamp through verbatim. The former `effectiveRate > 0.5 ? now : ...`
|
|
@@ -1505,7 +1501,5 @@ export function recomputeUtilityScores(db, stateDb, onlyEntryId) {
|
|
|
1505
1501
|
lastUsedAt,
|
|
1506
1502
|
});
|
|
1507
1503
|
}
|
|
1508
|
-
if (!scoped)
|
|
1509
|
-
setMeta(db, "last_utility_computed_at", new Date().toISOString());
|
|
1510
1504
|
})();
|
|
1511
1505
|
}
|
|
@@ -33,7 +33,7 @@ import { ftsQueryTokens, parseRefPrefixQuery } from "./fts-query.js";
|
|
|
33
33
|
import { applyFeedback, applyPrefer, applyRecency, reciprocalRankFusion, } from "./ranking.js";
|
|
34
34
|
import { attachSearchHitAttribution } from "./search-attribution.js";
|
|
35
35
|
import { enrichSearchHit } from "./search-hit-enrichers.js";
|
|
36
|
-
import { buildEditHint,
|
|
36
|
+
import { buildEditHint, findSourceForIndexedPath, isEditable } from "./search-source.js";
|
|
37
37
|
/**
|
|
38
38
|
* Age past which search surfaces a "run akm index" hint. Reads serve the
|
|
39
39
|
* existing index as-is (freshness is the writers' job — `indexWrittenAssets`
|
|
@@ -461,7 +461,7 @@ export function applyEntryFilters(items, opts) {
|
|
|
461
461
|
// index spans every configured source, so without this filter a narrowed
|
|
462
462
|
// --from request would still leak results from other sources.
|
|
463
463
|
const sourceFiltered = opts.restrictToSources
|
|
464
|
-
? items.filter((item) =>
|
|
464
|
+
? items.filter((item) => findSourceForIndexedPath(item.filePath, opts.sources) !== undefined)
|
|
465
465
|
: items;
|
|
466
466
|
// Scope filter: drop entries whose stored scope does not satisfy every
|
|
467
467
|
// supplied key.
|
|
@@ -623,7 +623,7 @@ function classifyVectorFailure(error) {
|
|
|
623
623
|
// ── Hit building ────────────────────────────────────────────────────────────
|
|
624
624
|
export async function buildDbHit(input) {
|
|
625
625
|
const absolutePath = path.resolve(input.path);
|
|
626
|
-
const source =
|
|
626
|
+
const source = findSourceForIndexedPath(absolutePath, input.sources);
|
|
627
627
|
const entryStashDir = source?.path ?? input.defaultStashDir;
|
|
628
628
|
const defaultBundleId = input.config?.defaultBundle ??
|
|
629
629
|
(source && path.resolve(source.path) === path.resolve(input.defaultStashDir)
|
|
@@ -185,22 +185,29 @@ export function resolveEntryContentDir(entry) {
|
|
|
185
185
|
* corrupting origin / editability / provenance decisions for the affected files.
|
|
186
186
|
*/
|
|
187
187
|
export function findSourceForPath(filePath, sources) {
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
}
|
|
198
|
-
return best;
|
|
199
|
-
};
|
|
188
|
+
// A caller's path may spell the library another way (a symlink); only then are real paths compared.
|
|
189
|
+
return (findSourceForIndexedPath(filePath, sources) ??
|
|
190
|
+
longestSource(sources, (base) => relativeToRoot(filePath, base) !== undefined));
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* {@link findSourceForPath} for a path read from the index. The index and the sources both store the library's real
|
|
194
|
+
* path, so the spelling is compared and nothing is read from disk: filtering every index row stays fast.
|
|
195
|
+
*/
|
|
196
|
+
export function findSourceForIndexedPath(filePath, sources) {
|
|
200
197
|
const resolved = path.resolve(filePath);
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
198
|
+
return longestSource(sources, (base) => resolved.startsWith(base + path.sep));
|
|
199
|
+
}
|
|
200
|
+
function longestSource(sources, within) {
|
|
201
|
+
let best;
|
|
202
|
+
let bestLen = -1;
|
|
203
|
+
for (const source of sources) {
|
|
204
|
+
const base = path.resolve(source.path);
|
|
205
|
+
if (base.length > bestLen && within(base)) {
|
|
206
|
+
best = source;
|
|
207
|
+
bestLen = base.length;
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
return best;
|
|
204
211
|
}
|
|
205
212
|
// ── Editability ─────────────────────────────────────────────────────────────
|
|
206
213
|
/**
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* The library's near-duplicate pairs, from the stored embeddings: what `assets.graph` reports so a host can list assets
|
|
6
|
+
* that say the same thing. To avoid comparing every pair, the assets are first bucketed by meaning (spherical k-means
|
|
7
|
+
* on the unit vectors) and only pairs inside a bucket are compared; the buckets are not reported. Nothing is embedded
|
|
8
|
+
* and nothing is written; the work yields to the event loop as it goes, and the result is kept while the vectors stay
|
|
9
|
+
* the same.
|
|
10
|
+
*/
|
|
11
|
+
import { createHash } from "node:crypto";
|
|
12
|
+
/** Fewest embedded assets duplicates are looked for in. */
|
|
13
|
+
const MIN_ASSETS = 8;
|
|
14
|
+
/** Most buckets, however large the library. */
|
|
15
|
+
const MAX_BUCKETS = 36;
|
|
16
|
+
/** Most vectors the bucket centres are found from; the rest are only assigned to them. */
|
|
17
|
+
const SAMPLE = 4000;
|
|
18
|
+
const ROUNDS = 16;
|
|
19
|
+
/** Cosine at or above which two assets of one bucket are reported as near-duplicates. */
|
|
20
|
+
export const DUPLICATE_COSINE = 0.96;
|
|
21
|
+
/** Most near-duplicate pairs reported, the most alike first. */
|
|
22
|
+
const MAX_PAIRS = 2000;
|
|
23
|
+
const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve));
|
|
24
|
+
/** How many buckets for `n` embedded assets: about the square root of n/2, from 2 to {@link MAX_BUCKETS}. */
|
|
25
|
+
export function bucketCount(n) {
|
|
26
|
+
return Math.max(2, Math.min(MAX_BUCKETS, Math.round(Math.sqrt(n / 2))));
|
|
27
|
+
}
|
|
28
|
+
let cache;
|
|
29
|
+
/** The vectors' content, in order: an index build that changed no embedding gives the same key. */
|
|
30
|
+
function contentKey(vectors) {
|
|
31
|
+
const hash = createHash("sha1");
|
|
32
|
+
for (const v of vectors)
|
|
33
|
+
hash.update(v ? new Uint8Array(v.buffer, v.byteOffset, v.byteLength) : NO_VECTOR);
|
|
34
|
+
return hash.digest("hex");
|
|
35
|
+
}
|
|
36
|
+
const NO_VECTOR = new Uint8Array(1);
|
|
37
|
+
/**
|
|
38
|
+
* Near-duplicate pairs among the assets: spherical k-means puts the assets into buckets (centres from an evenly spaced
|
|
39
|
+
* sample of at most 4,000, seeded deterministically, so the same vectors give the same pairs), and every pair inside a
|
|
40
|
+
* bucket whose cosine is at least {@link DUPLICATE_COSINE} is reported. `null` without at least eight embedded assets.
|
|
41
|
+
* The last result is reused while the vectors are the same, so index builds that change no embedding cost nothing here.
|
|
42
|
+
*/
|
|
43
|
+
export async function findDuplicates(vectors) {
|
|
44
|
+
const key = contentKey(vectors);
|
|
45
|
+
if (cache?.key === key)
|
|
46
|
+
return cache.result;
|
|
47
|
+
const dims = vectors.find((v) => v)?.length ?? 0;
|
|
48
|
+
const usable = [];
|
|
49
|
+
for (let i = 0; i < vectors.length; i++)
|
|
50
|
+
if (vectors[i]?.length === dims)
|
|
51
|
+
usable.push(i);
|
|
52
|
+
if (!dims || usable.length < MIN_ASSETS)
|
|
53
|
+
return null;
|
|
54
|
+
// Unit vectors, packed: cosine is then a dot product.
|
|
55
|
+
const unit = new Float32Array(usable.length * dims);
|
|
56
|
+
usable.forEach((i, row) => {
|
|
57
|
+
const v = vectors[i];
|
|
58
|
+
let norm = 0;
|
|
59
|
+
for (let d = 0; d < dims; d++)
|
|
60
|
+
norm += v[d] * v[d];
|
|
61
|
+
norm = Math.sqrt(norm) || 1;
|
|
62
|
+
for (let d = 0; d < dims; d++)
|
|
63
|
+
unit[row * dims + d] = v[d] / norm;
|
|
64
|
+
});
|
|
65
|
+
// A bit per dimension, set where the vector is above the library's mean. Vectors as alike as a duplicate differ in
|
|
66
|
+
// far fewer than a quarter of their bits (under a fifth on a 16,000-asset library), so a pair is multiplied out
|
|
67
|
+
// only when its codes are that close.
|
|
68
|
+
const words = Math.ceil(dims / 32);
|
|
69
|
+
const mean = new Float64Array(dims);
|
|
70
|
+
for (let row = 0; row < usable.length; row++)
|
|
71
|
+
for (let d = 0; d < dims; d++)
|
|
72
|
+
mean[d] += unit[row * dims + d];
|
|
73
|
+
const code = new Uint32Array(usable.length * words);
|
|
74
|
+
for (let row = 0; row < usable.length; row++)
|
|
75
|
+
for (let d = 0; d < dims; d++)
|
|
76
|
+
if (unit[row * dims + d] * usable.length > mean[d])
|
|
77
|
+
code[row * words + (d >> 5)] |= 1 << (d & 31);
|
|
78
|
+
const closeBits = Math.ceil(dims / 4);
|
|
79
|
+
const bitsApart = (x, y) => {
|
|
80
|
+
let bits = 0;
|
|
81
|
+
for (let w = 0; w < words; w++) {
|
|
82
|
+
let v = code[x * words + w] ^ code[y * words + w];
|
|
83
|
+
v -= (v >>> 1) & 0x55555555;
|
|
84
|
+
v = (v & 0x33333333) + ((v >>> 2) & 0x33333333);
|
|
85
|
+
bits += (((v + (v >>> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24;
|
|
86
|
+
}
|
|
87
|
+
return bits;
|
|
88
|
+
};
|
|
89
|
+
const dot = (row, centre, c) => {
|
|
90
|
+
let sum = 0;
|
|
91
|
+
for (let d = 0; d < dims; d++)
|
|
92
|
+
sum += unit[row * dims + d] * centre[c * dims + d];
|
|
93
|
+
return sum;
|
|
94
|
+
};
|
|
95
|
+
const k = bucketCount(usable.length);
|
|
96
|
+
const step = Math.max(1, usable.length / SAMPLE);
|
|
97
|
+
const sample = [];
|
|
98
|
+
for (let at = 0; at < usable.length && sample.length < SAMPLE; at += step)
|
|
99
|
+
sample.push(Math.floor(at));
|
|
100
|
+
// k-means++ from a fixed start, with a deterministic generator, so a library always gets the same buckets.
|
|
101
|
+
let seed = 7;
|
|
102
|
+
const random = () => {
|
|
103
|
+
seed = (seed * 1664525 + 1013904223) >>> 0;
|
|
104
|
+
return seed / 2 ** 32;
|
|
105
|
+
};
|
|
106
|
+
const centres = new Float32Array(k * dims);
|
|
107
|
+
const setCentre = (c, row) => centres.set(unit.subarray(row * dims, row * dims + dims), c * dims);
|
|
108
|
+
setCentre(0, sample[0]);
|
|
109
|
+
const far = new Float64Array(sample.length).fill(Number.POSITIVE_INFINITY);
|
|
110
|
+
for (let c = 1; c < k; c++) {
|
|
111
|
+
let total = 0;
|
|
112
|
+
for (let s = 0; s < sample.length; s++) {
|
|
113
|
+
far[s] = Math.min(far[s], Math.max(0, 1 - dot(sample[s], centres, c - 1)));
|
|
114
|
+
total += far[s] * far[s];
|
|
115
|
+
}
|
|
116
|
+
let r = random() * total;
|
|
117
|
+
let chosen = sample.length - 1;
|
|
118
|
+
for (let s = 0; s < sample.length; s++) {
|
|
119
|
+
r -= far[s] * far[s];
|
|
120
|
+
if (r <= 0) {
|
|
121
|
+
chosen = s;
|
|
122
|
+
break;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
setCentre(c, sample[chosen]);
|
|
126
|
+
}
|
|
127
|
+
const nearest = (row) => {
|
|
128
|
+
let best = 0;
|
|
129
|
+
let bestDot = Number.NEGATIVE_INFINITY;
|
|
130
|
+
for (let c = 0; c < k; c++) {
|
|
131
|
+
const value = dot(row, centres, c);
|
|
132
|
+
if (value > bestDot) {
|
|
133
|
+
bestDot = value;
|
|
134
|
+
best = c;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return best;
|
|
138
|
+
};
|
|
139
|
+
for (let round = 0; round < ROUNDS; round++) {
|
|
140
|
+
const sums = new Float64Array(k * dims);
|
|
141
|
+
for (const row of sample) {
|
|
142
|
+
const c = nearest(row);
|
|
143
|
+
for (let d = 0; d < dims; d++)
|
|
144
|
+
sums[c * dims + d] += unit[row * dims + d];
|
|
145
|
+
}
|
|
146
|
+
for (let c = 0; c < k; c++) {
|
|
147
|
+
let norm = 0;
|
|
148
|
+
for (let d = 0; d < dims; d++)
|
|
149
|
+
norm += sums[c * dims + d] ** 2;
|
|
150
|
+
norm = Math.sqrt(norm);
|
|
151
|
+
// A centre that lost every member keeps its place.
|
|
152
|
+
if (norm > 0)
|
|
153
|
+
for (let d = 0; d < dims; d++)
|
|
154
|
+
centres[c * dims + d] = sums[c * dims + d] / norm;
|
|
155
|
+
}
|
|
156
|
+
await yieldToLoop();
|
|
157
|
+
}
|
|
158
|
+
// Every asset to its nearest centre.
|
|
159
|
+
const members = Array.from({ length: k }, () => []);
|
|
160
|
+
for (let row = 0; row < usable.length; row++) {
|
|
161
|
+
members[nearest(row)].push(row);
|
|
162
|
+
if (row % 2000 === 1999)
|
|
163
|
+
await yieldToLoop();
|
|
164
|
+
}
|
|
165
|
+
// Near-duplicates: every pair inside a bucket. Pairs across buckets are not compared; two assets that alike almost
|
|
166
|
+
// always share a bucket.
|
|
167
|
+
// Only the best MAX_PAIRS are kept, so a library full of copies does not hold every pair in memory: the list is cut
|
|
168
|
+
// back whenever it grows to eight times that, to the same top pairs a full sort would keep.
|
|
169
|
+
let pairs = [];
|
|
170
|
+
let pairCount = 0;
|
|
171
|
+
const byAlike = (x, y) => y[2] - x[2] || x[0] - y[0] || x[1] - y[1];
|
|
172
|
+
for (const rows of members) {
|
|
173
|
+
for (let x = 0; x < rows.length; x++) {
|
|
174
|
+
const p = rows[x] * dims;
|
|
175
|
+
for (let y = x + 1; y < rows.length; y++) {
|
|
176
|
+
if (bitsApart(rows[x], rows[y]) > closeBits)
|
|
177
|
+
continue;
|
|
178
|
+
const q = rows[y] * dims;
|
|
179
|
+
let cosine = 0;
|
|
180
|
+
for (let d = 0; d < dims; d++)
|
|
181
|
+
cosine += unit[p + d] * unit[q + d];
|
|
182
|
+
if (cosine >= DUPLICATE_COSINE) {
|
|
183
|
+
pairs.push([usable[rows[x]], usable[rows[y]], cosine]);
|
|
184
|
+
pairCount++;
|
|
185
|
+
if (pairs.length >= MAX_PAIRS * 8)
|
|
186
|
+
pairs = pairs.sort(byAlike).slice(0, MAX_PAIRS);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
if (x % 200 === 199)
|
|
190
|
+
await yieldToLoop();
|
|
191
|
+
}
|
|
192
|
+
await yieldToLoop();
|
|
193
|
+
}
|
|
194
|
+
const kept = pairs.sort(byAlike).slice(0, MAX_PAIRS);
|
|
195
|
+
const result = {
|
|
196
|
+
threshold: DUPLICATE_COSINE,
|
|
197
|
+
a: kept.map((p) => p[0]),
|
|
198
|
+
b: kept.map((p) => p[1]),
|
|
199
|
+
cosine: kept.map((p) => Math.round(p[2] * 10000) / 10000),
|
|
200
|
+
truncated: pairCount > MAX_PAIRS,
|
|
201
|
+
};
|
|
202
|
+
cache = { key, result };
|
|
203
|
+
return result;
|
|
204
|
+
}
|
package/dist/workspace/graph.js
CHANGED
|
@@ -2,32 +2,29 @@
|
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
/** The whole library as a graph: every indexed asset, every declared link between them, and how each is used. */
|
|
5
|
-
import { createHash } from "node:crypto";
|
|
6
5
|
import fs from "node:fs";
|
|
7
6
|
import { loadConfig } from "../core/config/config.js";
|
|
8
7
|
import { ConfigError } from "../core/errors.js";
|
|
9
|
-
import { getDbPath } from "../core/paths.js";
|
|
10
8
|
import { applyEntryFilters } from "../indexer/search/db-search.js";
|
|
11
9
|
import { resolveSourceEntries } from "../indexer/search/search-source.js";
|
|
12
10
|
import { countFeedbackSignalsByRef, countUseByRef, usageRetentionSince } from "../indexer/usage/usage-events.js";
|
|
13
11
|
import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
14
12
|
import { getAllEntries } from "../storage/repositories/index-entries-repository.js";
|
|
15
13
|
import { listDeclaredLinks } from "../storage/repositories/index-links-repository.js";
|
|
16
|
-
import { getMeta } from "../storage/repositories/index-meta-repository.js";
|
|
17
14
|
import { listCurrentEmbeddings } from "../storage/repositories/index-vec-repository.js";
|
|
18
|
-
import {
|
|
15
|
+
import { findDuplicates } from "./graph-duplicates.js";
|
|
19
16
|
import { withReadonlyStateDb } from "./readers.js";
|
|
20
17
|
/** Types never drawn: protected files and session logs, as `assets.related` leaves them out. */
|
|
21
18
|
const HIDDEN_TYPES = ["secret", "env", "session"];
|
|
22
19
|
/**
|
|
23
20
|
* Every asset of the enabled bundles and the declared links between them, in column form: node `i` is `nodes.ref[i]`,
|
|
24
21
|
* and its type, bundle and belief state are indexes into `types`, `bundles` (largest first) and `beliefs` (-1: no state
|
|
25
|
-
* recorded). `shows`, `
|
|
22
|
+
* recorded). `shows`, `lastShownAt` and the feedback counts `positive` and `negative` are read live from the
|
|
26
23
|
* user-attributed usage events within retention (`use`); `modifiedAt` is the file's modification time. A link joins two
|
|
27
|
-
* nodes by index and names its kind by index into `kinds`. `
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
24
|
+
* nodes by index and names its kind by index into `kinds`. `nodes.tags[i]` indexes `tags`, the asset's own frontmatter
|
|
25
|
+
* tags. `duplicates` pairs assets whose stored embeddings are nearly the same (see {@link findDuplicates}); it is `null`
|
|
26
|
+
* when too few assets are embedded. Nothing is written. The duplicate search yields to the event loop as it goes, so a
|
|
27
|
+
* host serving other requests keeps answering them.
|
|
31
28
|
*/
|
|
32
29
|
export async function graph() {
|
|
33
30
|
try {
|
|
@@ -49,29 +46,24 @@ function emptyGraph() {
|
|
|
49
46
|
types: [],
|
|
50
47
|
bundles: [],
|
|
51
48
|
beliefs: [],
|
|
52
|
-
emptyBundles: [],
|
|
53
49
|
/** Use figures count events from `source` at or after `since`. */
|
|
54
50
|
use: null,
|
|
55
|
-
/** Declared links that are not drawn: target missing, hidden, or in a bundle that is not enabled. */
|
|
56
|
-
unresolvedLinks: 0,
|
|
57
51
|
kinds: [],
|
|
52
|
+
/** Every tag the assets carry, lower case; `nodes.tags[i]` indexes it. */
|
|
53
|
+
tags: [],
|
|
58
54
|
nodes: {
|
|
59
55
|
ref: [],
|
|
60
|
-
title: [],
|
|
61
56
|
type: [],
|
|
62
57
|
bundle: [],
|
|
63
58
|
belief: [],
|
|
64
59
|
shows: [],
|
|
65
|
-
searches: [],
|
|
66
60
|
lastShownAt: [],
|
|
67
61
|
modifiedAt: [],
|
|
68
62
|
positive: [],
|
|
69
63
|
negative: [],
|
|
70
|
-
|
|
64
|
+
tags: [],
|
|
71
65
|
},
|
|
72
66
|
links: { from: [], to: [], kind: [] },
|
|
73
|
-
meaning: null,
|
|
74
|
-
topics: [],
|
|
75
67
|
duplicates: null,
|
|
76
68
|
};
|
|
77
69
|
}
|
|
@@ -109,13 +101,13 @@ async function readGraph() {
|
|
|
109
101
|
const typeCode = code(result.types);
|
|
110
102
|
const beliefCode = code(result.beliefs);
|
|
111
103
|
const kindCode = code(result.kinds);
|
|
104
|
+
const tagCode = code(result.tags);
|
|
112
105
|
// Largest bundle first, so colour slots follow size; ties keep first-appearance order.
|
|
113
106
|
const perBundle = new Map();
|
|
114
107
|
for (const row of rows)
|
|
115
108
|
perBundle.set(row.bundleId, (perBundle.get(row.bundleId) ?? 0) + 1);
|
|
116
109
|
result.bundles.push(...[...perBundle.keys()].sort((x, y) => perBundle.get(y) - perBundle.get(x)));
|
|
117
110
|
const bundleAt = new Map(result.bundles.map((bundle, i) => [bundle, i]));
|
|
118
|
-
result.emptyBundles = sources.flatMap((source) => source.registryId && !perBundle.has(source.registryId) ? [source.registryId] : []);
|
|
119
111
|
const since = usageRetentionSince();
|
|
120
112
|
const use = withReadonlyStateDb((state) => countUseByRef(state, since), null);
|
|
121
113
|
const feedback = withReadonlyStateDb((state) => countFeedbackSignalsByRef(state, rows.map((row) => row.itemRef)), null);
|
|
@@ -123,25 +115,22 @@ async function readGraph() {
|
|
|
123
115
|
for (const row of rows) {
|
|
124
116
|
const counts = use?.get(row.itemRef);
|
|
125
117
|
result.nodes.ref.push(row.itemRef);
|
|
126
|
-
result.nodes.title.push(row.entry.toc?.find((heading) => heading.level === 1)?.text || row.entry.name);
|
|
127
118
|
result.nodes.type.push(typeCode(row.entry.type));
|
|
128
119
|
result.nodes.bundle.push(bundleAt.get(row.bundleId));
|
|
129
120
|
result.nodes.belief.push(row.entry.beliefState ? beliefCode(row.entry.beliefState) : -1);
|
|
130
121
|
result.nodes.shows.push(counts?.shows ?? 0);
|
|
131
|
-
result.nodes.searches.push(counts?.searches ?? 0);
|
|
132
122
|
result.nodes.lastShownAt.push(counts?.lastShownAt ?? null);
|
|
133
123
|
result.nodes.modifiedAt.push(modifiedAt(row.filePath));
|
|
134
124
|
const signals = feedback?.get(row.itemRef);
|
|
135
125
|
result.nodes.positive.push(signals?.pos ?? 0);
|
|
136
126
|
result.nodes.negative.push(signals?.neg ?? 0);
|
|
127
|
+
result.nodes.tags.push([...new Set((row.entry.tags ?? []).map((tag) => tag.trim().toLowerCase()))].filter(Boolean).map(tagCode));
|
|
137
128
|
}
|
|
138
129
|
for (const link of listDeclaredLinks(db)) {
|
|
139
130
|
const from = index.get(link.from);
|
|
140
131
|
if (from === undefined)
|
|
141
132
|
continue;
|
|
142
133
|
const to = link.to === null ? undefined : index.get(link.to);
|
|
143
|
-
if (to === undefined)
|
|
144
|
-
result.unresolvedLinks++;
|
|
145
134
|
if (to === undefined || from === to)
|
|
146
135
|
continue;
|
|
147
136
|
result.links.from.push(from);
|
|
@@ -149,27 +138,10 @@ async function readGraph() {
|
|
|
149
138
|
result.links.kind.push(kindCode(link.kind));
|
|
150
139
|
}
|
|
151
140
|
const vectors = listCurrentEmbeddings(db);
|
|
152
|
-
const build = `${getDbPath()}|${getMeta(db, "embeddingFingerprint") ?? ""}|${getMeta(db, "builtAt") ?? ""}`;
|
|
153
141
|
closeDatabase(db);
|
|
154
142
|
closed = true;
|
|
155
|
-
|
|
156
|
-
result.meaning = placed.embedded ? placed : null;
|
|
157
|
-
const libraryKey = createHash("sha1")
|
|
158
|
-
.update(rows.map((row) => row.id).join(","))
|
|
159
|
-
.digest("hex");
|
|
160
|
-
const found = await findTopics(rows.map((row) => vectors.get(row.id)), rows.map((row, i) => `${row.conceptId} ${result.nodes.title[i]}`),
|
|
161
|
-
// Topics are per asset: the same build read through a different set of enabled bundles is a different library.
|
|
162
|
-
`${build}|${libraryKey}`);
|
|
163
|
-
result.topics = found.topics;
|
|
164
|
-
result.nodes.topic = found.topic;
|
|
165
|
-
result.duplicates = found.topics.length ? found.duplicates : null;
|
|
143
|
+
result.duplicates = await findDuplicates(rows.map((row) => vectors.get(row.id)));
|
|
166
144
|
const noted = [`${rows.length} assets and ${result.links.from.length} links of the enabled bundles.`];
|
|
167
|
-
if (result.unresolvedLinks)
|
|
168
|
-
noted.push(`${result.unresolvedLinks} declared links are not drawn (target missing, hidden or in a bundle that is not enabled).`);
|
|
169
|
-
if (result.emptyBundles.length)
|
|
170
|
-
noted.push(`No assets in: ${result.emptyBundles.join(", ")}.`);
|
|
171
|
-
const embedded = rows.filter((row) => vectors.has(row.id)).length;
|
|
172
|
-
noted.push(embeddingNote(embedded, rows.length));
|
|
173
145
|
if (!use)
|
|
174
146
|
noted.push("Use counts are zero: no usage history is available.");
|
|
175
147
|
return {
|
|
@@ -198,97 +170,3 @@ function modifiedAt(file) {
|
|
|
198
170
|
return null;
|
|
199
171
|
}
|
|
200
172
|
}
|
|
201
|
-
/** Fewest vectors the principal components can be found from. */
|
|
202
|
-
const MIN_VECTORS = 4;
|
|
203
|
-
/** The note's sentence on how many assets have a position by meaning. */
|
|
204
|
-
export function embeddingNote(embedded, total) {
|
|
205
|
-
if (embedded === 0)
|
|
206
|
-
return "No assets are embedded, so there is no arrangement by meaning.";
|
|
207
|
-
if (embedded < MIN_VECTORS)
|
|
208
|
-
return `Only ${embedded} of ${total} assets are embedded; at least ${MIN_VECTORS} are needed to arrange by meaning.`;
|
|
209
|
-
if (embedded === total)
|
|
210
|
-
return `All ${total} assets are embedded.`;
|
|
211
|
-
return `${embedded} of ${total} assets are embedded; the rest have no position by meaning.`;
|
|
212
|
-
}
|
|
213
|
-
/** Most vectors the principal components are found from; the rest are only projected onto them. */
|
|
214
|
-
const SAMPLE = 4000;
|
|
215
|
-
/** Vectors projected between yields to the event loop. */
|
|
216
|
-
const CHUNK = 2000;
|
|
217
|
-
const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve));
|
|
218
|
-
/** The axes of the last projection, reused while the index's embedding model and full build stay the same. */
|
|
219
|
-
let axesCache;
|
|
220
|
-
/**
|
|
221
|
-
* Three coordinates per vector: its projection on the three principal components of the vectors, so vectors that are
|
|
222
|
-
* alike land near each other. The components come from an evenly spaced sample of at most 4,000 vectors (power
|
|
223
|
-
* iteration with deflation from a fixed start, so the same vectors give the same layout), scaled so the first
|
|
224
|
-
* component's standard deviation is 400. With a `key` (the index's embedding model and build), the components are
|
|
225
|
-
* kept and reused until the key changes: an asset embedded since is projected onto them. A missing vector gives three
|
|
226
|
-
* nulls. Work is done in slices with a yield to the event loop between them.
|
|
227
|
-
*/
|
|
228
|
-
export async function projectEmbeddings(vectors, key) {
|
|
229
|
-
const present = vectors.filter((vector) => vector !== undefined);
|
|
230
|
-
const dims = present[0]?.length ?? 0;
|
|
231
|
-
const usable = present.filter((vector) => vector.length === dims);
|
|
232
|
-
if (!dims || usable.length < MIN_VECTORS)
|
|
233
|
-
return { embedded: 0, xyz: vectors.flatMap(() => [null, null, null]) };
|
|
234
|
-
let axes = key !== undefined && axesCache?.key === key && axesCache.dims === dims ? axesCache : undefined;
|
|
235
|
-
if (!axes) {
|
|
236
|
-
const step = Math.max(1, usable.length / SAMPLE);
|
|
237
|
-
const sample = [];
|
|
238
|
-
for (let at = 0; at < usable.length && sample.length < SAMPLE; at += step)
|
|
239
|
-
sample.push(usable[Math.floor(at)]);
|
|
240
|
-
const mean = new Float64Array(dims);
|
|
241
|
-
for (const vector of sample)
|
|
242
|
-
for (let d = 0; d < dims; d++)
|
|
243
|
-
mean[d] += vector[d] / sample.length;
|
|
244
|
-
const components = [];
|
|
245
|
-
for (let c = 0; c < 3; c++) {
|
|
246
|
-
let axis = new Float64Array(dims).map((_, d) => 1 + ((d * 7 + c * 13) % 5));
|
|
247
|
-
for (let round = 0; round < 16; round++) {
|
|
248
|
-
const next = new Float64Array(dims);
|
|
249
|
-
for (const vector of sample) {
|
|
250
|
-
const weight = projectOn(vector, mean, axis);
|
|
251
|
-
for (let d = 0; d < dims; d++)
|
|
252
|
-
next[d] += weight * (vector[d] - mean[d]);
|
|
253
|
-
}
|
|
254
|
-
// Keep the axis orthogonal to the components already found.
|
|
255
|
-
for (const found of components) {
|
|
256
|
-
let dot = 0;
|
|
257
|
-
for (let d = 0; d < dims; d++)
|
|
258
|
-
dot += next[d] * found[d];
|
|
259
|
-
for (let d = 0; d < dims; d++)
|
|
260
|
-
next[d] -= dot * found[d];
|
|
261
|
-
}
|
|
262
|
-
let norm = 0;
|
|
263
|
-
for (let d = 0; d < dims; d++)
|
|
264
|
-
norm += next[d] * next[d];
|
|
265
|
-
norm = Math.sqrt(norm) || 1;
|
|
266
|
-
axis = next.map((value) => value / norm);
|
|
267
|
-
await yieldToLoop();
|
|
268
|
-
}
|
|
269
|
-
components.push(axis);
|
|
270
|
-
}
|
|
271
|
-
const spread = Math.sqrt(sample.reduce((sum, vector) => sum + projectOn(vector, mean, components[0]) ** 2, 0) / sample.length);
|
|
272
|
-
axes = { key: key ?? "", dims, mean, components, scale: spread > 0 ? 400 / spread : 1 };
|
|
273
|
-
if (key !== undefined)
|
|
274
|
-
axesCache = axes;
|
|
275
|
-
}
|
|
276
|
-
const xyz = [];
|
|
277
|
-
for (let at = 0; at < vectors.length; at += CHUNK) {
|
|
278
|
-
for (const vector of vectors.slice(at, at + CHUNK)) {
|
|
279
|
-
if (vector && vector.length === dims)
|
|
280
|
-
for (const axis of axes.components)
|
|
281
|
-
xyz.push(Math.round(projectOn(vector, axes.mean, axis) * axes.scale * 10) / 10);
|
|
282
|
-
else
|
|
283
|
-
xyz.push(null, null, null);
|
|
284
|
-
}
|
|
285
|
-
await yieldToLoop();
|
|
286
|
-
}
|
|
287
|
-
return { embedded: usable.length, xyz };
|
|
288
|
-
}
|
|
289
|
-
function projectOn(vector, mean, axis) {
|
|
290
|
-
let sum = 0;
|
|
291
|
-
for (let d = 0; d < axis.length; d++)
|
|
292
|
-
sum += (vector[d] - mean[d]) * axis[d];
|
|
293
|
-
return sum;
|
|
294
|
-
}
|
|
@@ -72,7 +72,6 @@ Vector search is an exact scan of `embeddings` (`src/storage/repositories/index-
|
|
|
72
72
|
#### `index_meta`
|
|
73
73
|
|
|
74
74
|
`key` TEXT PRIMARY KEY, `value` TEXT NOT NULL. Keys: `version`, `builtAt`,
|
|
75
|
-
`stashDir`, `stashDirs`, `sourceOwners`, `last_utility_computed_at`, `embeddingFingerprint` (the model the
|
|
76
75
|
vectors were made under), `hasEmbeddings` (`"0"` or `"1"`), `semanticSearch` (what the
|
|
77
76
|
last `akm index` found about semantic search; `akm doctor` reads it), `vacuumPending`,
|
|
78
77
|
and `ftsTotalsStale` (set when a delete removed a row from `entries_fts`;
|
|
@@ -195,49 +195,36 @@ them, in column form, so 30,000 assets stay a few megabytes:
|
|
|
195
195
|
"types": ["knowledge", "memory"],
|
|
196
196
|
"bundles": ["team-wiki"],
|
|
197
197
|
"beliefs": ["superseded"],
|
|
198
|
-
"emptyBundles": [],
|
|
199
198
|
"use": { "since": "2026-08-15T08:00:00.000Z", "source": "user" },
|
|
200
|
-
"unresolvedLinks": 2,
|
|
201
199
|
"kinds": ["xref", "links_to"],
|
|
202
|
-
"
|
|
203
|
-
|
|
204
|
-
"
|
|
200
|
+
"tags": ["deploy"],
|
|
201
|
+
"nodes": { "ref": ["team-wiki//knowledge/a"], "type": [0], "bundle": [0], "belief": [-1], "shows": [12],
|
|
202
|
+
"lastShownAt": ["2026-11-02T09:14:03Z"], "modifiedAt": ["2026-03-30T17:02:11.000Z"],
|
|
203
|
+
"positive": [2], "negative": [0], "tags": [[0]] },
|
|
205
204
|
"links": { "from": [0], "to": [3], "kind": [1] },
|
|
206
|
-
"meaning": { "embedded": 29840, "xyz": [12.5, -40.1, 3.2] },
|
|
207
|
-
"topics": [{ "label": "deploy · canary", "terms": ["deploy", "canary", "rollout"], "size": 812,
|
|
208
|
-
"related": [{ "topic": 7, "cosine": 0.81 }] }],
|
|
209
205
|
"duplicates": { "threshold": 0.96, "a": [0], "b": [9], "cosine": [0.9871], "truncated": false }
|
|
210
206
|
}
|
|
211
207
|
```
|
|
212
208
|
|
|
213
209
|
Node `i` is index `i` of every `nodes` array; `type`, `bundle`, `belief` and `kind` index `types`, `bundles`, `beliefs`
|
|
214
|
-
and `kinds
|
|
215
|
-
the
|
|
216
|
-
the asset that declares it to its target;
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
embeddings
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
largest first, the centres found from an evenly spaced sample of at most 4,000 vectors from a fixed start, so the same
|
|
233
|
-
index gives the same topics. A topic's `terms` are the words of its assets' paths and titles most particular to it
|
|
234
|
-
(frequent in the topic, rare in the library), and its `label` the first two; `related` names the three topics whose
|
|
235
|
-
centres are nearest, with the cosine between them. `nodes.topic` indexes `topics`, `-1` for an asset with no embedding.
|
|
236
|
-
`duplicates` lists pairs of assets of one topic whose vectors have a cosine of at least `threshold` (0.96), most alike
|
|
237
|
-
first, at most 2,000 (`truncated` says there were more); pairs across topics are not compared. Both are `[]` and `null`
|
|
238
|
-
without at least eight embedded assets. They are found once per index build and kept, like the projection: the first
|
|
239
|
-
read of a 20,000-asset index takes several seconds. `available` is `false`, with a `note`,
|
|
240
|
-
when there is no index.
|
|
210
|
+
and `kinds`, and each entry of `nodes.tags[i]` indexes `tags`, the asset's own frontmatter tags (lower case, once each).
|
|
211
|
+
`bundles` runs from the bundle with the most assets to the one with the fewest. `belief` is the asset's recorded belief
|
|
212
|
+
state, `-1` when it has none. A link runs from the asset that declares it to its target; links whose target is missing,
|
|
213
|
+
of a type that is not returned, or in a bundle that is not enabled are left out. `shows` is counted from the usage
|
|
214
|
+
events when the operation is called: events of source `user` (people and their agent sessions, not scheduled tasks or
|
|
215
|
+
Improve) at or after `use.since`, the start of the usage retention window. It is zero, and the note says so, when no
|
|
216
|
+
usage history is available. `lastShownAt` is the latest of those shows (UTC), `null` when there is none in the window.
|
|
217
|
+
`positive` and `negative` count feedback from people inside the retention window, as search ranks by it (a negative
|
|
218
|
+
answered by an accepted fix does not count). `modifiedAt` is the file's modification time, `null` when it cannot be
|
|
219
|
+
read; in a Git checkout it is when the file was last written there, not its last commit.
|
|
220
|
+
|
|
221
|
+
`duplicates` lists pairs of assets (indexes into `nodes`) whose stored embeddings have a cosine of at least `threshold`
|
|
222
|
+
(0.96), most alike first, at most 2,000 (`truncated` says there were more). To avoid comparing every pair, the assets
|
|
223
|
+
are first grouped by meaning (spherical k-means on their vectors, from a fixed start, so the same index gives the same
|
|
224
|
+
pairs) and only pairs inside a group are compared. It is `null` without at least eight embedded assets. It is kept
|
|
225
|
+
while the embeddings are the same, so an index build that changes none costs nothing; after one that does, the first
|
|
226
|
+
read of a 16,000-asset library takes about five seconds. `available` is `false`,
|
|
227
|
+
with a `note`, when there is no index.
|
|
241
228
|
|
|
242
229
|
## `bundles.list`
|
|
243
230
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "akm-cli",
|
|
3
|
-
"version": "0.10.
|
|
3
|
+
"version": "0.10.261126-beta.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cursor.",
|
|
6
6
|
"keywords": [
|
|
@@ -1,249 +0,0 @@
|
|
|
1
|
-
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
-
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
-
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
/**
|
|
5
|
-
* The library's topics and near-duplicates, from the stored embeddings: what `assets.graph` reports so a host can say
|
|
6
|
-
* what the library holds, where it overlaps, and which assets say the same thing. Nothing is embedded and nothing is
|
|
7
|
-
* written; the work yields to the event loop as it goes and is kept for one index build.
|
|
8
|
-
*/
|
|
9
|
-
/** Fewest embedded assets topics are found for. */
|
|
10
|
-
const MIN_ASSETS = 8;
|
|
11
|
-
/** Most topics, however large the library. */
|
|
12
|
-
const MAX_TOPICS = 36;
|
|
13
|
-
/** Most vectors the topic centres are found from; the rest are only assigned to them. */
|
|
14
|
-
const SAMPLE = 4000;
|
|
15
|
-
const ROUNDS = 16;
|
|
16
|
-
/** Cosine at or above which two assets of one topic are reported as near-duplicates. */
|
|
17
|
-
export const DUPLICATE_COSINE = 0.96;
|
|
18
|
-
/** Most near-duplicate pairs reported, the most alike first. */
|
|
19
|
-
const MAX_PAIRS = 2000;
|
|
20
|
-
/** Topics each topic names as nearest. */
|
|
21
|
-
const RELATED_TOPICS = 3;
|
|
22
|
-
const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve));
|
|
23
|
-
/** How many topics for `n` embedded assets: about the square root of n/2, from 2 to {@link MAX_TOPICS}. */
|
|
24
|
-
export function topicCount(n) {
|
|
25
|
-
return Math.max(2, Math.min(MAX_TOPICS, Math.round(Math.sqrt(n / 2))));
|
|
26
|
-
}
|
|
27
|
-
let cache;
|
|
28
|
-
/**
|
|
29
|
-
* Groups the assets by meaning (spherical k-means on the unit vectors: centres from an evenly spaced sample of at most
|
|
30
|
-
* 4,000, seeded deterministically, so the same vectors give the same topics), names each topic by the words of its
|
|
31
|
-
* assets' paths and titles that are most particular to it, and finds near-duplicate pairs inside each topic. `names[i]`
|
|
32
|
-
* is asset i's path and title. With a `key` (the index's build), the result is reused until the key changes.
|
|
33
|
-
*/
|
|
34
|
-
export async function findTopics(vectors, names, key) {
|
|
35
|
-
if (key !== undefined && cache?.key === key && cache.result.topic.length === vectors.length)
|
|
36
|
-
return cache.result;
|
|
37
|
-
const empty = {
|
|
38
|
-
topics: [],
|
|
39
|
-
topic: vectors.map(() => -1),
|
|
40
|
-
duplicates: { threshold: DUPLICATE_COSINE, a: [], b: [], cosine: [], truncated: false },
|
|
41
|
-
};
|
|
42
|
-
const dims = vectors.find((v) => v)?.length ?? 0;
|
|
43
|
-
const usable = [];
|
|
44
|
-
for (let i = 0; i < vectors.length; i++)
|
|
45
|
-
if (vectors[i]?.length === dims)
|
|
46
|
-
usable.push(i);
|
|
47
|
-
if (!dims || usable.length < MIN_ASSETS)
|
|
48
|
-
return empty;
|
|
49
|
-
// Unit vectors, packed: cosine is then a dot product.
|
|
50
|
-
const unit = new Float32Array(usable.length * dims);
|
|
51
|
-
usable.forEach((i, row) => {
|
|
52
|
-
const v = vectors[i];
|
|
53
|
-
let norm = 0;
|
|
54
|
-
for (let d = 0; d < dims; d++)
|
|
55
|
-
norm += v[d] * v[d];
|
|
56
|
-
norm = Math.sqrt(norm) || 1;
|
|
57
|
-
for (let d = 0; d < dims; d++)
|
|
58
|
-
unit[row * dims + d] = v[d] / norm;
|
|
59
|
-
});
|
|
60
|
-
const dot = (row, centre, c) => {
|
|
61
|
-
let sum = 0;
|
|
62
|
-
for (let d = 0; d < dims; d++)
|
|
63
|
-
sum += unit[row * dims + d] * centre[c * dims + d];
|
|
64
|
-
return sum;
|
|
65
|
-
};
|
|
66
|
-
const k = topicCount(usable.length);
|
|
67
|
-
const step = Math.max(1, usable.length / SAMPLE);
|
|
68
|
-
const sample = [];
|
|
69
|
-
for (let at = 0; at < usable.length && sample.length < SAMPLE; at += step)
|
|
70
|
-
sample.push(Math.floor(at));
|
|
71
|
-
// k-means++ from a fixed start, with a deterministic generator, so a library always gets the same topics.
|
|
72
|
-
let seed = 7;
|
|
73
|
-
const random = () => {
|
|
74
|
-
seed = (seed * 1664525 + 1013904223) >>> 0;
|
|
75
|
-
return seed / 2 ** 32;
|
|
76
|
-
};
|
|
77
|
-
const centres = new Float32Array(k * dims);
|
|
78
|
-
const setCentre = (c, row) => centres.set(unit.subarray(row * dims, row * dims + dims), c * dims);
|
|
79
|
-
setCentre(0, sample[0]);
|
|
80
|
-
const far = new Float64Array(sample.length).fill(Number.POSITIVE_INFINITY);
|
|
81
|
-
for (let c = 1; c < k; c++) {
|
|
82
|
-
let total = 0;
|
|
83
|
-
for (let s = 0; s < sample.length; s++) {
|
|
84
|
-
far[s] = Math.min(far[s], Math.max(0, 1 - dot(sample[s], centres, c - 1)));
|
|
85
|
-
total += far[s] * far[s];
|
|
86
|
-
}
|
|
87
|
-
let r = random() * total;
|
|
88
|
-
let chosen = sample.length - 1;
|
|
89
|
-
for (let s = 0; s < sample.length; s++) {
|
|
90
|
-
r -= far[s] * far[s];
|
|
91
|
-
if (r <= 0) {
|
|
92
|
-
chosen = s;
|
|
93
|
-
break;
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
setCentre(c, sample[chosen]);
|
|
97
|
-
}
|
|
98
|
-
const nearest = (row) => {
|
|
99
|
-
let best = 0;
|
|
100
|
-
let bestDot = Number.NEGATIVE_INFINITY;
|
|
101
|
-
for (let c = 0; c < k; c++) {
|
|
102
|
-
const value = dot(row, centres, c);
|
|
103
|
-
if (value > bestDot) {
|
|
104
|
-
bestDot = value;
|
|
105
|
-
best = c;
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
return best;
|
|
109
|
-
};
|
|
110
|
-
for (let round = 0; round < ROUNDS; round++) {
|
|
111
|
-
const sums = new Float64Array(k * dims);
|
|
112
|
-
for (const row of sample) {
|
|
113
|
-
const c = nearest(row);
|
|
114
|
-
for (let d = 0; d < dims; d++)
|
|
115
|
-
sums[c * dims + d] += unit[row * dims + d];
|
|
116
|
-
}
|
|
117
|
-
for (let c = 0; c < k; c++) {
|
|
118
|
-
let norm = 0;
|
|
119
|
-
for (let d = 0; d < dims; d++)
|
|
120
|
-
norm += sums[c * dims + d] ** 2;
|
|
121
|
-
norm = Math.sqrt(norm);
|
|
122
|
-
// A centre that lost every member keeps its place.
|
|
123
|
-
if (norm > 0)
|
|
124
|
-
for (let d = 0; d < dims; d++)
|
|
125
|
-
centres[c * dims + d] = sums[c * dims + d] / norm;
|
|
126
|
-
}
|
|
127
|
-
await yieldToLoop();
|
|
128
|
-
}
|
|
129
|
-
// Every asset to its nearest centre; topics are renumbered largest first, and empty ones dropped.
|
|
130
|
-
const raw = new Int32Array(usable.length);
|
|
131
|
-
for (let row = 0; row < usable.length; row++) {
|
|
132
|
-
raw[row] = nearest(row);
|
|
133
|
-
if (row % 2000 === 1999)
|
|
134
|
-
await yieldToLoop();
|
|
135
|
-
}
|
|
136
|
-
const sizes = new Array(k).fill(0);
|
|
137
|
-
for (const c of raw)
|
|
138
|
-
sizes[c]++;
|
|
139
|
-
const order = sizes
|
|
140
|
-
.map((size, c) => ({ size, c }))
|
|
141
|
-
.filter((t) => t.size > 0)
|
|
142
|
-
.sort((x, y) => y.size - x.size || x.c - y.c);
|
|
143
|
-
const renumber = new Map(order.map((t, n) => [t.c, n]));
|
|
144
|
-
const topic = vectors.map(() => -1);
|
|
145
|
-
usable.forEach((i, row) => {
|
|
146
|
-
topic[i] = renumber.get(raw[row]);
|
|
147
|
-
});
|
|
148
|
-
const members = order.map(() => []);
|
|
149
|
-
usable.forEach((i, row) => {
|
|
150
|
-
members[topic[i]].push(row);
|
|
151
|
-
});
|
|
152
|
-
const centreOf = (n) => order[n].c;
|
|
153
|
-
const topics = order.map((t, n) => ({
|
|
154
|
-
...nameTopic(members[n].map((row) => names[usable[row]] ?? ""), names),
|
|
155
|
-
size: t.size,
|
|
156
|
-
related: order
|
|
157
|
-
.map((_, m) => {
|
|
158
|
-
let cosine = 0;
|
|
159
|
-
for (let d = 0; d < dims; d++)
|
|
160
|
-
cosine += centres[centreOf(n) * dims + d] * centres[centreOf(m) * dims + d];
|
|
161
|
-
return { topic: m, cosine: Math.round(cosine * 1000) / 1000 };
|
|
162
|
-
})
|
|
163
|
-
.filter((r) => r.topic !== n)
|
|
164
|
-
.sort((x, y) => y.cosine - x.cosine)
|
|
165
|
-
.slice(0, RELATED_TOPICS),
|
|
166
|
-
}));
|
|
167
|
-
// Near-duplicates: every pair inside a topic. Pairs across topics are not compared; two assets that alike almost
|
|
168
|
-
// always share a topic.
|
|
169
|
-
// Only the best MAX_PAIRS are kept, so a library full of copies does not hold every pair in memory: the list is cut
|
|
170
|
-
// back whenever it grows to eight times that, to the same top pairs a full sort would keep.
|
|
171
|
-
let pairs = [];
|
|
172
|
-
let pairCount = 0;
|
|
173
|
-
const byAlike = (x, y) => y[2] - x[2] || x[0] - y[0] || x[1] - y[1];
|
|
174
|
-
for (const rows of members) {
|
|
175
|
-
for (let x = 0; x < rows.length; x++) {
|
|
176
|
-
const p = rows[x] * dims;
|
|
177
|
-
for (let y = x + 1; y < rows.length; y++) {
|
|
178
|
-
const q = rows[y] * dims;
|
|
179
|
-
let cosine = 0;
|
|
180
|
-
for (let d = 0; d < dims; d++)
|
|
181
|
-
cosine += unit[p + d] * unit[q + d];
|
|
182
|
-
if (cosine >= DUPLICATE_COSINE) {
|
|
183
|
-
pairs.push([usable[rows[x]], usable[rows[y]], cosine]);
|
|
184
|
-
pairCount++;
|
|
185
|
-
if (pairs.length >= MAX_PAIRS * 8)
|
|
186
|
-
pairs = pairs.sort(byAlike).slice(0, MAX_PAIRS);
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
if (x % 200 === 199)
|
|
190
|
-
await yieldToLoop();
|
|
191
|
-
}
|
|
192
|
-
await yieldToLoop();
|
|
193
|
-
}
|
|
194
|
-
const kept = pairs.sort(byAlike).slice(0, MAX_PAIRS);
|
|
195
|
-
const result = {
|
|
196
|
-
topics,
|
|
197
|
-
topic,
|
|
198
|
-
duplicates: {
|
|
199
|
-
threshold: DUPLICATE_COSINE,
|
|
200
|
-
a: kept.map((p) => p[0]),
|
|
201
|
-
b: kept.map((p) => p[1]),
|
|
202
|
-
cosine: kept.map((p) => Math.round(p[2] * 10000) / 10000),
|
|
203
|
-
truncated: pairCount > MAX_PAIRS,
|
|
204
|
-
},
|
|
205
|
-
};
|
|
206
|
-
if (key !== undefined)
|
|
207
|
-
cache = { key, result };
|
|
208
|
-
return result;
|
|
209
|
-
}
|
|
210
|
-
const STOP = new Set("the and for with from into this that what when how why are was not use using about your you our its md readme index skill skills knowledge memory memories command commands agent agents script scripts workflow workflows fact facts lesson lessons references reference assets asset src lib test tests docs".split(" "));
|
|
211
|
-
/** The words of a path or title worth naming a topic by: lower case, three letters or more, no digits, no stop words. */
|
|
212
|
-
export function words(text) {
|
|
213
|
-
return text
|
|
214
|
-
.toLowerCase()
|
|
215
|
-
.split(/[^a-z0-9]+/)
|
|
216
|
-
.filter((w) => w.length >= 3 && !/\d/.test(w) && !STOP.has(w));
|
|
217
|
-
}
|
|
218
|
-
let documentFrequency;
|
|
219
|
-
/**
|
|
220
|
-
* A topic's distinctive words: those most frequent among its assets relative to how many assets of the whole library
|
|
221
|
-
* use them (each asset counts a word once). The label is the top two.
|
|
222
|
-
*/
|
|
223
|
-
export function nameTopic(memberNames, allNames) {
|
|
224
|
-
if (documentFrequency?.names !== allNames) {
|
|
225
|
-
const counts = new Map();
|
|
226
|
-
for (const name of allNames)
|
|
227
|
-
for (const w of new Set(words(name)))
|
|
228
|
-
counts.set(w, (counts.get(w) ?? 0) + 1);
|
|
229
|
-
documentFrequency = { names: allNames, counts };
|
|
230
|
-
}
|
|
231
|
-
const inTopic = new Map();
|
|
232
|
-
for (const name of memberNames)
|
|
233
|
-
for (const w of new Set(words(name)))
|
|
234
|
-
inTopic.set(w, (inTopic.get(w) ?? 0) + 1);
|
|
235
|
-
const total = allNames.length;
|
|
236
|
-
const terms = [...inTopic]
|
|
237
|
-
// A word one asset uses names nothing.
|
|
238
|
-
.filter(([, n]) => n >= Math.min(2, memberNames.length))
|
|
239
|
-
.map(([w, n]) => ({
|
|
240
|
-
w,
|
|
241
|
-
score: (n / memberNames.length) * Math.log(total / (documentFrequency.counts.get(w) ?? 1)),
|
|
242
|
-
}))
|
|
243
|
-
// A word every asset of the library has says nothing about this topic.
|
|
244
|
-
.filter((t) => t.score > 0)
|
|
245
|
-
.sort((x, y) => y.score - x.score || (x.w < y.w ? -1 : 1))
|
|
246
|
-
.slice(0, 5)
|
|
247
|
-
.map((t) => t.w);
|
|
248
|
-
return { label: terms.slice(0, 2).join(" · ") || "unnamed", terms };
|
|
249
|
-
}
|