akm-cli 0.10.261125 → 0.10.261126

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -6,6 +6,20 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.10.261126] - 2026-11-15
10
+
11
+ ### Changed
12
+
13
+ - AKM Desktop's library map is a mind map: the library, its asset types, and the tags each type's assets carry most, every label drawn, with a bar under each tag for its figure in the view. Contents, Use and Problems each draw the tags their question is about (by size, by reads, by problems), for all bundles or the one picked. Choosing a type or tag draws lines to every group its assets link to or nearly duplicate, and the column beside the map answers the view's question for it: what it is made of and its most linked assets; what is read, never read and searched for without a result; what is outdated, duplicated or disliked; and which other bundles repeat it. Every listed asset can show its closest assets by content, with a Compare link. It replaces the topic circles of 0.10.261125, whose topic names were word pairs from paths, and drops the Relations view. The map and the replay no longer need WebGL.
14
+ - AKM Desktop's activity replay is a timeline: events per kind stacked over the period with a playhead you click or drag, the mind map of the assets the period touched counting the events on each type and tag, and a log of the latest events. A curate counts as a search, and its results as that search's hits. It replaces the point cloud with flying pulses and the per-agent lanes.
15
+ - `akm-cli/workspace`: `assets.graph()` returns each asset's own tags (`nodes.tags`, indexes into a new `tags` list, lower case); it no longer returns `meaning`, `topics`, `nodes.topic`, `nodes.title`, `nodes.searches`, `emptyBundles` or `unresolvedLinks`. `duplicates` stays.
16
+
17
+ ### Fixed
18
+
19
+ - AKM Desktop's library map no longer shows "This view needs WebGL" after the pointer leaves an item, and the map and replay pages scroll as one page instead of a side panel scrolling inside a scrolling page.
20
+ - Matching indexed files to their bundle (curate, related, `search --from`, AKM Desktop's library map) no longer resolves real paths on disk for files outside the enabled bundles: the index and the sources both store real paths, so the spelling is compared. Since 0.10.261125 this added about 3.6 seconds to each library map read on a library of 20,000 indexed files.
21
+ - Each index build no longer applies a full day of decay to utility scores: a score now decays by the time since it was last written, so many builds in a day decay it about one day in total, and a new score starts at its select or feedback rate (itlackey/akm#1659).
22
+
9
23
  ## [0.10.261125] - 2026-11-14
10
24
 
11
25
  ### Added
@@ -1383,7 +1383,7 @@ function readLookupEntry(db, id, fallbackConceptId, sourceRoot) {
1383
1383
  * Recompute utility scores based on usage_events data: every entry by default,
1384
1384
  * or just `onlyEntryId` (what `akm feedback` passes so the value it stores is
1385
1385
  * the one `akm index` would compute, not a second formula). A one-entry call
1386
- * leaves the retention purge and the `last_utility_computed_at` watermark alone.
1386
+ * leaves the retention purge alone.
1387
1387
  *
1388
1388
  * For each entry:
1389
1389
  * - Count search appearances (event_type = 'search')
@@ -1405,18 +1405,6 @@ export function recomputeUtilityScores(db, stateDb, onlyEntryId) {
1405
1405
  // (Chunk-8 WI-8.3); its table is created by state migration 020.
1406
1406
  if (!scoped)
1407
1407
  purgeOldUsageEvents(stateDb, USAGE_EVENT_RETENTION_DAYS);
1408
- // Decay is raised to the days since the last run, floored at one day. Every
1409
- // run therefore applies at least one full round of EMA decay (so runs less
1410
- // than a day apart each decay a full round), and a longer gap applies
1411
- // proportionally more rounds. The first run counts as one day.
1412
- const lastComputedAt = getMeta(db, "last_utility_computed_at");
1413
- let elapsedDays = 1; // default for first run
1414
- if (lastComputedAt) {
1415
- const ms = Date.now() - new Date(lastComputedAt).getTime();
1416
- elapsedDays = Math.max(1, ms / (1000 * 60 * 60 * 24));
1417
- }
1418
- const emaDecay = EMA_DECAY ** elapsedDays;
1419
- const emaNew = 1 - emaDecay; // complement so weights still sum to 1
1420
1408
  // Aggregate explicit user demand per `entry_ref` from state.db's usage_events
1421
1409
  // (the stable identity every event carries; search and curate telemetry hold
1422
1410
  // only the ref, not the index row id), then map each ref to its CURRENT entry
@@ -1461,11 +1449,12 @@ export function recomputeUtilityScores(db, stateDb, onlyEntryId) {
1461
1449
  // Batch-load existing utility scores
1462
1450
  const existingScores = new Map();
1463
1451
  const scoreRows = db
1464
- .prepare(`SELECT entry_id, utility, last_used_at FROM utility_scores${scoped ? " WHERE entry_id = ?" : ""}`)
1452
+ .prepare(`SELECT entry_id, utility, updated_at FROM utility_scores${scoped ? " WHERE entry_id = ?" : ""}`)
1465
1453
  .all(...(scoped ? [onlyEntryId] : []));
1466
1454
  for (const row of scoreRows) {
1467
- existingScores.set(row.entry_id, { utility: row.utility, lastUsedAt: row.last_used_at ?? undefined });
1455
+ existingScores.set(row.entry_id, { utility: row.utility, updatedAt: row.updated_at });
1468
1456
  }
1457
+ const now = Date.now();
1469
1458
  const entryIds = new Set([...existingScores.keys(), ...usageByEntry.keys()]);
1470
1459
  // One transaction for every row: per-row autocommits cost one WAL sync each (~43 s of a 45 s finalize on a
1471
1460
  // 20,900-entry library). The reads above stay outside, so state.db is held no longer than before.
@@ -1484,8 +1473,15 @@ export function recomputeUtilityScores(db, stateDb, onlyEntryId) {
1484
1473
  const feedbackRate = feedbackTotal > 0 ? Math.max(0, row.positive_feedback_count - row.negative_feedback_count) / feedbackTotal : 0;
1485
1474
  const effectiveRate = Math.max(selectRate, feedbackRate);
1486
1475
  const existing = existingScores.get(row.entry_id);
1487
- const prevUtility = existing?.utility ?? 0;
1488
- const utility = prevUtility * emaDecay + effectiveRate * emaNew;
1476
+ // Decay by the time since this row was last written, so any number of builds
1477
+ // in a day apply about one day of decay in total. A new row starts at its rate.
1478
+ let utility = effectiveRate;
1479
+ if (existing) {
1480
+ const updatedMs = existing.updatedAt ? Date.parse(`${existing.updatedAt.replace(" ", "T")}Z`) : Number.NaN;
1481
+ const elapsedDays = Number.isFinite(updatedMs) ? Math.max(0, (now - updatedMs) / 86_400_000) : 1;
1482
+ const decay = EMA_DECAY ** elapsedDays;
1483
+ utility = existing.utility * decay + effectiveRate * (1 - decay);
1484
+ }
1489
1485
  // `utility_scores.last_used_at` is consumed by salience as the timestamp of
1490
1486
  // the most-recent retrieval. Preserve that meaning by carrying the event's
1491
1487
  // timestamp through verbatim. The former `effectiveRate > 0.5 ? now : ...`
@@ -1505,7 +1501,5 @@ export function recomputeUtilityScores(db, stateDb, onlyEntryId) {
1505
1501
  lastUsedAt,
1506
1502
  });
1507
1503
  }
1508
- if (!scoped)
1509
- setMeta(db, "last_utility_computed_at", new Date().toISOString());
1510
1504
  })();
1511
1505
  }
@@ -33,7 +33,7 @@ import { ftsQueryTokens, parseRefPrefixQuery } from "./fts-query.js";
33
33
  import { applyFeedback, applyPrefer, applyRecency, reciprocalRankFusion, } from "./ranking.js";
34
34
  import { attachSearchHitAttribution } from "./search-attribution.js";
35
35
  import { enrichSearchHit } from "./search-hit-enrichers.js";
36
- import { buildEditHint, findSourceForPath, isEditable } from "./search-source.js";
36
+ import { buildEditHint, findSourceForIndexedPath, isEditable } from "./search-source.js";
37
37
  /**
38
38
  * Age past which search surfaces a "run akm index" hint. Reads serve the
39
39
  * existing index as-is (freshness is the writers' job — `indexWrittenAssets`
@@ -461,7 +461,7 @@ export function applyEntryFilters(items, opts) {
461
461
  // index spans every configured source, so without this filter a narrowed
462
462
  // --from request would still leak results from other sources.
463
463
  const sourceFiltered = opts.restrictToSources
464
- ? items.filter((item) => findSourceForPath(item.filePath, opts.sources) !== undefined)
464
+ ? items.filter((item) => findSourceForIndexedPath(item.filePath, opts.sources) !== undefined)
465
465
  : items;
466
466
  // Scope filter: drop entries whose stored scope does not satisfy every
467
467
  // supplied key.
@@ -623,7 +623,7 @@ function classifyVectorFailure(error) {
623
623
  // ── Hit building ────────────────────────────────────────────────────────────
624
624
  export async function buildDbHit(input) {
625
625
  const absolutePath = path.resolve(input.path);
626
- const source = findSourceForPath(absolutePath, input.sources);
626
+ const source = findSourceForIndexedPath(absolutePath, input.sources);
627
627
  const entryStashDir = source?.path ?? input.defaultStashDir;
628
628
  const defaultBundleId = input.config?.defaultBundle ??
629
629
  (source && path.resolve(source.path) === path.resolve(input.defaultStashDir)
@@ -185,22 +185,29 @@ export function resolveEntryContentDir(entry) {
185
185
  * corrupting origin / editability / provenance decisions for the affected files.
186
186
  */
187
187
  export function findSourceForPath(filePath, sources) {
188
- const longest = (within) => {
189
- let best;
190
- let bestLen = -1;
191
- for (const source of sources) {
192
- const base = path.resolve(source.path);
193
- if (base.length > bestLen && within(base)) {
194
- best = source;
195
- bestLen = base.length;
196
- }
197
- }
198
- return best;
199
- };
188
+ // A caller's path may spell the library another way (a symlink); only then are real paths compared.
189
+ return (findSourceForIndexedPath(filePath, sources) ??
190
+ longestSource(sources, (base) => relativeToRoot(filePath, base) !== undefined));
191
+ }
192
+ /**
193
+ * {@link findSourceForPath} for a path read from the index. The index and the sources both store the library's real
194
+ * path, so the spelling is compared and nothing is read from disk: filtering every index row stays fast.
195
+ */
196
+ export function findSourceForIndexedPath(filePath, sources) {
200
197
  const resolved = path.resolve(filePath);
201
- // Paths from the index and the sources share one spelling; a caller's may not (a symlinked library).
202
- return (longest((base) => resolved.startsWith(base + path.sep)) ??
203
- longest((base) => relativeToRoot(resolved, base) !== undefined));
198
+ return longestSource(sources, (base) => resolved.startsWith(base + path.sep));
199
+ }
200
+ function longestSource(sources, within) {
201
+ let best;
202
+ let bestLen = -1;
203
+ for (const source of sources) {
204
+ const base = path.resolve(source.path);
205
+ if (base.length > bestLen && within(base)) {
206
+ best = source;
207
+ bestLen = base.length;
208
+ }
209
+ }
210
+ return best;
204
211
  }
205
212
  // ── Editability ─────────────────────────────────────────────────────────────
206
213
  /**
@@ -0,0 +1,204 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * The library's near-duplicate pairs, from the stored embeddings: what `assets.graph` reports so a host can list assets
6
+ * that say the same thing. To avoid comparing every pair, the assets are first bucketed by meaning (spherical k-means
7
+ * on the unit vectors) and only pairs inside a bucket are compared; the buckets are not reported. Nothing is embedded
8
+ * and nothing is written; the work yields to the event loop as it goes, and the result is kept while the vectors stay
9
+ * the same.
10
+ */
11
+ import { createHash } from "node:crypto";
12
+ /** Fewest embedded assets duplicates are looked for in. */
13
+ const MIN_ASSETS = 8;
14
+ /** Most buckets, however large the library. */
15
+ const MAX_BUCKETS = 36;
16
+ /** Most vectors the bucket centres are found from; the rest are only assigned to them. */
17
+ const SAMPLE = 4000;
18
+ const ROUNDS = 16;
19
+ /** Cosine at or above which two assets of one bucket are reported as near-duplicates. */
20
+ export const DUPLICATE_COSINE = 0.96;
21
+ /** Most near-duplicate pairs reported, the most alike first. */
22
+ const MAX_PAIRS = 2000;
23
+ const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve));
24
+ /** How many buckets for `n` embedded assets: about the square root of n/2, from 2 to {@link MAX_BUCKETS}. */
25
+ export function bucketCount(n) {
26
+ return Math.max(2, Math.min(MAX_BUCKETS, Math.round(Math.sqrt(n / 2))));
27
+ }
28
+ let cache;
29
+ /** The vectors' content, in order: an index build that changed no embedding gives the same key. */
30
+ function contentKey(vectors) {
31
+ const hash = createHash("sha1");
32
+ for (const v of vectors)
33
+ hash.update(v ? new Uint8Array(v.buffer, v.byteOffset, v.byteLength) : NO_VECTOR);
34
+ return hash.digest("hex");
35
+ }
36
+ const NO_VECTOR = new Uint8Array(1);
37
+ /**
38
+ * Near-duplicate pairs among the assets: spherical k-means puts the assets into buckets (centres from an evenly spaced
39
+ * sample of at most 4,000, seeded deterministically, so the same vectors give the same pairs), and every pair inside a
40
+ * bucket whose cosine is at least {@link DUPLICATE_COSINE} is reported. `null` without at least eight embedded assets.
41
+ * The last result is reused while the vectors are the same, so index builds that change no embedding cost nothing here.
42
+ */
43
+ export async function findDuplicates(vectors) {
44
+ const key = contentKey(vectors);
45
+ if (cache?.key === key)
46
+ return cache.result;
47
+ const dims = vectors.find((v) => v)?.length ?? 0;
48
+ const usable = [];
49
+ for (let i = 0; i < vectors.length; i++)
50
+ if (vectors[i]?.length === dims)
51
+ usable.push(i);
52
+ if (!dims || usable.length < MIN_ASSETS)
53
+ return null;
54
+ // Unit vectors, packed: cosine is then a dot product.
55
+ const unit = new Float32Array(usable.length * dims);
56
+ usable.forEach((i, row) => {
57
+ const v = vectors[i];
58
+ let norm = 0;
59
+ for (let d = 0; d < dims; d++)
60
+ norm += v[d] * v[d];
61
+ norm = Math.sqrt(norm) || 1;
62
+ for (let d = 0; d < dims; d++)
63
+ unit[row * dims + d] = v[d] / norm;
64
+ });
65
+ // A bit per dimension, set where the vector is above the library's mean. Vectors as alike as a duplicate differ in
66
+ // far fewer than a quarter of their bits (under a fifth on a 16,000-asset library), so a pair is multiplied out
67
+ // only when its codes are that close.
68
+ const words = Math.ceil(dims / 32);
69
+ const mean = new Float64Array(dims);
70
+ for (let row = 0; row < usable.length; row++)
71
+ for (let d = 0; d < dims; d++)
72
+ mean[d] += unit[row * dims + d];
73
+ const code = new Uint32Array(usable.length * words);
74
+ for (let row = 0; row < usable.length; row++)
75
+ for (let d = 0; d < dims; d++)
76
+ if (unit[row * dims + d] * usable.length > mean[d])
77
+ code[row * words + (d >> 5)] |= 1 << (d & 31);
78
+ const closeBits = Math.ceil(dims / 4);
79
+ const bitsApart = (x, y) => {
80
+ let bits = 0;
81
+ for (let w = 0; w < words; w++) {
82
+ let v = code[x * words + w] ^ code[y * words + w];
83
+ v -= (v >>> 1) & 0x55555555;
84
+ v = (v & 0x33333333) + ((v >>> 2) & 0x33333333);
85
+ bits += (((v + (v >>> 4)) & 0x0f0f0f0f) * 0x01010101) >>> 24;
86
+ }
87
+ return bits;
88
+ };
89
+ const dot = (row, centre, c) => {
90
+ let sum = 0;
91
+ for (let d = 0; d < dims; d++)
92
+ sum += unit[row * dims + d] * centre[c * dims + d];
93
+ return sum;
94
+ };
95
+ const k = bucketCount(usable.length);
96
+ const step = Math.max(1, usable.length / SAMPLE);
97
+ const sample = [];
98
+ for (let at = 0; at < usable.length && sample.length < SAMPLE; at += step)
99
+ sample.push(Math.floor(at));
100
+ // k-means++ from a fixed start, with a deterministic generator, so a library always gets the same buckets.
101
+ let seed = 7;
102
+ const random = () => {
103
+ seed = (seed * 1664525 + 1013904223) >>> 0;
104
+ return seed / 2 ** 32;
105
+ };
106
+ const centres = new Float32Array(k * dims);
107
+ const setCentre = (c, row) => centres.set(unit.subarray(row * dims, row * dims + dims), c * dims);
108
+ setCentre(0, sample[0]);
109
+ const far = new Float64Array(sample.length).fill(Number.POSITIVE_INFINITY);
110
+ for (let c = 1; c < k; c++) {
111
+ let total = 0;
112
+ for (let s = 0; s < sample.length; s++) {
113
+ far[s] = Math.min(far[s], Math.max(0, 1 - dot(sample[s], centres, c - 1)));
114
+ total += far[s] * far[s];
115
+ }
116
+ let r = random() * total;
117
+ let chosen = sample.length - 1;
118
+ for (let s = 0; s < sample.length; s++) {
119
+ r -= far[s] * far[s];
120
+ if (r <= 0) {
121
+ chosen = s;
122
+ break;
123
+ }
124
+ }
125
+ setCentre(c, sample[chosen]);
126
+ }
127
+ const nearest = (row) => {
128
+ let best = 0;
129
+ let bestDot = Number.NEGATIVE_INFINITY;
130
+ for (let c = 0; c < k; c++) {
131
+ const value = dot(row, centres, c);
132
+ if (value > bestDot) {
133
+ bestDot = value;
134
+ best = c;
135
+ }
136
+ }
137
+ return best;
138
+ };
139
+ for (let round = 0; round < ROUNDS; round++) {
140
+ const sums = new Float64Array(k * dims);
141
+ for (const row of sample) {
142
+ const c = nearest(row);
143
+ for (let d = 0; d < dims; d++)
144
+ sums[c * dims + d] += unit[row * dims + d];
145
+ }
146
+ for (let c = 0; c < k; c++) {
147
+ let norm = 0;
148
+ for (let d = 0; d < dims; d++)
149
+ norm += sums[c * dims + d] ** 2;
150
+ norm = Math.sqrt(norm);
151
+ // A centre that lost every member keeps its place.
152
+ if (norm > 0)
153
+ for (let d = 0; d < dims; d++)
154
+ centres[c * dims + d] = sums[c * dims + d] / norm;
155
+ }
156
+ await yieldToLoop();
157
+ }
158
+ // Every asset to its nearest centre.
159
+ const members = Array.from({ length: k }, () => []);
160
+ for (let row = 0; row < usable.length; row++) {
161
+ members[nearest(row)].push(row);
162
+ if (row % 2000 === 1999)
163
+ await yieldToLoop();
164
+ }
165
+ // Near-duplicates: every pair inside a bucket. Pairs across buckets are not compared; two assets that alike almost
166
+ // always share a bucket.
167
+ // Only the best MAX_PAIRS are kept, so a library full of copies does not hold every pair in memory: the list is cut
168
+ // back whenever it grows to eight times that, to the same top pairs a full sort would keep.
169
+ let pairs = [];
170
+ let pairCount = 0;
171
+ const byAlike = (x, y) => y[2] - x[2] || x[0] - y[0] || x[1] - y[1];
172
+ for (const rows of members) {
173
+ for (let x = 0; x < rows.length; x++) {
174
+ const p = rows[x] * dims;
175
+ for (let y = x + 1; y < rows.length; y++) {
176
+ if (bitsApart(rows[x], rows[y]) > closeBits)
177
+ continue;
178
+ const q = rows[y] * dims;
179
+ let cosine = 0;
180
+ for (let d = 0; d < dims; d++)
181
+ cosine += unit[p + d] * unit[q + d];
182
+ if (cosine >= DUPLICATE_COSINE) {
183
+ pairs.push([usable[rows[x]], usable[rows[y]], cosine]);
184
+ pairCount++;
185
+ if (pairs.length >= MAX_PAIRS * 8)
186
+ pairs = pairs.sort(byAlike).slice(0, MAX_PAIRS);
187
+ }
188
+ }
189
+ if (x % 200 === 199)
190
+ await yieldToLoop();
191
+ }
192
+ await yieldToLoop();
193
+ }
194
+ const kept = pairs.sort(byAlike).slice(0, MAX_PAIRS);
195
+ const result = {
196
+ threshold: DUPLICATE_COSINE,
197
+ a: kept.map((p) => p[0]),
198
+ b: kept.map((p) => p[1]),
199
+ cosine: kept.map((p) => Math.round(p[2] * 10000) / 10000),
200
+ truncated: pairCount > MAX_PAIRS,
201
+ };
202
+ cache = { key, result };
203
+ return result;
204
+ }
@@ -2,32 +2,29 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  /** The whole library as a graph: every indexed asset, every declared link between them, and how each is used. */
5
- import { createHash } from "node:crypto";
6
5
  import fs from "node:fs";
7
6
  import { loadConfig } from "../core/config/config.js";
8
7
  import { ConfigError } from "../core/errors.js";
9
- import { getDbPath } from "../core/paths.js";
10
8
  import { applyEntryFilters } from "../indexer/search/db-search.js";
11
9
  import { resolveSourceEntries } from "../indexer/search/search-source.js";
12
10
  import { countFeedbackSignalsByRef, countUseByRef, usageRetentionSince } from "../indexer/usage/usage-events.js";
13
11
  import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
14
12
  import { getAllEntries } from "../storage/repositories/index-entries-repository.js";
15
13
  import { listDeclaredLinks } from "../storage/repositories/index-links-repository.js";
16
- import { getMeta } from "../storage/repositories/index-meta-repository.js";
17
14
  import { listCurrentEmbeddings } from "../storage/repositories/index-vec-repository.js";
18
- import { findTopics } from "./graph-topics.js";
15
+ import { findDuplicates } from "./graph-duplicates.js";
19
16
  import { withReadonlyStateDb } from "./readers.js";
20
17
  /** Types never drawn: protected files and session logs, as `assets.related` leaves them out. */
21
18
  const HIDDEN_TYPES = ["secret", "env", "session"];
22
19
  /**
23
20
  * Every asset of the enabled bundles and the declared links between them, in column form: node `i` is `nodes.ref[i]`,
24
21
  * and its type, bundle and belief state are indexes into `types`, `bundles` (largest first) and `beliefs` (-1: no state
25
- * recorded). `shows`, `searches`, `lastShownAt` and the feedback counts `positive` and `negative` are read live from the
22
+ * recorded). `shows`, `lastShownAt` and the feedback counts `positive` and `negative` are read live from the
26
23
  * user-attributed usage events within retention (`use`); `modifiedAt` is the file's modification time. A link joins two
27
- * nodes by index and names its kind by index into `kinds`. `topic` indexes `topics` (see {@link findTopics}), -1 for
28
- * an asset with no stored embedding, and `duplicates` pairs assets of one topic whose embeddings are nearly the same. `meaning` places each node with a stored embedding in three dimensions (see
29
- * {@link projectEmbeddings}); it is `null` when the index has no embeddings. Nothing is written. The projection yields
30
- * to the event loop as it goes, so a host serving other requests keeps answering them.
24
+ * nodes by index and names its kind by index into `kinds`. `nodes.tags[i]` indexes `tags`, the asset's own frontmatter
25
+ * tags. `duplicates` pairs assets whose stored embeddings are nearly the same (see {@link findDuplicates}); it is `null`
26
+ * when too few assets are embedded. Nothing is written. The duplicate search yields to the event loop as it goes, so a
27
+ * host serving other requests keeps answering them.
31
28
  */
32
29
  export async function graph() {
33
30
  try {
@@ -49,29 +46,24 @@ function emptyGraph() {
49
46
  types: [],
50
47
  bundles: [],
51
48
  beliefs: [],
52
- emptyBundles: [],
53
49
  /** Use figures count events from `source` at or after `since`. */
54
50
  use: null,
55
- /** Declared links that are not drawn: target missing, hidden, or in a bundle that is not enabled. */
56
- unresolvedLinks: 0,
57
51
  kinds: [],
52
+ /** Every tag the assets carry, lower case; `nodes.tags[i]` indexes it. */
53
+ tags: [],
58
54
  nodes: {
59
55
  ref: [],
60
- title: [],
61
56
  type: [],
62
57
  bundle: [],
63
58
  belief: [],
64
59
  shows: [],
65
- searches: [],
66
60
  lastShownAt: [],
67
61
  modifiedAt: [],
68
62
  positive: [],
69
63
  negative: [],
70
- topic: [],
64
+ tags: [],
71
65
  },
72
66
  links: { from: [], to: [], kind: [] },
73
- meaning: null,
74
- topics: [],
75
67
  duplicates: null,
76
68
  };
77
69
  }
@@ -109,13 +101,13 @@ async function readGraph() {
109
101
  const typeCode = code(result.types);
110
102
  const beliefCode = code(result.beliefs);
111
103
  const kindCode = code(result.kinds);
104
+ const tagCode = code(result.tags);
112
105
  // Largest bundle first, so colour slots follow size; ties keep first-appearance order.
113
106
  const perBundle = new Map();
114
107
  for (const row of rows)
115
108
  perBundle.set(row.bundleId, (perBundle.get(row.bundleId) ?? 0) + 1);
116
109
  result.bundles.push(...[...perBundle.keys()].sort((x, y) => perBundle.get(y) - perBundle.get(x)));
117
110
  const bundleAt = new Map(result.bundles.map((bundle, i) => [bundle, i]));
118
- result.emptyBundles = sources.flatMap((source) => source.registryId && !perBundle.has(source.registryId) ? [source.registryId] : []);
119
111
  const since = usageRetentionSince();
120
112
  const use = withReadonlyStateDb((state) => countUseByRef(state, since), null);
121
113
  const feedback = withReadonlyStateDb((state) => countFeedbackSignalsByRef(state, rows.map((row) => row.itemRef)), null);
@@ -123,25 +115,22 @@ async function readGraph() {
123
115
  for (const row of rows) {
124
116
  const counts = use?.get(row.itemRef);
125
117
  result.nodes.ref.push(row.itemRef);
126
- result.nodes.title.push(row.entry.toc?.find((heading) => heading.level === 1)?.text || row.entry.name);
127
118
  result.nodes.type.push(typeCode(row.entry.type));
128
119
  result.nodes.bundle.push(bundleAt.get(row.bundleId));
129
120
  result.nodes.belief.push(row.entry.beliefState ? beliefCode(row.entry.beliefState) : -1);
130
121
  result.nodes.shows.push(counts?.shows ?? 0);
131
- result.nodes.searches.push(counts?.searches ?? 0);
132
122
  result.nodes.lastShownAt.push(counts?.lastShownAt ?? null);
133
123
  result.nodes.modifiedAt.push(modifiedAt(row.filePath));
134
124
  const signals = feedback?.get(row.itemRef);
135
125
  result.nodes.positive.push(signals?.pos ?? 0);
136
126
  result.nodes.negative.push(signals?.neg ?? 0);
127
+ result.nodes.tags.push([...new Set((row.entry.tags ?? []).map((tag) => tag.trim().toLowerCase()))].filter(Boolean).map(tagCode));
137
128
  }
138
129
  for (const link of listDeclaredLinks(db)) {
139
130
  const from = index.get(link.from);
140
131
  if (from === undefined)
141
132
  continue;
142
133
  const to = link.to === null ? undefined : index.get(link.to);
143
- if (to === undefined)
144
- result.unresolvedLinks++;
145
134
  if (to === undefined || from === to)
146
135
  continue;
147
136
  result.links.from.push(from);
@@ -149,27 +138,10 @@ async function readGraph() {
149
138
  result.links.kind.push(kindCode(link.kind));
150
139
  }
151
140
  const vectors = listCurrentEmbeddings(db);
152
- const build = `${getDbPath()}|${getMeta(db, "embeddingFingerprint") ?? ""}|${getMeta(db, "builtAt") ?? ""}`;
153
141
  closeDatabase(db);
154
142
  closed = true;
155
- const placed = await projectEmbeddings(rows.map((row) => vectors.get(row.id)), build);
156
- result.meaning = placed.embedded ? placed : null;
157
- const libraryKey = createHash("sha1")
158
- .update(rows.map((row) => row.id).join(","))
159
- .digest("hex");
160
- const found = await findTopics(rows.map((row) => vectors.get(row.id)), rows.map((row, i) => `${row.conceptId} ${result.nodes.title[i]}`),
161
- // Topics are per asset: the same build read through a different set of enabled bundles is a different library.
162
- `${build}|${libraryKey}`);
163
- result.topics = found.topics;
164
- result.nodes.topic = found.topic;
165
- result.duplicates = found.topics.length ? found.duplicates : null;
143
+ result.duplicates = await findDuplicates(rows.map((row) => vectors.get(row.id)));
166
144
  const noted = [`${rows.length} assets and ${result.links.from.length} links of the enabled bundles.`];
167
- if (result.unresolvedLinks)
168
- noted.push(`${result.unresolvedLinks} declared links are not drawn (target missing, hidden or in a bundle that is not enabled).`);
169
- if (result.emptyBundles.length)
170
- noted.push(`No assets in: ${result.emptyBundles.join(", ")}.`);
171
- const embedded = rows.filter((row) => vectors.has(row.id)).length;
172
- noted.push(embeddingNote(embedded, rows.length));
173
145
  if (!use)
174
146
  noted.push("Use counts are zero: no usage history is available.");
175
147
  return {
@@ -198,97 +170,3 @@ function modifiedAt(file) {
198
170
  return null;
199
171
  }
200
172
  }
201
- /** Fewest vectors the principal components can be found from. */
202
- const MIN_VECTORS = 4;
203
- /** The note's sentence on how many assets have a position by meaning. */
204
- export function embeddingNote(embedded, total) {
205
- if (embedded === 0)
206
- return "No assets are embedded, so there is no arrangement by meaning.";
207
- if (embedded < MIN_VECTORS)
208
- return `Only ${embedded} of ${total} assets are embedded; at least ${MIN_VECTORS} are needed to arrange by meaning.`;
209
- if (embedded === total)
210
- return `All ${total} assets are embedded.`;
211
- return `${embedded} of ${total} assets are embedded; the rest have no position by meaning.`;
212
- }
213
- /** Most vectors the principal components are found from; the rest are only projected onto them. */
214
- const SAMPLE = 4000;
215
- /** Vectors projected between yields to the event loop. */
216
- const CHUNK = 2000;
217
- const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve));
218
- /** The axes of the last projection, reused while the index's embedding model and full build stay the same. */
219
- let axesCache;
220
- /**
221
- * Three coordinates per vector: its projection on the three principal components of the vectors, so vectors that are
222
- * alike land near each other. The components come from an evenly spaced sample of at most 4,000 vectors (power
223
- * iteration with deflation from a fixed start, so the same vectors give the same layout), scaled so the first
224
- * component's standard deviation is 400. With a `key` (the index's embedding model and build), the components are
225
- * kept and reused until the key changes: an asset embedded since is projected onto them. A missing vector gives three
226
- * nulls. Work is done in slices with a yield to the event loop between them.
227
- */
228
- export async function projectEmbeddings(vectors, key) {
229
- const present = vectors.filter((vector) => vector !== undefined);
230
- const dims = present[0]?.length ?? 0;
231
- const usable = present.filter((vector) => vector.length === dims);
232
- if (!dims || usable.length < MIN_VECTORS)
233
- return { embedded: 0, xyz: vectors.flatMap(() => [null, null, null]) };
234
- let axes = key !== undefined && axesCache?.key === key && axesCache.dims === dims ? axesCache : undefined;
235
- if (!axes) {
236
- const step = Math.max(1, usable.length / SAMPLE);
237
- const sample = [];
238
- for (let at = 0; at < usable.length && sample.length < SAMPLE; at += step)
239
- sample.push(usable[Math.floor(at)]);
240
- const mean = new Float64Array(dims);
241
- for (const vector of sample)
242
- for (let d = 0; d < dims; d++)
243
- mean[d] += vector[d] / sample.length;
244
- const components = [];
245
- for (let c = 0; c < 3; c++) {
246
- let axis = new Float64Array(dims).map((_, d) => 1 + ((d * 7 + c * 13) % 5));
247
- for (let round = 0; round < 16; round++) {
248
- const next = new Float64Array(dims);
249
- for (const vector of sample) {
250
- const weight = projectOn(vector, mean, axis);
251
- for (let d = 0; d < dims; d++)
252
- next[d] += weight * (vector[d] - mean[d]);
253
- }
254
- // Keep the axis orthogonal to the components already found.
255
- for (const found of components) {
256
- let dot = 0;
257
- for (let d = 0; d < dims; d++)
258
- dot += next[d] * found[d];
259
- for (let d = 0; d < dims; d++)
260
- next[d] -= dot * found[d];
261
- }
262
- let norm = 0;
263
- for (let d = 0; d < dims; d++)
264
- norm += next[d] * next[d];
265
- norm = Math.sqrt(norm) || 1;
266
- axis = next.map((value) => value / norm);
267
- await yieldToLoop();
268
- }
269
- components.push(axis);
270
- }
271
- const spread = Math.sqrt(sample.reduce((sum, vector) => sum + projectOn(vector, mean, components[0]) ** 2, 0) / sample.length);
272
- axes = { key: key ?? "", dims, mean, components, scale: spread > 0 ? 400 / spread : 1 };
273
- if (key !== undefined)
274
- axesCache = axes;
275
- }
276
- const xyz = [];
277
- for (let at = 0; at < vectors.length; at += CHUNK) {
278
- for (const vector of vectors.slice(at, at + CHUNK)) {
279
- if (vector && vector.length === dims)
280
- for (const axis of axes.components)
281
- xyz.push(Math.round(projectOn(vector, axes.mean, axis) * axes.scale * 10) / 10);
282
- else
283
- xyz.push(null, null, null);
284
- }
285
- await yieldToLoop();
286
- }
287
- return { embedded: usable.length, xyz };
288
- }
289
- function projectOn(vector, mean, axis) {
290
- let sum = 0;
291
- for (let d = 0; d < axis.length; d++)
292
- sum += (vector[d] - mean[d]) * axis[d];
293
- return sum;
294
- }
@@ -72,7 +72,6 @@ Vector search is an exact scan of `embeddings` (`src/storage/repositories/index-
72
72
  #### `index_meta`
73
73
 
74
74
  `key` TEXT PRIMARY KEY, `value` TEXT NOT NULL. Keys: `version`, `builtAt`,
75
- `stashDir`, `stashDirs`, `sourceOwners`, `last_utility_computed_at`, `embeddingFingerprint` (the model the
76
75
  vectors were made under), `hasEmbeddings` (`"0"` or `"1"`), `semanticSearch` (what the
77
76
  last `akm index` found about semantic search; `akm doctor` reads it), `vacuumPending`,
78
77
  and `ftsTotalsStale` (set when a delete removed a row from `entries_fts`;
@@ -195,49 +195,36 @@ them, in column form, so 30,000 assets stay a few megabytes:
195
195
  "types": ["knowledge", "memory"],
196
196
  "bundles": ["team-wiki"],
197
197
  "beliefs": ["superseded"],
198
- "emptyBundles": [],
199
198
  "use": { "since": "2026-08-15T08:00:00.000Z", "source": "user" },
200
- "unresolvedLinks": 2,
201
199
  "kinds": ["xref", "links_to"],
202
- "nodes": { "ref": ["team-wiki//knowledge/a"], "title": ["A"], "type": [0], "bundle": [0], "belief": [-1],
203
- "shows": [12], "searches": [40], "lastShownAt": ["2026-11-02T09:14:03Z"],
204
- "modifiedAt": ["2026-03-30T17:02:11.000Z"], "positive": [2], "negative": [0], "topic": [4] },
200
+ "tags": ["deploy"],
201
+ "nodes": { "ref": ["team-wiki//knowledge/a"], "type": [0], "bundle": [0], "belief": [-1], "shows": [12],
202
+ "lastShownAt": ["2026-11-02T09:14:03Z"], "modifiedAt": ["2026-03-30T17:02:11.000Z"],
203
+ "positive": [2], "negative": [0], "tags": [[0]] },
205
204
  "links": { "from": [0], "to": [3], "kind": [1] },
206
- "meaning": { "embedded": 29840, "xyz": [12.5, -40.1, 3.2] },
207
- "topics": [{ "label": "deploy · canary", "terms": ["deploy", "canary", "rollout"], "size": 812,
208
- "related": [{ "topic": 7, "cosine": 0.81 }] }],
209
205
  "duplicates": { "threshold": 0.96, "a": [0], "b": [9], "cosine": [0.9871], "truncated": false }
210
206
  }
211
207
  ```
212
208
 
213
209
  Node `i` is index `i` of every `nodes` array; `type`, `bundle`, `belief` and `kind` index `types`, `bundles`, `beliefs`
214
- and `kinds`. `bundles` runs from the bundle with the most assets to the one with the fewest, and `emptyBundles` names
215
- the enabled bundles with none. `belief` is the asset's recorded belief state, `-1` when it has none. A link runs from
216
- the asset that declares it to its target; `unresolvedLinks` counts the declared links that are left out because the
217
- target is missing, of a type that is not returned, or in a bundle that is not enabled. `shows` and `searches` are
218
- counted from the usage events when the operation is called: events of source `user` (people and their agent
219
- sessions, not scheduled tasks or Improve) at or after `use.since`, the start of the usage retention window. They are
220
- zero, and the note says so, when no usage history is available. `lastShownAt` is the latest of those shows (UTC), `null`
221
- when there is none in the window. `positive` and `negative` count feedback from people inside the retention window, as
222
- search ranks by it (a negative answered by an accepted fix does not count). `modifiedAt` is the file's modification
223
- time, `null` when it cannot be read; in a Git checkout it is when the file was last written there, not its last commit. `meaning` places each asset with a stored embedding
224
- of the index's current model in three dimensions (three coordinates per node, `null` for one with no embedding): its
225
- projection on the three principal components of the vectors, so assets with similar content land near each other. The
226
- components are found from an evenly spaced sample of at most 4,000 vectors and kept for the life of the process until
227
- the index's embedding model or full build changes; an asset embedded since is projected onto them. The work yields to
228
- the event loop as it goes, so a host's other requests are served meanwhile. `meaning` is `null` when the index has no
229
- embeddings.
230
-
231
- `topics` groups the embedded assets by meaning: spherical k-means on their stored vectors, about √(n/2) topics (2 to 36),
232
- largest first, the centres found from an evenly spaced sample of at most 4,000 vectors from a fixed start, so the same
233
- index gives the same topics. A topic's `terms` are the words of its assets' paths and titles most particular to it
234
- (frequent in the topic, rare in the library), and its `label` the first two; `related` names the three topics whose
235
- centres are nearest, with the cosine between them. `nodes.topic` indexes `topics`, `-1` for an asset with no embedding.
236
- `duplicates` lists pairs of assets of one topic whose vectors have a cosine of at least `threshold` (0.96), most alike
237
- first, at most 2,000 (`truncated` says there were more); pairs across topics are not compared. Both are `[]` and `null`
238
- without at least eight embedded assets. They are found once per index build and kept, like the projection: the first
239
- read of a 20,000-asset index takes several seconds. `available` is `false`, with a `note`,
240
- when there is no index.
210
+ and `kinds`, and each entry of `nodes.tags[i]` indexes `tags`, the asset's own frontmatter tags (lower case, once each).
211
+ `bundles` runs from the bundle with the most assets to the one with the fewest. `belief` is the asset's recorded belief
212
+ state, `-1` when it has none. A link runs from the asset that declares it to its target; links whose target is missing,
213
+ of a type that is not returned, or in a bundle that is not enabled are left out. `shows` is counted from the usage
214
+ events when the operation is called: events of source `user` (people and their agent sessions, not scheduled tasks or
215
+ Improve) at or after `use.since`, the start of the usage retention window. It is zero, and the note says so, when no
216
+ usage history is available. `lastShownAt` is the latest of those shows (UTC), `null` when there is none in the window.
217
+ `positive` and `negative` count feedback from people inside the retention window, as search ranks by it (a negative
218
+ answered by an accepted fix does not count). `modifiedAt` is the file's modification time, `null` when it cannot be
219
+ read; in a Git checkout it is when the file was last written there, not its last commit.
220
+
221
+ `duplicates` lists pairs of assets (indexes into `nodes`) whose stored embeddings have a cosine of at least `threshold`
222
+ (0.96), most alike first, at most 2,000 (`truncated` says there were more). To avoid comparing every pair, the assets
223
+ are first grouped by meaning (spherical k-means on their vectors, from a fixed start, so the same index gives the same
224
+ pairs) and only pairs inside a group are compared. It is `null` without at least eight embedded assets. It is kept
225
+ while the embeddings are the same, so an index build that changes none costs nothing; after one that does, the first
226
+ read of a 16,000-asset library takes about five seconds. `available` is `false`,
227
+ with a `note`, when there is no index.
241
228
 
242
229
  ## `bundles.list`
243
230
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "akm-cli",
3
- "version": "0.10.261125",
3
+ "version": "0.10.261126",
4
4
  "type": "module",
5
5
  "description": "akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cursor.",
6
6
  "keywords": [
@@ -1,249 +0,0 @@
1
- // This Source Code Form is subject to the terms of the Mozilla Public
2
- // License, v. 2.0. If a copy of the MPL was not distributed with this
3
- // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
- /**
5
- * The library's topics and near-duplicates, from the stored embeddings: what `assets.graph` reports so a host can say
6
- * what the library holds, where it overlaps, and which assets say the same thing. Nothing is embedded and nothing is
7
- * written; the work yields to the event loop as it goes and is kept for one index build.
8
- */
9
- /** Fewest embedded assets topics are found for. */
10
- const MIN_ASSETS = 8;
11
- /** Most topics, however large the library. */
12
- const MAX_TOPICS = 36;
13
- /** Most vectors the topic centres are found from; the rest are only assigned to them. */
14
- const SAMPLE = 4000;
15
- const ROUNDS = 16;
16
- /** Cosine at or above which two assets of one topic are reported as near-duplicates. */
17
- export const DUPLICATE_COSINE = 0.96;
18
- /** Most near-duplicate pairs reported, the most alike first. */
19
- const MAX_PAIRS = 2000;
20
- /** Topics each topic names as nearest. */
21
- const RELATED_TOPICS = 3;
22
- const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve));
23
- /** How many topics for `n` embedded assets: about the square root of n/2, from 2 to {@link MAX_TOPICS}. */
24
- export function topicCount(n) {
25
- return Math.max(2, Math.min(MAX_TOPICS, Math.round(Math.sqrt(n / 2))));
26
- }
27
- let cache;
28
- /**
29
- * Groups the assets by meaning (spherical k-means on the unit vectors: centres from an evenly spaced sample of at most
30
- * 4,000, seeded deterministically, so the same vectors give the same topics), names each topic by the words of its
31
- * assets' paths and titles that are most particular to it, and finds near-duplicate pairs inside each topic. `names[i]`
32
- * is asset i's path and title. With a `key` (the index's build), the result is reused until the key changes.
33
- */
34
- export async function findTopics(vectors, names, key) {
35
- if (key !== undefined && cache?.key === key && cache.result.topic.length === vectors.length)
36
- return cache.result;
37
- const empty = {
38
- topics: [],
39
- topic: vectors.map(() => -1),
40
- duplicates: { threshold: DUPLICATE_COSINE, a: [], b: [], cosine: [], truncated: false },
41
- };
42
- const dims = vectors.find((v) => v)?.length ?? 0;
43
- const usable = [];
44
- for (let i = 0; i < vectors.length; i++)
45
- if (vectors[i]?.length === dims)
46
- usable.push(i);
47
- if (!dims || usable.length < MIN_ASSETS)
48
- return empty;
49
- // Unit vectors, packed: cosine is then a dot product.
50
- const unit = new Float32Array(usable.length * dims);
51
- usable.forEach((i, row) => {
52
- const v = vectors[i];
53
- let norm = 0;
54
- for (let d = 0; d < dims; d++)
55
- norm += v[d] * v[d];
56
- norm = Math.sqrt(norm) || 1;
57
- for (let d = 0; d < dims; d++)
58
- unit[row * dims + d] = v[d] / norm;
59
- });
60
- const dot = (row, centre, c) => {
61
- let sum = 0;
62
- for (let d = 0; d < dims; d++)
63
- sum += unit[row * dims + d] * centre[c * dims + d];
64
- return sum;
65
- };
66
- const k = topicCount(usable.length);
67
- const step = Math.max(1, usable.length / SAMPLE);
68
- const sample = [];
69
- for (let at = 0; at < usable.length && sample.length < SAMPLE; at += step)
70
- sample.push(Math.floor(at));
71
- // k-means++ from a fixed start, with a deterministic generator, so a library always gets the same topics.
72
- let seed = 7;
73
- const random = () => {
74
- seed = (seed * 1664525 + 1013904223) >>> 0;
75
- return seed / 2 ** 32;
76
- };
77
- const centres = new Float32Array(k * dims);
78
- const setCentre = (c, row) => centres.set(unit.subarray(row * dims, row * dims + dims), c * dims);
79
- setCentre(0, sample[0]);
80
- const far = new Float64Array(sample.length).fill(Number.POSITIVE_INFINITY);
81
- for (let c = 1; c < k; c++) {
82
- let total = 0;
83
- for (let s = 0; s < sample.length; s++) {
84
- far[s] = Math.min(far[s], Math.max(0, 1 - dot(sample[s], centres, c - 1)));
85
- total += far[s] * far[s];
86
- }
87
- let r = random() * total;
88
- let chosen = sample.length - 1;
89
- for (let s = 0; s < sample.length; s++) {
90
- r -= far[s] * far[s];
91
- if (r <= 0) {
92
- chosen = s;
93
- break;
94
- }
95
- }
96
- setCentre(c, sample[chosen]);
97
- }
98
- const nearest = (row) => {
99
- let best = 0;
100
- let bestDot = Number.NEGATIVE_INFINITY;
101
- for (let c = 0; c < k; c++) {
102
- const value = dot(row, centres, c);
103
- if (value > bestDot) {
104
- bestDot = value;
105
- best = c;
106
- }
107
- }
108
- return best;
109
- };
110
- for (let round = 0; round < ROUNDS; round++) {
111
- const sums = new Float64Array(k * dims);
112
- for (const row of sample) {
113
- const c = nearest(row);
114
- for (let d = 0; d < dims; d++)
115
- sums[c * dims + d] += unit[row * dims + d];
116
- }
117
- for (let c = 0; c < k; c++) {
118
- let norm = 0;
119
- for (let d = 0; d < dims; d++)
120
- norm += sums[c * dims + d] ** 2;
121
- norm = Math.sqrt(norm);
122
- // A centre that lost every member keeps its place.
123
- if (norm > 0)
124
- for (let d = 0; d < dims; d++)
125
- centres[c * dims + d] = sums[c * dims + d] / norm;
126
- }
127
- await yieldToLoop();
128
- }
129
- // Every asset to its nearest centre; topics are renumbered largest first, and empty ones dropped.
130
- const raw = new Int32Array(usable.length);
131
- for (let row = 0; row < usable.length; row++) {
132
- raw[row] = nearest(row);
133
- if (row % 2000 === 1999)
134
- await yieldToLoop();
135
- }
136
- const sizes = new Array(k).fill(0);
137
- for (const c of raw)
138
- sizes[c]++;
139
- const order = sizes
140
- .map((size, c) => ({ size, c }))
141
- .filter((t) => t.size > 0)
142
- .sort((x, y) => y.size - x.size || x.c - y.c);
143
- const renumber = new Map(order.map((t, n) => [t.c, n]));
144
- const topic = vectors.map(() => -1);
145
- usable.forEach((i, row) => {
146
- topic[i] = renumber.get(raw[row]);
147
- });
148
- const members = order.map(() => []);
149
- usable.forEach((i, row) => {
150
- members[topic[i]].push(row);
151
- });
152
- const centreOf = (n) => order[n].c;
153
- const topics = order.map((t, n) => ({
154
- ...nameTopic(members[n].map((row) => names[usable[row]] ?? ""), names),
155
- size: t.size,
156
- related: order
157
- .map((_, m) => {
158
- let cosine = 0;
159
- for (let d = 0; d < dims; d++)
160
- cosine += centres[centreOf(n) * dims + d] * centres[centreOf(m) * dims + d];
161
- return { topic: m, cosine: Math.round(cosine * 1000) / 1000 };
162
- })
163
- .filter((r) => r.topic !== n)
164
- .sort((x, y) => y.cosine - x.cosine)
165
- .slice(0, RELATED_TOPICS),
166
- }));
167
- // Near-duplicates: every pair inside a topic. Pairs across topics are not compared; two assets that alike almost
168
- // always share a topic.
169
- // Only the best MAX_PAIRS are kept, so a library full of copies does not hold every pair in memory: the list is cut
170
- // back whenever it grows to eight times that, to the same top pairs a full sort would keep.
171
- let pairs = [];
172
- let pairCount = 0;
173
- const byAlike = (x, y) => y[2] - x[2] || x[0] - y[0] || x[1] - y[1];
174
- for (const rows of members) {
175
- for (let x = 0; x < rows.length; x++) {
176
- const p = rows[x] * dims;
177
- for (let y = x + 1; y < rows.length; y++) {
178
- const q = rows[y] * dims;
179
- let cosine = 0;
180
- for (let d = 0; d < dims; d++)
181
- cosine += unit[p + d] * unit[q + d];
182
- if (cosine >= DUPLICATE_COSINE) {
183
- pairs.push([usable[rows[x]], usable[rows[y]], cosine]);
184
- pairCount++;
185
- if (pairs.length >= MAX_PAIRS * 8)
186
- pairs = pairs.sort(byAlike).slice(0, MAX_PAIRS);
187
- }
188
- }
189
- if (x % 200 === 199)
190
- await yieldToLoop();
191
- }
192
- await yieldToLoop();
193
- }
194
- const kept = pairs.sort(byAlike).slice(0, MAX_PAIRS);
195
- const result = {
196
- topics,
197
- topic,
198
- duplicates: {
199
- threshold: DUPLICATE_COSINE,
200
- a: kept.map((p) => p[0]),
201
- b: kept.map((p) => p[1]),
202
- cosine: kept.map((p) => Math.round(p[2] * 10000) / 10000),
203
- truncated: pairCount > MAX_PAIRS,
204
- },
205
- };
206
- if (key !== undefined)
207
- cache = { key, result };
208
- return result;
209
- }
210
- const STOP = new Set("the and for with from into this that what when how why are was not use using about your you our its md readme index skill skills knowledge memory memories command commands agent agents script scripts workflow workflows fact facts lesson lessons references reference assets asset src lib test tests docs".split(" "));
211
- /** The words of a path or title worth naming a topic by: lower case, three letters or more, no digits, no stop words. */
212
- export function words(text) {
213
- return text
214
- .toLowerCase()
215
- .split(/[^a-z0-9]+/)
216
- .filter((w) => w.length >= 3 && !/\d/.test(w) && !STOP.has(w));
217
- }
218
- let documentFrequency;
219
- /**
220
- * A topic's distinctive words: those most frequent among its assets relative to how many assets of the whole library
221
- * use them (each asset counts a word once). The label is the top two.
222
- */
223
- export function nameTopic(memberNames, allNames) {
224
- if (documentFrequency?.names !== allNames) {
225
- const counts = new Map();
226
- for (const name of allNames)
227
- for (const w of new Set(words(name)))
228
- counts.set(w, (counts.get(w) ?? 0) + 1);
229
- documentFrequency = { names: allNames, counts };
230
- }
231
- const inTopic = new Map();
232
- for (const name of memberNames)
233
- for (const w of new Set(words(name)))
234
- inTopic.set(w, (inTopic.get(w) ?? 0) + 1);
235
- const total = allNames.length;
236
- const terms = [...inTopic]
237
- // A word one asset uses names nothing.
238
- .filter(([, n]) => n >= Math.min(2, memberNames.length))
239
- .map(([w, n]) => ({
240
- w,
241
- score: (n / memberNames.length) * Math.log(total / (documentFrequency.counts.get(w) ?? 1)),
242
- }))
243
- // A word every asset of the library has says nothing about this topic.
244
- .filter((t) => t.score > 0)
245
- .sort((x, y) => y.score - x.score || (x.w < y.w ? -1 : 1))
246
- .slice(0, 5)
247
- .map((t) => t.w);
248
- return { label: terms.slice(0, 2).join(" · ") || "unnamed", terms };
249
- }