akm-cli 0.10.261124 → 0.10.261125-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/cli.js +10 -0
  3. package/dist/commands/feedback-cli.js +12 -3
  4. package/dist/commands/health/semantic-search-check.js +12 -2
  5. package/dist/commands/health.js +8 -15
  6. package/dist/commands/lint/index.js +6 -6
  7. package/dist/commands/read/knowledge.js +8 -8
  8. package/dist/commands/read/related.js +3 -9
  9. package/dist/commands/read/show.js +17 -9
  10. package/dist/commands/sources/bundle-cli.js +1 -0
  11. package/dist/commands/sources/installed-stashes.js +36 -27
  12. package/dist/core/asset/resolve-ref.js +8 -0
  13. package/dist/core/common.js +21 -0
  14. package/dist/core/endpoint-url.js +4 -4
  15. package/dist/core/events.js +8 -2
  16. package/dist/core/file-lock.js +52 -1
  17. package/dist/core/state-db.js +47 -15
  18. package/dist/core/stdio-reader-gone.js +19 -0
  19. package/dist/core/write-source.js +2 -2
  20. package/dist/indexer/index-build.js +6 -3
  21. package/dist/indexer/index-readiness.js +30 -4
  22. package/dist/indexer/indexer.js +56 -8
  23. package/dist/indexer/search/db-search.js +7 -1
  24. package/dist/indexer/search/search-source.js +23 -25
  25. package/dist/indexer/usage/usage-events.js +13 -4
  26. package/dist/registry/origin-resolve.js +2 -3
  27. package/dist/scripts/akm-migrate-node.js +118 -72
  28. package/dist/scripts/akm-migrate.js +118 -72
  29. package/dist/setup/endpoint.js +1 -1
  30. package/dist/storage/repositories/index-connection.js +44 -41
  31. package/dist/storage/repositories/index-entries-repository.js +14 -7
  32. package/dist/storage/repositories/index-entry-schema.js +7 -5
  33. package/dist/storage/repositories/index-schema.js +23 -0
  34. package/dist/workspace/assets.js +24 -44
  35. package/dist/workspace/bundle-files.js +2 -2
  36. package/dist/workspace/documents.js +7 -12
  37. package/dist/workspace/graph-topics.js +249 -0
  38. package/dist/workspace/graph.js +68 -4
  39. package/docs/reference/workspace.md +21 -4
  40. package/package.json +1 -1
@@ -569,20 +569,27 @@ export function findEntryIdByRef(db, ref, bundleId) {
569
569
  export function withMdVariants(name) {
570
570
  return name.endsWith(".md") ? [name, name.slice(0, -3)] : [name, `${name}.md`];
571
571
  }
572
+ /** True when `file` is a `.md` file named by the last segment of `conceptId`, up to its `.md`. */
573
+ export function namesMarkdownFile(conceptId, file) {
574
+ const last = (conceptId.split("/").pop() ?? "").replace(/\.md$/, "");
575
+ return path.extname(file) === ".md" && path.basename(file, ".md") === last;
576
+ }
572
577
  /**
573
578
  * Current (`[bundle//]conceptId`) id lookup: match `item_ref` exactly when
574
579
  * bundle-qualified or by `//conceptId` suffix when short.
575
580
  */
576
581
  function findEntryIdByBundleRef(db, ref, bundleId) {
577
582
  const parsed = parseBundleRef(ref);
578
- const conceptVariants = withMdVariants(parsed.conceptId);
579
583
  // item_ref is the canonical stored spelling post-flip.
580
- for (const conceptId of conceptVariants) {
581
- const id = matchIdByItemRef(db, parsed.bundle, conceptId, bundleId);
582
- if (id !== undefined)
583
- return id;
584
- }
585
- return undefined;
584
+ const exact = matchIdByItemRef(db, parsed.bundle, parsed.conceptId, bundleId);
585
+ if (exact !== undefined)
586
+ return exact;
587
+ // The `.md` toggle names a markdown file by its own name: `tasks/x.md` is not a spelling of `tasks/x.yml`,
588
+ // nor `skills/x.md` of `skills/x/SKILL.md`.
589
+ const [, toggled] = withMdVariants(parsed.conceptId);
590
+ const id = matchIdByItemRef(db, parsed.bundle, toggled, bundleId);
591
+ const file = id === undefined ? undefined : getEntryFilePathById(db, id);
592
+ return file !== undefined && namesMarkdownFile(parsed.conceptId, file) ? id : undefined;
586
593
  }
587
594
  /**
588
595
  * Match a row `id` by `item_ref`: exact `bundle//conceptId` when `bundle` is
@@ -142,14 +142,16 @@ export function retiredEntryColumns(db) {
142
142
  return RETIRED_ENTRY_COLUMNS.filter((column) => present.has(column));
143
143
  }
144
144
  /**
145
- * Whether an index database has an `entries` table this release can read. A
146
- * fresh or empty file has none; an index older than layout 21 has one keyed
147
- * by columns this release no longer reads (or lacks the ones it does), and
148
- * serves nothing until the next `akm index` recreates it.
145
+ * Whether an index database has an `entries` table this release can read,
146
+ * with its full-text table. A fresh or empty file has none; an index older
147
+ * than layout 21 has one keyed by columns this release no longer reads (or
148
+ * lacks the ones it does); a file whose schema was cut short has entries
149
+ * without `entries_fts`. Each serves nothing until the next `akm index`.
149
150
  */
150
151
  export function hasCurrentEntriesTable(db) {
151
152
  try {
152
- return missingEntryColumns(db).length === 0 && retiredEntryColumns(db).length === 0;
153
+ const fts = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'entries_fts'").get();
154
+ return Boolean(fts) && missingEntryColumns(db).length === 0 && retiredEntryColumns(db).length === 0;
153
155
  }
154
156
  catch {
155
157
  return false;
@@ -205,7 +205,30 @@ function ensureColumn(db, table, column, type) {
205
205
  db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${type}`);
206
206
  return true;
207
207
  }
208
+ /**
209
+ * Bring `index.db` to the current layout in one transaction, so no reader or
210
+ * crash can observe a half-migrated file (entries without their FTS table).
211
+ * One `BEGIN IMMEDIATE`, waiting out the connection's busy timeout once: a
212
+ * busy index surfaces to the caller (the build reports it as contention)
213
+ * rather than being retried here. Joins a transaction the caller already holds.
214
+ */
208
215
  export function ensureSchema(db) {
216
+ if (db.inTransaction) {
217
+ applySchema(db);
218
+ return;
219
+ }
220
+ db.exec("BEGIN IMMEDIATE");
221
+ try {
222
+ applySchema(db);
223
+ db.exec("COMMIT");
224
+ }
225
+ catch (error) {
226
+ if (db.inTransaction)
227
+ db.exec("ROLLBACK");
228
+ throw error;
229
+ }
230
+ }
231
+ function applySchema(db) {
209
232
  db.exec(`
210
233
  CREATE TABLE IF NOT EXISTS index_meta (
211
234
  key TEXT PRIMARY KEY,
@@ -13,12 +13,11 @@ import { loadConfig } from "../core/config/config.js";
13
13
  import { resolveRefOwner } from "../core/env-secret-ref.js";
14
14
  import { UsageError } from "../core/errors.js";
15
15
  import { includedTypes, includesProposed } from "../core/include.js";
16
- import { getDbPath } from "../core/paths.js";
17
16
  import { projectView, resolveProject } from "../core/projects.js";
17
+ import { readUsableIndex } from "../indexer/index-readiness.js";
18
18
  import { resolveReadSources } from "../indexer/read-preflight.js";
19
19
  import { countBrowseEntries } from "../indexer/search/db-search.js";
20
20
  import { findSourceForPath, resolveSourceEntries } from "../indexer/search/search-source.js";
21
- import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
22
21
  import { getEntryIdByFilePath, getItemRefById } from "../storage/repositories/index-entries-repository.js";
23
22
  import { countFeedbackBySignal, listFeedbackEventRows } from "../storage/repositories/metrics-repository.js";
24
23
  import { assetImprove } from "./asset-improve.js";
@@ -81,16 +80,9 @@ export function projectOverview(input) {
81
80
  ? local.map((source) => ({ name: source.registryId ?? source.path, source }))
82
81
  : [project.from].flat().map((from) => ({ name: from, source: narrowSourcesToName(local, [from])[0] }));
83
82
  const readable = selected.flatMap((bundle) => (bundle.source ? [bundle.source] : []));
84
- let counts;
85
- const db = readable.length > 0 ? openReadonlyExistingDatabase(getDbPath()) : undefined;
86
- if (db) {
87
- try {
88
- counts = countBrowseEntries(db, readable, project.filters, (config.search?.defaultExcludeTypes ?? ["session"]).filter((type) => !includedTypes(project.include).includes(type)), includesProposed(project.include));
89
- }
90
- finally {
91
- closeDatabase(db);
92
- }
93
- }
83
+ const counts = readable.length > 0
84
+ ? readUsableIndex((db) => countBrowseEntries(db, readable, project.filters, (config.search?.defaultExcludeTypes ?? ["session"]).filter((type) => !includedTypes(project.include).includes(type)), includesProposed(project.include)), undefined)
85
+ : undefined;
94
86
  const writeName = project.write.bundle ?? config.defaultWriteTarget ?? automaticWriteTarget(config)?.name ?? null;
95
87
  const writeSource = writeName === null ? undefined : local.find((source) => source.registryId === writeName);
96
88
  return {
@@ -121,16 +113,10 @@ export function projectOverview(input) {
121
113
  }
122
114
  /** The durable `bundle//conceptId` of the indexed asset at `filePath`, the key feedback rows use. */
123
115
  function durableRefForPath(filePath) {
124
- const db = openReadonlyExistingDatabase(getDbPath());
125
- if (!db)
126
- return null;
127
- try {
116
+ return readUsableIndex((db) => {
128
117
  const id = getEntryIdByFilePath(db, filePath);
129
118
  return id === undefined ? null : getItemRefById(db, id);
130
- }
131
- finally {
132
- closeDatabase(db);
133
- }
119
+ }, null);
134
120
  }
135
121
  function feedbackFor(durableRef) {
136
122
  const empty = { counts: {}, history: [] };
@@ -174,32 +160,26 @@ export async function show(input) {
174
160
  const links = {};
175
161
  if (shown.path && durableRef) {
176
162
  const owner = parseBundleRef(durableRef).bundle;
177
- const db = openReadonlyExistingDatabase(getDbPath());
178
- if (db) {
179
- try {
180
- for (const raw of markdownLinkTargets(content, false)) {
181
- if (/^(?:[a-z][a-z0-9+.-]*:|\/\/|#)/i.test(raw))
182
- continue;
183
- const [file, fragment] = raw.split("#", 2);
184
- if (!file?.toLowerCase().endsWith(".md") || file.startsWith("/"))
185
- continue;
186
- let decoded;
187
- try {
188
- decoded = decodeURIComponent(file);
189
- }
190
- catch {
191
- continue;
192
- }
193
- const id = getEntryIdByFilePath(db, path.resolve(path.dirname(shown.path), decoded));
194
- const target = id === undefined ? undefined : getItemRefById(db, id);
195
- if (target && parseBundleRef(target).bundle === owner)
196
- links[raw] = target + (fragment ? `#${fragment}` : "");
163
+ readUsableIndex((db) => {
164
+ for (const raw of markdownLinkTargets(content, false)) {
165
+ if (/^(?:[a-z][a-z0-9+.-]*:|\/\/|#)/i.test(raw))
166
+ continue;
167
+ const [file, fragment] = raw.split("#", 2);
168
+ if (!file?.toLowerCase().endsWith(".md") || file.startsWith("/"))
169
+ continue;
170
+ let decoded;
171
+ try {
172
+ decoded = decodeURIComponent(file);
197
173
  }
174
+ catch {
175
+ continue;
176
+ }
177
+ const id = getEntryIdByFilePath(db, path.resolve(path.dirname(shown.path), decoded));
178
+ const target = id === undefined ? undefined : getItemRefById(db, id);
179
+ if (target && parseBundleRef(target).bundle === owner)
180
+ links[raw] = target + (fragment ? `#${fragment}` : "");
198
181
  }
199
- finally {
200
- closeDatabase(db);
201
- }
202
- }
182
+ }, undefined);
203
183
  }
204
184
  return {
205
185
  redacted: false,
@@ -3,7 +3,7 @@
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import fs from "node:fs";
5
5
  import path from "node:path";
6
- import { isWithin, safeRealpath } from "../core/common.js";
6
+ import { isWithin, relativeToRoot, safeRealpath } from "../core/common.js";
7
7
  import { NotFoundError, UsageError } from "../core/errors.js";
8
8
  import { findSourceForPath, resolveSourceEntries } from "../indexer/search/search-source.js";
9
9
  import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
@@ -14,7 +14,7 @@ export function bundleFileLocation(filePath) {
14
14
  const source = findSourceForPath(filePath, resolveSourceEntries());
15
15
  if (!source?.registryId || source.unresolved || !isWithin(filePath, source.path))
16
16
  return null;
17
- const relative = path.relative(source.path, filePath);
17
+ const relative = relativeToRoot(filePath, source.path) ?? path.relative(source.path, filePath);
18
18
  return {
19
19
  bundle: source.registryId,
20
20
  path: portable(relative),
@@ -24,7 +24,7 @@ import { appendEvent } from "../core/events.js";
24
24
  import { resolveBundleWriteTarget } from "../core/mutation-target.js";
25
25
  import { warn } from "../core/warn.js";
26
26
  import { assertAkmAssetWrite } from "../core/write-source.js";
27
- import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
27
+ import { NO_INDEX_MESSAGE, readUsableIndex } from "../indexer/index-readiness.js";
28
28
  import { getEntryIdByFilePath, getItemRefById } from "../storage/repositories/index-entries-repository.js";
29
29
  import { interactive } from "./lease.js";
30
30
  import { redactUrlsInText } from "./redact.js";
@@ -37,17 +37,12 @@ async function resolveWritableIndexedAsset(ref) {
37
37
  const shown = await akmShowUnified({ ref, readOnly: true });
38
38
  if (!shown.path)
39
39
  throw new UsageError("The document has no local source file.");
40
- const db = openReadonlyExistingDatabase();
41
- let canonical;
42
- if (db) {
43
- try {
44
- const id = getEntryIdByFilePath(db, shown.path);
45
- canonical = id === undefined ? undefined : (getItemRefById(db, id) ?? undefined);
46
- }
47
- finally {
48
- closeDatabase(db);
49
- }
50
- }
40
+ const canonical = readUsableIndex((db) => {
41
+ const id = getEntryIdByFilePath(db, shown.path);
42
+ return id === undefined ? undefined : (getItemRefById(db, id) ?? undefined);
43
+ }, null);
44
+ if (canonical === null)
45
+ throw new UsageError(NO_INDEX_MESSAGE, "MISSING_REQUIRED_ARGUMENT", "akm index");
51
46
  if (!canonical)
52
47
  throw new UsageError("Only indexed documents can be edited.");
53
48
  const target = resolveBundleWriteTarget(loadConfig(), parseBundleRef(canonical).bundle);
@@ -0,0 +1,249 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * The library's topics and near-duplicates, from the stored embeddings: what `assets.graph` reports so a host can say
6
+ * what the library holds, where it overlaps, and which assets say the same thing. Nothing is embedded and nothing is
7
+ * written; the work yields to the event loop as it goes and is kept for one index build.
8
+ */
9
+ /** Fewest embedded assets topics are found for. */
10
+ const MIN_ASSETS = 8;
11
+ /** Most topics, however large the library. */
12
+ const MAX_TOPICS = 36;
13
+ /** Most vectors the topic centres are found from; the rest are only assigned to them. */
14
+ const SAMPLE = 4000;
15
+ const ROUNDS = 16;
16
+ /** Cosine at or above which two assets of one topic are reported as near-duplicates. */
17
+ export const DUPLICATE_COSINE = 0.96;
18
+ /** Most near-duplicate pairs reported, the most alike first. */
19
+ const MAX_PAIRS = 2000;
20
+ /** Topics each topic names as nearest. */
21
+ const RELATED_TOPICS = 3;
22
+ const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve));
23
+ /** How many topics for `n` embedded assets: about the square root of n/2, from 2 to {@link MAX_TOPICS}. */
24
+ export function topicCount(n) {
25
+ return Math.max(2, Math.min(MAX_TOPICS, Math.round(Math.sqrt(n / 2))));
26
+ }
27
+ let cache;
28
+ /**
29
+ * Groups the assets by meaning (spherical k-means on the unit vectors: centres from an evenly spaced sample of at most
30
+ * 4,000, seeded deterministically, so the same vectors give the same topics), names each topic by the words of its
31
+ * assets' paths and titles that are most particular to it, and finds near-duplicate pairs inside each topic. `names[i]`
32
+ * is asset i's path and title. With a `key` (the index's build), the result is reused until the key changes.
33
+ */
34
+ export async function findTopics(vectors, names, key) {
35
+ if (key !== undefined && cache?.key === key && cache.result.topic.length === vectors.length)
36
+ return cache.result;
37
+ const empty = {
38
+ topics: [],
39
+ topic: vectors.map(() => -1),
40
+ duplicates: { threshold: DUPLICATE_COSINE, a: [], b: [], cosine: [], truncated: false },
41
+ };
42
+ const dims = vectors.find((v) => v)?.length ?? 0;
43
+ const usable = [];
44
+ for (let i = 0; i < vectors.length; i++)
45
+ if (vectors[i]?.length === dims)
46
+ usable.push(i);
47
+ if (!dims || usable.length < MIN_ASSETS)
48
+ return empty;
49
+ // Unit vectors, packed: cosine is then a dot product.
50
+ const unit = new Float32Array(usable.length * dims);
51
+ usable.forEach((i, row) => {
52
+ const v = vectors[i];
53
+ let norm = 0;
54
+ for (let d = 0; d < dims; d++)
55
+ norm += v[d] * v[d];
56
+ norm = Math.sqrt(norm) || 1;
57
+ for (let d = 0; d < dims; d++)
58
+ unit[row * dims + d] = v[d] / norm;
59
+ });
60
+ const dot = (row, centre, c) => {
61
+ let sum = 0;
62
+ for (let d = 0; d < dims; d++)
63
+ sum += unit[row * dims + d] * centre[c * dims + d];
64
+ return sum;
65
+ };
66
+ const k = topicCount(usable.length);
67
+ const step = Math.max(1, usable.length / SAMPLE);
68
+ const sample = [];
69
+ for (let at = 0; at < usable.length && sample.length < SAMPLE; at += step)
70
+ sample.push(Math.floor(at));
71
+ // k-means++ from a fixed start, with a deterministic generator, so a library always gets the same topics.
72
+ let seed = 7;
73
+ const random = () => {
74
+ seed = (seed * 1664525 + 1013904223) >>> 0;
75
+ return seed / 2 ** 32;
76
+ };
77
+ const centres = new Float32Array(k * dims);
78
+ const setCentre = (c, row) => centres.set(unit.subarray(row * dims, row * dims + dims), c * dims);
79
+ setCentre(0, sample[0]);
80
+ const far = new Float64Array(sample.length).fill(Number.POSITIVE_INFINITY);
81
+ for (let c = 1; c < k; c++) {
82
+ let total = 0;
83
+ for (let s = 0; s < sample.length; s++) {
84
+ far[s] = Math.min(far[s], Math.max(0, 1 - dot(sample[s], centres, c - 1)));
85
+ total += far[s] * far[s];
86
+ }
87
+ let r = random() * total;
88
+ let chosen = sample.length - 1;
89
+ for (let s = 0; s < sample.length; s++) {
90
+ r -= far[s] * far[s];
91
+ if (r <= 0) {
92
+ chosen = s;
93
+ break;
94
+ }
95
+ }
96
+ setCentre(c, sample[chosen]);
97
+ }
98
+ const nearest = (row) => {
99
+ let best = 0;
100
+ let bestDot = Number.NEGATIVE_INFINITY;
101
+ for (let c = 0; c < k; c++) {
102
+ const value = dot(row, centres, c);
103
+ if (value > bestDot) {
104
+ bestDot = value;
105
+ best = c;
106
+ }
107
+ }
108
+ return best;
109
+ };
110
+ for (let round = 0; round < ROUNDS; round++) {
111
+ const sums = new Float64Array(k * dims);
112
+ for (const row of sample) {
113
+ const c = nearest(row);
114
+ for (let d = 0; d < dims; d++)
115
+ sums[c * dims + d] += unit[row * dims + d];
116
+ }
117
+ for (let c = 0; c < k; c++) {
118
+ let norm = 0;
119
+ for (let d = 0; d < dims; d++)
120
+ norm += sums[c * dims + d] ** 2;
121
+ norm = Math.sqrt(norm);
122
+ // A centre that lost every member keeps its place.
123
+ if (norm > 0)
124
+ for (let d = 0; d < dims; d++)
125
+ centres[c * dims + d] = sums[c * dims + d] / norm;
126
+ }
127
+ await yieldToLoop();
128
+ }
129
+ // Every asset to its nearest centre; topics are renumbered largest first, and empty ones dropped.
130
+ const raw = new Int32Array(usable.length);
131
+ for (let row = 0; row < usable.length; row++) {
132
+ raw[row] = nearest(row);
133
+ if (row % 2000 === 1999)
134
+ await yieldToLoop();
135
+ }
136
+ const sizes = new Array(k).fill(0);
137
+ for (const c of raw)
138
+ sizes[c]++;
139
+ const order = sizes
140
+ .map((size, c) => ({ size, c }))
141
+ .filter((t) => t.size > 0)
142
+ .sort((x, y) => y.size - x.size || x.c - y.c);
143
+ const renumber = new Map(order.map((t, n) => [t.c, n]));
144
+ const topic = vectors.map(() => -1);
145
+ usable.forEach((i, row) => {
146
+ topic[i] = renumber.get(raw[row]);
147
+ });
148
+ const members = order.map(() => []);
149
+ usable.forEach((i, row) => {
150
+ members[topic[i]].push(row);
151
+ });
152
+ const centreOf = (n) => order[n].c;
153
+ const topics = order.map((t, n) => ({
154
+ ...nameTopic(members[n].map((row) => names[usable[row]] ?? ""), names),
155
+ size: t.size,
156
+ related: order
157
+ .map((_, m) => {
158
+ let cosine = 0;
159
+ for (let d = 0; d < dims; d++)
160
+ cosine += centres[centreOf(n) * dims + d] * centres[centreOf(m) * dims + d];
161
+ return { topic: m, cosine: Math.round(cosine * 1000) / 1000 };
162
+ })
163
+ .filter((r) => r.topic !== n)
164
+ .sort((x, y) => y.cosine - x.cosine)
165
+ .slice(0, RELATED_TOPICS),
166
+ }));
167
+ // Near-duplicates: every pair inside a topic. Pairs across topics are not compared; two assets that alike almost
168
+ // always share a topic.
169
+ // Only the best MAX_PAIRS are kept, so a library full of copies does not hold every pair in memory: the list is cut
170
+ // back whenever it grows to eight times that, to the same top pairs a full sort would keep.
171
+ let pairs = [];
172
+ let pairCount = 0;
173
+ const byAlike = (x, y) => y[2] - x[2] || x[0] - y[0] || x[1] - y[1];
174
+ for (const rows of members) {
175
+ for (let x = 0; x < rows.length; x++) {
176
+ const p = rows[x] * dims;
177
+ for (let y = x + 1; y < rows.length; y++) {
178
+ const q = rows[y] * dims;
179
+ let cosine = 0;
180
+ for (let d = 0; d < dims; d++)
181
+ cosine += unit[p + d] * unit[q + d];
182
+ if (cosine >= DUPLICATE_COSINE) {
183
+ pairs.push([usable[rows[x]], usable[rows[y]], cosine]);
184
+ pairCount++;
185
+ if (pairs.length >= MAX_PAIRS * 8)
186
+ pairs = pairs.sort(byAlike).slice(0, MAX_PAIRS);
187
+ }
188
+ }
189
+ if (x % 200 === 199)
190
+ await yieldToLoop();
191
+ }
192
+ await yieldToLoop();
193
+ }
194
+ const kept = pairs.sort(byAlike).slice(0, MAX_PAIRS);
195
+ const result = {
196
+ topics,
197
+ topic,
198
+ duplicates: {
199
+ threshold: DUPLICATE_COSINE,
200
+ a: kept.map((p) => p[0]),
201
+ b: kept.map((p) => p[1]),
202
+ cosine: kept.map((p) => Math.round(p[2] * 10000) / 10000),
203
+ truncated: pairCount > MAX_PAIRS,
204
+ },
205
+ };
206
+ if (key !== undefined)
207
+ cache = { key, result };
208
+ return result;
209
+ }
210
+ const STOP = new Set("the and for with from into this that what when how why are was not use using about your you our its md readme index skill skills knowledge memory memories command commands agent agents script scripts workflow workflows fact facts lesson lessons references reference assets asset src lib test tests docs".split(" "));
211
+ /** The words of a path or title worth naming a topic by: lower case, three letters or more, no digits, no stop words. */
212
+ export function words(text) {
213
+ return text
214
+ .toLowerCase()
215
+ .split(/[^a-z0-9]+/)
216
+ .filter((w) => w.length >= 3 && !/\d/.test(w) && !STOP.has(w));
217
+ }
218
+ let documentFrequency;
219
+ /**
220
+ * A topic's distinctive words: those most frequent among its assets relative to how many assets of the whole library
221
+ * use them (each asset counts a word once). The label is the top two.
222
+ */
223
+ export function nameTopic(memberNames, allNames) {
224
+ if (documentFrequency?.names !== allNames) {
225
+ const counts = new Map();
226
+ for (const name of allNames)
227
+ for (const w of new Set(words(name)))
228
+ counts.set(w, (counts.get(w) ?? 0) + 1);
229
+ documentFrequency = { names: allNames, counts };
230
+ }
231
+ const inTopic = new Map();
232
+ for (const name of memberNames)
233
+ for (const w of new Set(words(name)))
234
+ inTopic.set(w, (inTopic.get(w) ?? 0) + 1);
235
+ const total = allNames.length;
236
+ const terms = [...inTopic]
237
+ // A word one asset uses names nothing.
238
+ .filter(([, n]) => n >= Math.min(2, memberNames.length))
239
+ .map(([w, n]) => ({
240
+ w,
241
+ score: (n / memberNames.length) * Math.log(total / (documentFrequency.counts.get(w) ?? 1)),
242
+ }))
243
+ // A word every asset of the library has says nothing about this topic.
244
+ .filter((t) => t.score > 0)
245
+ .sort((x, y) => y.score - x.score || (x.w < y.w ? -1 : 1))
246
+ .slice(0, 5)
247
+ .map((t) => t.w);
248
+ return { label: terms.slice(0, 2).join(" · ") || "unnamed", terms };
249
+ }
@@ -2,29 +2,50 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  /** The whole library as a graph: every indexed asset, every declared link between them, and how each is used. */
5
+ import { createHash } from "node:crypto";
6
+ import fs from "node:fs";
5
7
  import { loadConfig } from "../core/config/config.js";
8
+ import { ConfigError } from "../core/errors.js";
6
9
  import { getDbPath } from "../core/paths.js";
7
10
  import { applyEntryFilters } from "../indexer/search/db-search.js";
8
11
  import { resolveSourceEntries } from "../indexer/search/search-source.js";
9
- import { countUseByRef, usageRetentionSince } from "../indexer/usage/usage-events.js";
12
+ import { countFeedbackSignalsByRef, countUseByRef, usageRetentionSince } from "../indexer/usage/usage-events.js";
10
13
  import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
11
14
  import { getAllEntries } from "../storage/repositories/index-entries-repository.js";
12
15
  import { listDeclaredLinks } from "../storage/repositories/index-links-repository.js";
13
16
  import { getMeta } from "../storage/repositories/index-meta-repository.js";
14
17
  import { listCurrentEmbeddings } from "../storage/repositories/index-vec-repository.js";
18
+ import { findTopics } from "./graph-topics.js";
15
19
  import { withReadonlyStateDb } from "./readers.js";
16
20
  /** Types never drawn: protected files and session logs, as `assets.related` leaves them out. */
17
21
  const HIDDEN_TYPES = ["secret", "env", "session"];
18
22
  /**
19
23
  * Every asset of the enabled bundles and the declared links between them, in column form: node `i` is `nodes.ref[i]`,
20
24
  * and its type, bundle and belief state are indexes into `types`, `bundles` (largest first) and `beliefs` (-1: no state
21
- * recorded). `shows` and `searches` are read live from the user-attributed usage events within retention (`use`). A link joins two nodes by index and names its kind
22
- * by index into `kinds`. `meaning` places each node with a stored embedding in three dimensions (see
25
+ * recorded). `shows`, `searches`, `lastShownAt` and the feedback counts `positive` and `negative` are read live from the
26
+ * user-attributed usage events within retention (`use`); `modifiedAt` is the file's modification time. A link joins two
27
+ * nodes by index and names its kind by index into `kinds`. `topic` indexes `topics` (see {@link findTopics}), -1 for
28
+ * an asset with no stored embedding, and `duplicates` pairs assets of one topic whose embeddings are nearly the same. `meaning` places each node with a stored embedding in three dimensions (see
23
29
  * {@link projectEmbeddings}); it is `null` when the index has no embeddings. Nothing is written. The projection yields
24
30
  * to the event loop as it goes, so a host serving other requests keeps answering them.
25
31
  */
26
32
  export async function graph() {
27
- const empty = {
33
+ try {
34
+ return await readGraph();
35
+ }
36
+ catch (error) {
37
+ // A damaged or unreadable index file is "no usable index" to the map, not a raw SQLite error.
38
+ if (error instanceof Error && error.name === "SQLiteError")
39
+ return {
40
+ available: false,
41
+ note: `The index could not be read (${error.message}). Rebuild it with 'akm index --full'.`,
42
+ ...emptyGraph(),
43
+ };
44
+ throw error;
45
+ }
46
+ }
47
+ function emptyGraph() {
48
+ return {
28
49
  types: [],
29
50
  bundles: [],
30
51
  beliefs: [],
@@ -42,15 +63,28 @@ export async function graph() {
42
63
  belief: [],
43
64
  shows: [],
44
65
  searches: [],
66
+ lastShownAt: [],
67
+ modifiedAt: [],
68
+ positive: [],
69
+ negative: [],
70
+ topic: [],
45
71
  },
46
72
  links: { from: [], to: [], kind: [] },
47
73
  meaning: null,
74
+ topics: [],
75
+ duplicates: null,
48
76
  };
77
+ }
78
+ async function readGraph() {
79
+ const empty = emptyGraph();
49
80
  const db = openReadonlyExistingDatabase();
50
81
  if (!db)
51
82
  return { available: false, note: "No index is available. Build the AKM index first.", ...empty };
52
83
  let closed = false;
53
84
  try {
85
+ // An index still being built for the first time has its meta table but not yet its entries.
86
+ if (!db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'entries'").get())
87
+ return { available: false, note: "The index is still being built. Try again when it finishes.", ...empty };
54
88
  const sources = resolveSourceEntries(undefined, loadConfig()).filter((source) => !source.unresolved);
55
89
  const rows = applyEntryFilters(getAllEntries(db, undefined, HIDDEN_TYPES), {
56
90
  db,
@@ -84,6 +118,7 @@ export async function graph() {
84
118
  result.emptyBundles = sources.flatMap((source) => source.registryId && !perBundle.has(source.registryId) ? [source.registryId] : []);
85
119
  const since = usageRetentionSince();
86
120
  const use = withReadonlyStateDb((state) => countUseByRef(state, since), null);
121
+ const feedback = withReadonlyStateDb((state) => countFeedbackSignalsByRef(state, rows.map((row) => row.itemRef)), null);
87
122
  result.use = { since, source: "user" };
88
123
  for (const row of rows) {
89
124
  const counts = use?.get(row.itemRef);
@@ -94,6 +129,11 @@ export async function graph() {
94
129
  result.nodes.belief.push(row.entry.beliefState ? beliefCode(row.entry.beliefState) : -1);
95
130
  result.nodes.shows.push(counts?.shows ?? 0);
96
131
  result.nodes.searches.push(counts?.searches ?? 0);
132
+ result.nodes.lastShownAt.push(counts?.lastShownAt ?? null);
133
+ result.nodes.modifiedAt.push(modifiedAt(row.filePath));
134
+ const signals = feedback?.get(row.itemRef);
135
+ result.nodes.positive.push(signals?.pos ?? 0);
136
+ result.nodes.negative.push(signals?.neg ?? 0);
97
137
  }
98
138
  for (const link of listDeclaredLinks(db)) {
99
139
  const from = index.get(link.from);
@@ -114,6 +154,15 @@ export async function graph() {
114
154
  closed = true;
115
155
  const placed = await projectEmbeddings(rows.map((row) => vectors.get(row.id)), build);
116
156
  result.meaning = placed.embedded ? placed : null;
157
+ const libraryKey = createHash("sha1")
158
+ .update(rows.map((row) => row.id).join(","))
159
+ .digest("hex");
160
+ const found = await findTopics(rows.map((row) => vectors.get(row.id)), rows.map((row, i) => `${row.conceptId} ${result.nodes.title[i]}`),
161
+ // Topics are per asset: the same build read through a different set of enabled bundles is a different library.
162
+ `${build}|${libraryKey}`);
163
+ result.topics = found.topics;
164
+ result.nodes.topic = found.topic;
165
+ result.duplicates = found.topics.length ? found.duplicates : null;
117
166
  const noted = [`${rows.length} assets and ${result.links.from.length} links of the enabled bundles.`];
118
167
  if (result.unresolvedLinks)
119
168
  noted.push(`${result.unresolvedLinks} declared links are not drawn (target missing, hidden or in a bundle that is not enabled).`);
@@ -129,11 +178,26 @@ export async function graph() {
129
178
  ...result,
130
179
  };
131
180
  }
181
+ catch (error) {
182
+ // A query that fails on a damaged index is no index; permission and newer-layout errors still raise.
183
+ if (closed || error instanceof ConfigError)
184
+ throw error;
185
+ return { available: false, note: "The index could not be read. Rebuild it with `akm index`.", ...empty };
186
+ }
132
187
  finally {
133
188
  if (!closed)
134
189
  closeDatabase(db);
135
190
  }
136
191
  }
192
+ /** A file's modification time as an ISO timestamp, or null when it cannot be read. */
193
+ function modifiedAt(file) {
194
+ try {
195
+ return fs.statSync(file).mtime.toISOString();
196
+ }
197
+ catch {
198
+ return null;
199
+ }
200
+ }
137
201
  /** Fewest vectors the principal components can be found from. */
138
202
  const MIN_VECTORS = 4;
139
203
  /** The note's sentence on how many assets have a position by meaning. */