akm-cli 0.10.261124 → 0.10.261125

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/CHANGELOG.md +25 -0
  2. package/dist/cli/usage.js +4 -0
  3. package/dist/cli.js +10 -0
  4. package/dist/commands/feedback-cli.js +12 -3
  5. package/dist/commands/health/semantic-search-check.js +12 -2
  6. package/dist/commands/health.js +8 -15
  7. package/dist/commands/lint/index.js +6 -6
  8. package/dist/commands/read/knowledge.js +8 -8
  9. package/dist/commands/read/related.js +3 -9
  10. package/dist/commands/read/show.js +17 -9
  11. package/dist/commands/sources/bundle-cli.js +1 -0
  12. package/dist/commands/sources/installed-stashes.js +36 -27
  13. package/dist/core/asset/resolve-ref.js +8 -0
  14. package/dist/core/common.js +21 -0
  15. package/dist/core/endpoint-url.js +4 -4
  16. package/dist/core/events.js +8 -2
  17. package/dist/core/file-lock.js +52 -1
  18. package/dist/core/state-db.js +47 -15
  19. package/dist/core/stdio-reader-gone.js +19 -0
  20. package/dist/core/write-source.js +2 -2
  21. package/dist/indexer/index-build.js +6 -3
  22. package/dist/indexer/index-readiness.js +30 -4
  23. package/dist/indexer/indexer.js +56 -8
  24. package/dist/indexer/search/db-search.js +7 -1
  25. package/dist/indexer/search/search-source.js +23 -25
  26. package/dist/indexer/usage/usage-events.js +13 -4
  27. package/dist/indexer/vector-file.js +3 -3
  28. package/dist/registry/origin-resolve.js +2 -3
  29. package/dist/scripts/akm-migrate-node.js +136 -88
  30. package/dist/scripts/akm-migrate.js +136 -88
  31. package/dist/setup/endpoint.js +1 -1
  32. package/dist/storage/database.js +29 -36
  33. package/dist/storage/repositories/index-connection.js +44 -41
  34. package/dist/storage/repositories/index-entries-repository.js +14 -7
  35. package/dist/storage/repositories/index-entry-schema.js +7 -5
  36. package/dist/storage/repositories/index-schema.js +23 -0
  37. package/dist/storage/sqlite-read-snapshot.js +2 -2
  38. package/dist/workspace/assets.js +24 -44
  39. package/dist/workspace/bundle-files.js +2 -2
  40. package/dist/workspace/documents.js +7 -12
  41. package/dist/workspace/graph-topics.js +249 -0
  42. package/dist/workspace/graph.js +68 -4
  43. package/docs/reference/workspace.md +21 -4
  44. package/package.json +1 -1
@@ -14,7 +14,7 @@ import fs from "node:fs";
14
14
  import { ConfigError } from "../../core/errors.js";
15
15
  import { classifyPathAccess, describeInaccessiblePath } from "../../core/path-access.js";
16
16
  import { getDbPath } from "../../core/paths.js";
17
- import { warn, warnOnce } from "../../core/warn.js";
17
+ import { warnOnce } from "../../core/warn.js";
18
18
  import { openDatabase } from "../database.js";
19
19
  import { openManagedDatabase } from "../managed-db.js";
20
20
  import { SQLITE_BUSY_TIMEOUT_MS } from "../sqlite-pragmas.js";
@@ -22,28 +22,56 @@ import { openSqliteReadSnapshot, SqliteReadSnapshotUnavailableError } from "../s
22
22
  import { CANONICAL_INDEX_DB_VERSION } from "./index-entry-schema.js";
23
23
  import { ensureSchema, newerIndexLayoutError } from "./index-schema.js";
24
24
  /**
25
- * Whether `error` is SQLite reporting on-disk corruption (`SQLITE_CORRUPT`,
26
- * "database disk image is malformed") rather than a permission, lock, or
27
- * schema problem. Matched on both `code` (bun:sqlite, better-sqlite3) and
28
- * message text, since driver error shapes are not perfectly uniform.
25
+ * Whether `error` is SQLite reporting that the file is not a usable database
26
+ * (`SQLITE_CORRUPT*` "database disk image is malformed", `SQLITE_NOTADB` "file
27
+ * is not a database") rather than a permission, lock, or schema problem.
28
+ * Matched on both `code` (bun:sqlite, better-sqlite3) and message text, since
29
+ * driver error shapes are not perfectly uniform.
29
30
  */
30
- function isCorruptionError(error) {
31
+ export function isCorruptionError(error) {
31
32
  const code = error?.code;
32
- if (code === "SQLITE_CORRUPT")
33
+ if (typeof code === "string" && (code.startsWith("SQLITE_CORRUPT") || code === "SQLITE_NOTADB"))
33
34
  return true;
34
35
  const message = error instanceof Error ? error.message : String(error);
35
- return message.includes("database disk image is malformed") || message.includes("SQLITE_CORRUPT");
36
+ return (message.includes("database disk image is malformed") ||
37
+ message.includes("file is not a database") ||
38
+ message.includes("SQLITE_CORRUPT") ||
39
+ message.includes("SQLITE_NOTADB"));
36
40
  }
37
41
  /**
38
- * Open `index.db` for writing, creating and migrating it. A corrupt file is
39
- * deleted and rebuilt only when the caller says it may (`rebuildCorrupt`): the
40
- * build, which holds the rebuild lock. Any other writable opener raises the
41
- * corruption error instead of deleting a file a build or a reader may be using.
42
+ * Whether `index.db` itself is damaged: it cannot be opened, or `PRAGMA quick_check` reports a problem.
43
+ * A corruption error from a build or a read may come from another database (state.db), so callers ask
44
+ * this before deleting or blaming the index. An absent or unreadable-by-permission file is not damaged.
42
45
  */
46
+ export function indexDbIsDamaged(dbPath = getDbPath()) {
47
+ let db;
48
+ try {
49
+ db = openDatabase(dbPath, { readonly: true, create: false });
50
+ const row = db?.prepare("PRAGMA quick_check(1)").get();
51
+ return row !== undefined && row.quick_check !== "ok";
52
+ }
53
+ catch (error) {
54
+ return isCorruptionError(error);
55
+ }
56
+ finally {
57
+ db?.close();
58
+ }
59
+ }
60
+ /** Delete `index.db` and its sidecars; the build rebuilds it from the library. Best-effort per file. */
61
+ export function deleteIndexDatabase(dbPath) {
62
+ for (const suffix of ["", "-wal", "-shm"]) {
63
+ try {
64
+ fs.rmSync(`${dbPath}${suffix}`, { force: true });
65
+ }
66
+ catch {
67
+ // Best-effort cleanup; the next open still fails loudly if the file could not actually be removed.
68
+ }
69
+ }
70
+ }
71
+ /** Open `index.db` for writing, creating and migrating it. Corruption raises; only the build, which holds the rebuild lock, deletes and rebuilds. */
43
72
  export function openIndexDatabase(dbPath, options) {
44
- const resolvedPath = dbPath ?? getDbPath();
45
- const spec = {
46
- path: resolvedPath,
73
+ return openManagedDatabase({
74
+ path: dbPath ?? getDbPath(),
47
75
  init: (db) => {
48
76
  // Source update uses this narrow lifecycle seam to ATTACH state.db and
49
77
  // open its coordinator-owned outer transaction before ensureSchema or
@@ -51,32 +79,7 @@ export function openIndexDatabase(dbPath, options) {
51
79
  options?.beforeSchema?.(db);
52
80
  ensureSchema(db);
53
81
  },
54
- };
55
- try {
56
- return openManagedDatabase(spec);
57
- }
58
- catch (error) {
59
- // index.db is a derived cache, fully regenerable from the stash on disk
60
- // (see src/core/state-db.ts's "Why a separate database from index.db"
61
- // note) — so real on-disk corruption is recovered by deleting the file
62
- // and rebuilding, not by surfacing a raw SQLITE_CORRUPT to the caller or
63
- // quietly falling through to an unreadable index (#865). This is the ONE
64
- // from-scratch rebuild: an older layout is migrated in place by
65
- // ensureSchema, never dropped.
66
- if (!options?.rebuildCorrupt || !isCorruptionError(error))
67
- throw error;
68
- warn(`Index database is corrupt at ${resolvedPath} — rebuilding.`);
69
- for (const suffix of ["", "-wal", "-shm"]) {
70
- try {
71
- fs.rmSync(`${resolvedPath}${suffix}`, { force: true });
72
- }
73
- catch {
74
- // Best-effort cleanup; the retried open below still fails loudly if
75
- // the file could not actually be removed.
76
- }
77
- }
78
- return openManagedDatabase(spec);
79
- }
82
+ });
80
83
  }
81
84
  export function openExistingDatabase(dbPath) {
82
85
  // Existing-DB callers do not mutate schema or embedding metadata on open;
@@ -569,20 +569,27 @@ export function findEntryIdByRef(db, ref, bundleId) {
569
569
  export function withMdVariants(name) {
570
570
  return name.endsWith(".md") ? [name, name.slice(0, -3)] : [name, `${name}.md`];
571
571
  }
572
+ /** True when `file` is a `.md` file named by the last segment of `conceptId`, up to its `.md`. */
573
+ export function namesMarkdownFile(conceptId, file) {
574
+ const last = (conceptId.split("/").pop() ?? "").replace(/\.md$/, "");
575
+ return path.extname(file) === ".md" && path.basename(file, ".md") === last;
576
+ }
572
577
  /**
573
578
  * Current (`[bundle//]conceptId`) id lookup: match `item_ref` exactly when
574
579
  * bundle-qualified or by `//conceptId` suffix when short.
575
580
  */
576
581
  function findEntryIdByBundleRef(db, ref, bundleId) {
577
582
  const parsed = parseBundleRef(ref);
578
- const conceptVariants = withMdVariants(parsed.conceptId);
579
583
  // item_ref is the canonical stored spelling post-flip.
580
- for (const conceptId of conceptVariants) {
581
- const id = matchIdByItemRef(db, parsed.bundle, conceptId, bundleId);
582
- if (id !== undefined)
583
- return id;
584
- }
585
- return undefined;
584
+ const exact = matchIdByItemRef(db, parsed.bundle, parsed.conceptId, bundleId);
585
+ if (exact !== undefined)
586
+ return exact;
587
+ // The `.md` toggle names a markdown file by its own name: `tasks/x.md` is not a spelling of `tasks/x.yml`,
588
+ // nor `skills/x.md` of `skills/x/SKILL.md`.
589
+ const [, toggled] = withMdVariants(parsed.conceptId);
590
+ const id = matchIdByItemRef(db, parsed.bundle, toggled, bundleId);
591
+ const file = id === undefined ? undefined : getEntryFilePathById(db, id);
592
+ return file !== undefined && namesMarkdownFile(parsed.conceptId, file) ? id : undefined;
586
593
  }
587
594
  /**
588
595
  * Match a row `id` by `item_ref`: exact `bundle//conceptId` when `bundle` is
@@ -142,14 +142,16 @@ export function retiredEntryColumns(db) {
142
142
  return RETIRED_ENTRY_COLUMNS.filter((column) => present.has(column));
143
143
  }
144
144
  /**
145
- * Whether an index database has an `entries` table this release can read. A
146
- * fresh or empty file has none; an index older than layout 21 has one keyed
147
- * by columns this release no longer reads (or lacks the ones it does), and
148
- * serves nothing until the next `akm index` recreates it.
145
+ * Whether an index database has an `entries` table this release can read,
146
+ * with its full-text table. A fresh or empty file has none; an index older
147
+ * than layout 21 has one keyed by columns this release no longer reads (or
148
+ * lacks the ones it does); a file whose schema was cut short has entries
149
+ * without `entries_fts`. Each serves nothing until the next `akm index`.
149
150
  */
150
151
  export function hasCurrentEntriesTable(db) {
151
152
  try {
152
- return missingEntryColumns(db).length === 0 && retiredEntryColumns(db).length === 0;
153
+ const fts = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'entries_fts'").get();
154
+ return Boolean(fts) && missingEntryColumns(db).length === 0 && retiredEntryColumns(db).length === 0;
153
155
  }
154
156
  catch {
155
157
  return false;
@@ -205,7 +205,30 @@ function ensureColumn(db, table, column, type) {
205
205
  db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${type}`);
206
206
  return true;
207
207
  }
208
+ /**
209
+ * Bring `index.db` to the current layout in one transaction, so no reader or
210
+ * crash can observe a half-migrated file (entries without their FTS table).
211
+ * One `BEGIN IMMEDIATE`, waiting out the connection's busy timeout once: a
212
+ * busy index surfaces to the caller (the build reports it as contention)
213
+ * rather than being retried here. Joins a transaction the caller already holds.
214
+ */
208
215
  export function ensureSchema(db) {
216
+ if (db.inTransaction) {
217
+ applySchema(db);
218
+ return;
219
+ }
220
+ db.exec("BEGIN IMMEDIATE");
221
+ try {
222
+ applySchema(db);
223
+ db.exec("COMMIT");
224
+ }
225
+ catch (error) {
226
+ if (db.inTransaction)
227
+ db.exec("ROLLBACK");
228
+ throw error;
229
+ }
230
+ }
231
+ function applySchema(db) {
209
232
  db.exec(`
210
233
  CREATE TABLE IF NOT EXISTS index_meta (
211
234
  key TEXT PRIMARY KEY,
@@ -19,7 +19,7 @@ import fs from "node:fs";
19
19
  import os from "node:os";
20
20
  import path from "node:path";
21
21
  import { sleepSync } from "../runtime.js";
22
- import { openDatabaseFinalizing } from "./database.js";
22
+ import { openDatabase } from "./database.js";
23
23
  const SNAPSHOT_MAX_ATTEMPTS = 8;
24
24
  const SNAPSHOT_BACKOFF_MS = 25;
25
25
  export class SqliteReadSnapshotUnavailableError extends Error {
@@ -151,7 +151,7 @@ export function openSqliteReadSnapshot(dbPath) {
151
151
  let db;
152
152
  try {
153
153
  copySqliteSnapshot(dbPath, snapshotPath);
154
- db = openDatabaseFinalizing(snapshotPath, { readonly: true, create: false });
154
+ db = openDatabase(snapshotPath, { readonly: true, create: false });
155
155
  const closeSnapshot = db.close.bind(db);
156
156
  let closed = false;
157
157
  const cleanup = (bestEffort) => {
@@ -13,12 +13,11 @@ import { loadConfig } from "../core/config/config.js";
13
13
  import { resolveRefOwner } from "../core/env-secret-ref.js";
14
14
  import { UsageError } from "../core/errors.js";
15
15
  import { includedTypes, includesProposed } from "../core/include.js";
16
- import { getDbPath } from "../core/paths.js";
17
16
  import { projectView, resolveProject } from "../core/projects.js";
17
+ import { readUsableIndex } from "../indexer/index-readiness.js";
18
18
  import { resolveReadSources } from "../indexer/read-preflight.js";
19
19
  import { countBrowseEntries } from "../indexer/search/db-search.js";
20
20
  import { findSourceForPath, resolveSourceEntries } from "../indexer/search/search-source.js";
21
- import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
22
21
  import { getEntryIdByFilePath, getItemRefById } from "../storage/repositories/index-entries-repository.js";
23
22
  import { countFeedbackBySignal, listFeedbackEventRows } from "../storage/repositories/metrics-repository.js";
24
23
  import { assetImprove } from "./asset-improve.js";
@@ -81,16 +80,9 @@ export function projectOverview(input) {
81
80
  ? local.map((source) => ({ name: source.registryId ?? source.path, source }))
82
81
  : [project.from].flat().map((from) => ({ name: from, source: narrowSourcesToName(local, [from])[0] }));
83
82
  const readable = selected.flatMap((bundle) => (bundle.source ? [bundle.source] : []));
84
- let counts;
85
- const db = readable.length > 0 ? openReadonlyExistingDatabase(getDbPath()) : undefined;
86
- if (db) {
87
- try {
88
- counts = countBrowseEntries(db, readable, project.filters, (config.search?.defaultExcludeTypes ?? ["session"]).filter((type) => !includedTypes(project.include).includes(type)), includesProposed(project.include));
89
- }
90
- finally {
91
- closeDatabase(db);
92
- }
93
- }
83
+ const counts = readable.length > 0
84
+ ? readUsableIndex((db) => countBrowseEntries(db, readable, project.filters, (config.search?.defaultExcludeTypes ?? ["session"]).filter((type) => !includedTypes(project.include).includes(type)), includesProposed(project.include)), undefined)
85
+ : undefined;
94
86
  const writeName = project.write.bundle ?? config.defaultWriteTarget ?? automaticWriteTarget(config)?.name ?? null;
95
87
  const writeSource = writeName === null ? undefined : local.find((source) => source.registryId === writeName);
96
88
  return {
@@ -121,16 +113,10 @@ export function projectOverview(input) {
121
113
  }
122
114
  /** The durable `bundle//conceptId` of the indexed asset at `filePath`, the key feedback rows use. */
123
115
  function durableRefForPath(filePath) {
124
- const db = openReadonlyExistingDatabase(getDbPath());
125
- if (!db)
126
- return null;
127
- try {
116
+ return readUsableIndex((db) => {
128
117
  const id = getEntryIdByFilePath(db, filePath);
129
118
  return id === undefined ? null : getItemRefById(db, id);
130
- }
131
- finally {
132
- closeDatabase(db);
133
- }
119
+ }, null);
134
120
  }
135
121
  function feedbackFor(durableRef) {
136
122
  const empty = { counts: {}, history: [] };
@@ -174,32 +160,26 @@ export async function show(input) {
174
160
  const links = {};
175
161
  if (shown.path && durableRef) {
176
162
  const owner = parseBundleRef(durableRef).bundle;
177
- const db = openReadonlyExistingDatabase(getDbPath());
178
- if (db) {
179
- try {
180
- for (const raw of markdownLinkTargets(content, false)) {
181
- if (/^(?:[a-z][a-z0-9+.-]*:|\/\/|#)/i.test(raw))
182
- continue;
183
- const [file, fragment] = raw.split("#", 2);
184
- if (!file?.toLowerCase().endsWith(".md") || file.startsWith("/"))
185
- continue;
186
- let decoded;
187
- try {
188
- decoded = decodeURIComponent(file);
189
- }
190
- catch {
191
- continue;
192
- }
193
- const id = getEntryIdByFilePath(db, path.resolve(path.dirname(shown.path), decoded));
194
- const target = id === undefined ? undefined : getItemRefById(db, id);
195
- if (target && parseBundleRef(target).bundle === owner)
196
- links[raw] = target + (fragment ? `#${fragment}` : "");
163
+ readUsableIndex((db) => {
164
+ for (const raw of markdownLinkTargets(content, false)) {
165
+ if (/^(?:[a-z][a-z0-9+.-]*:|\/\/|#)/i.test(raw))
166
+ continue;
167
+ const [file, fragment] = raw.split("#", 2);
168
+ if (!file?.toLowerCase().endsWith(".md") || file.startsWith("/"))
169
+ continue;
170
+ let decoded;
171
+ try {
172
+ decoded = decodeURIComponent(file);
197
173
  }
174
+ catch {
175
+ continue;
176
+ }
177
+ const id = getEntryIdByFilePath(db, path.resolve(path.dirname(shown.path), decoded));
178
+ const target = id === undefined ? undefined : getItemRefById(db, id);
179
+ if (target && parseBundleRef(target).bundle === owner)
180
+ links[raw] = target + (fragment ? `#${fragment}` : "");
198
181
  }
199
- finally {
200
- closeDatabase(db);
201
- }
202
- }
182
+ }, undefined);
203
183
  }
204
184
  return {
205
185
  redacted: false,
@@ -3,7 +3,7 @@
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import fs from "node:fs";
5
5
  import path from "node:path";
6
- import { isWithin, safeRealpath } from "../core/common.js";
6
+ import { isWithin, relativeToRoot, safeRealpath } from "../core/common.js";
7
7
  import { NotFoundError, UsageError } from "../core/errors.js";
8
8
  import { findSourceForPath, resolveSourceEntries } from "../indexer/search/search-source.js";
9
9
  import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
@@ -14,7 +14,7 @@ export function bundleFileLocation(filePath) {
14
14
  const source = findSourceForPath(filePath, resolveSourceEntries());
15
15
  if (!source?.registryId || source.unresolved || !isWithin(filePath, source.path))
16
16
  return null;
17
- const relative = path.relative(source.path, filePath);
17
+ const relative = relativeToRoot(filePath, source.path) ?? path.relative(source.path, filePath);
18
18
  return {
19
19
  bundle: source.registryId,
20
20
  path: portable(relative),
@@ -24,7 +24,7 @@ import { appendEvent } from "../core/events.js";
24
24
  import { resolveBundleWriteTarget } from "../core/mutation-target.js";
25
25
  import { warn } from "../core/warn.js";
26
26
  import { assertAkmAssetWrite } from "../core/write-source.js";
27
- import { closeDatabase, openReadonlyExistingDatabase } from "../storage/repositories/index-connection.js";
27
+ import { NO_INDEX_MESSAGE, readUsableIndex } from "../indexer/index-readiness.js";
28
28
  import { getEntryIdByFilePath, getItemRefById } from "../storage/repositories/index-entries-repository.js";
29
29
  import { interactive } from "./lease.js";
30
30
  import { redactUrlsInText } from "./redact.js";
@@ -37,17 +37,12 @@ async function resolveWritableIndexedAsset(ref) {
37
37
  const shown = await akmShowUnified({ ref, readOnly: true });
38
38
  if (!shown.path)
39
39
  throw new UsageError("The document has no local source file.");
40
- const db = openReadonlyExistingDatabase();
41
- let canonical;
42
- if (db) {
43
- try {
44
- const id = getEntryIdByFilePath(db, shown.path);
45
- canonical = id === undefined ? undefined : (getItemRefById(db, id) ?? undefined);
46
- }
47
- finally {
48
- closeDatabase(db);
49
- }
50
- }
40
+ const canonical = readUsableIndex((db) => {
41
+ const id = getEntryIdByFilePath(db, shown.path);
42
+ return id === undefined ? undefined : (getItemRefById(db, id) ?? undefined);
43
+ }, null);
44
+ if (canonical === null)
45
+ throw new UsageError(NO_INDEX_MESSAGE, "MISSING_REQUIRED_ARGUMENT", "akm index");
51
46
  if (!canonical)
52
47
  throw new UsageError("Only indexed documents can be edited.");
53
48
  const target = resolveBundleWriteTarget(loadConfig(), parseBundleRef(canonical).bundle);
@@ -0,0 +1,249 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * The library's topics and near-duplicates, from the stored embeddings: what `assets.graph` reports so a host can say
6
+ * what the library holds, where it overlaps, and which assets say the same thing. Nothing is embedded and nothing is
7
+ * written; the work yields to the event loop as it goes and is kept for one index build.
8
+ */
9
+ /** Fewest embedded assets topics are found for. */
10
+ const MIN_ASSETS = 8;
11
+ /** Most topics, however large the library. */
12
+ const MAX_TOPICS = 36;
13
+ /** Most vectors the topic centres are found from; the rest are only assigned to them. */
14
+ const SAMPLE = 4000;
15
+ const ROUNDS = 16;
16
+ /** Cosine at or above which two assets of one topic are reported as near-duplicates. */
17
+ export const DUPLICATE_COSINE = 0.96;
18
+ /** Most near-duplicate pairs reported, the most alike first. */
19
+ const MAX_PAIRS = 2000;
20
+ /** Topics each topic names as nearest. */
21
+ const RELATED_TOPICS = 3;
22
+ const yieldToLoop = () => new Promise((resolve) => setImmediate(resolve));
23
+ /** How many topics for `n` embedded assets: about the square root of n/2, from 2 to {@link MAX_TOPICS}. */
24
+ export function topicCount(n) {
25
+ return Math.max(2, Math.min(MAX_TOPICS, Math.round(Math.sqrt(n / 2))));
26
+ }
27
+ let cache;
28
+ /**
29
+ * Groups the assets by meaning (spherical k-means on the unit vectors: centres from an evenly spaced sample of at most
30
+ * 4,000, seeded deterministically, so the same vectors give the same topics), names each topic by the words of its
31
+ * assets' paths and titles that are most particular to it, and finds near-duplicate pairs inside each topic. `names[i]`
32
+ * is asset i's path and title. With a `key` (the index's build), the result is reused until the key changes.
33
+ */
34
+ export async function findTopics(vectors, names, key) {
35
+ if (key !== undefined && cache?.key === key && cache.result.topic.length === vectors.length)
36
+ return cache.result;
37
+ const empty = {
38
+ topics: [],
39
+ topic: vectors.map(() => -1),
40
+ duplicates: { threshold: DUPLICATE_COSINE, a: [], b: [], cosine: [], truncated: false },
41
+ };
42
+ const dims = vectors.find((v) => v)?.length ?? 0;
43
+ const usable = [];
44
+ for (let i = 0; i < vectors.length; i++)
45
+ if (vectors[i]?.length === dims)
46
+ usable.push(i);
47
+ if (!dims || usable.length < MIN_ASSETS)
48
+ return empty;
49
+ // Unit vectors, packed: cosine is then a dot product.
50
+ const unit = new Float32Array(usable.length * dims);
51
+ usable.forEach((i, row) => {
52
+ const v = vectors[i];
53
+ let norm = 0;
54
+ for (let d = 0; d < dims; d++)
55
+ norm += v[d] * v[d];
56
+ norm = Math.sqrt(norm) || 1;
57
+ for (let d = 0; d < dims; d++)
58
+ unit[row * dims + d] = v[d] / norm;
59
+ });
60
+ const dot = (row, centre, c) => {
61
+ let sum = 0;
62
+ for (let d = 0; d < dims; d++)
63
+ sum += unit[row * dims + d] * centre[c * dims + d];
64
+ return sum;
65
+ };
66
+ const k = topicCount(usable.length);
67
+ const step = Math.max(1, usable.length / SAMPLE);
68
+ const sample = [];
69
+ for (let at = 0; at < usable.length && sample.length < SAMPLE; at += step)
70
+ sample.push(Math.floor(at));
71
+ // k-means++ from a fixed start, with a deterministic generator, so a library always gets the same topics.
72
+ let seed = 7;
73
+ const random = () => {
74
+ seed = (seed * 1664525 + 1013904223) >>> 0;
75
+ return seed / 2 ** 32;
76
+ };
77
+ const centres = new Float32Array(k * dims);
78
+ const setCentre = (c, row) => centres.set(unit.subarray(row * dims, row * dims + dims), c * dims);
79
+ setCentre(0, sample[0]);
80
+ const far = new Float64Array(sample.length).fill(Number.POSITIVE_INFINITY);
81
+ for (let c = 1; c < k; c++) {
82
+ let total = 0;
83
+ for (let s = 0; s < sample.length; s++) {
84
+ far[s] = Math.min(far[s], Math.max(0, 1 - dot(sample[s], centres, c - 1)));
85
+ total += far[s] * far[s];
86
+ }
87
+ let r = random() * total;
88
+ let chosen = sample.length - 1;
89
+ for (let s = 0; s < sample.length; s++) {
90
+ r -= far[s] * far[s];
91
+ if (r <= 0) {
92
+ chosen = s;
93
+ break;
94
+ }
95
+ }
96
+ setCentre(c, sample[chosen]);
97
+ }
98
+ const nearest = (row) => {
99
+ let best = 0;
100
+ let bestDot = Number.NEGATIVE_INFINITY;
101
+ for (let c = 0; c < k; c++) {
102
+ const value = dot(row, centres, c);
103
+ if (value > bestDot) {
104
+ bestDot = value;
105
+ best = c;
106
+ }
107
+ }
108
+ return best;
109
+ };
110
+ for (let round = 0; round < ROUNDS; round++) {
111
+ const sums = new Float64Array(k * dims);
112
+ for (const row of sample) {
113
+ const c = nearest(row);
114
+ for (let d = 0; d < dims; d++)
115
+ sums[c * dims + d] += unit[row * dims + d];
116
+ }
117
+ for (let c = 0; c < k; c++) {
118
+ let norm = 0;
119
+ for (let d = 0; d < dims; d++)
120
+ norm += sums[c * dims + d] ** 2;
121
+ norm = Math.sqrt(norm);
122
+ // A centre that lost every member keeps its place.
123
+ if (norm > 0)
124
+ for (let d = 0; d < dims; d++)
125
+ centres[c * dims + d] = sums[c * dims + d] / norm;
126
+ }
127
+ await yieldToLoop();
128
+ }
129
+ // Every asset to its nearest centre; topics are renumbered largest first, and empty ones dropped.
130
+ const raw = new Int32Array(usable.length);
131
+ for (let row = 0; row < usable.length; row++) {
132
+ raw[row] = nearest(row);
133
+ if (row % 2000 === 1999)
134
+ await yieldToLoop();
135
+ }
136
+ const sizes = new Array(k).fill(0);
137
+ for (const c of raw)
138
+ sizes[c]++;
139
+ const order = sizes
140
+ .map((size, c) => ({ size, c }))
141
+ .filter((t) => t.size > 0)
142
+ .sort((x, y) => y.size - x.size || x.c - y.c);
143
+ const renumber = new Map(order.map((t, n) => [t.c, n]));
144
+ const topic = vectors.map(() => -1);
145
+ usable.forEach((i, row) => {
146
+ topic[i] = renumber.get(raw[row]);
147
+ });
148
+ const members = order.map(() => []);
149
+ usable.forEach((i, row) => {
150
+ members[topic[i]].push(row);
151
+ });
152
+ const centreOf = (n) => order[n].c;
153
+ const topics = order.map((t, n) => ({
154
+ ...nameTopic(members[n].map((row) => names[usable[row]] ?? ""), names),
155
+ size: t.size,
156
+ related: order
157
+ .map((_, m) => {
158
+ let cosine = 0;
159
+ for (let d = 0; d < dims; d++)
160
+ cosine += centres[centreOf(n) * dims + d] * centres[centreOf(m) * dims + d];
161
+ return { topic: m, cosine: Math.round(cosine * 1000) / 1000 };
162
+ })
163
+ .filter((r) => r.topic !== n)
164
+ .sort((x, y) => y.cosine - x.cosine)
165
+ .slice(0, RELATED_TOPICS),
166
+ }));
167
+ // Near-duplicates: every pair inside a topic. Pairs across topics are not compared; two assets that alike almost
168
+ // always share a topic.
169
+ // Only the best MAX_PAIRS are kept, so a library full of copies does not hold every pair in memory: the list is cut
170
+ // back whenever it grows to eight times that, to the same top pairs a full sort would keep.
171
+ let pairs = [];
172
+ let pairCount = 0;
173
+ const byAlike = (x, y) => y[2] - x[2] || x[0] - y[0] || x[1] - y[1];
174
+ for (const rows of members) {
175
+ for (let x = 0; x < rows.length; x++) {
176
+ const p = rows[x] * dims;
177
+ for (let y = x + 1; y < rows.length; y++) {
178
+ const q = rows[y] * dims;
179
+ let cosine = 0;
180
+ for (let d = 0; d < dims; d++)
181
+ cosine += unit[p + d] * unit[q + d];
182
+ if (cosine >= DUPLICATE_COSINE) {
183
+ pairs.push([usable[rows[x]], usable[rows[y]], cosine]);
184
+ pairCount++;
185
+ if (pairs.length >= MAX_PAIRS * 8)
186
+ pairs = pairs.sort(byAlike).slice(0, MAX_PAIRS);
187
+ }
188
+ }
189
+ if (x % 200 === 199)
190
+ await yieldToLoop();
191
+ }
192
+ await yieldToLoop();
193
+ }
194
+ const kept = pairs.sort(byAlike).slice(0, MAX_PAIRS);
195
+ const result = {
196
+ topics,
197
+ topic,
198
+ duplicates: {
199
+ threshold: DUPLICATE_COSINE,
200
+ a: kept.map((p) => p[0]),
201
+ b: kept.map((p) => p[1]),
202
+ cosine: kept.map((p) => Math.round(p[2] * 10000) / 10000),
203
+ truncated: pairCount > MAX_PAIRS,
204
+ },
205
+ };
206
+ if (key !== undefined)
207
+ cache = { key, result };
208
+ return result;
209
+ }
210
+ const STOP = new Set("the and for with from into this that what when how why are was not use using about your you our its md readme index skill skills knowledge memory memories command commands agent agents script scripts workflow workflows fact facts lesson lessons references reference assets asset src lib test tests docs".split(" "));
211
+ /** The words of a path or title worth naming a topic by: lower case, three letters or more, no digits, no stop words. */
212
+ export function words(text) {
213
+ return text
214
+ .toLowerCase()
215
+ .split(/[^a-z0-9]+/)
216
+ .filter((w) => w.length >= 3 && !/\d/.test(w) && !STOP.has(w));
217
+ }
218
+ let documentFrequency;
219
+ /**
220
+ * A topic's distinctive words: those most frequent among its assets relative to how many assets of the whole library
221
+ * use them (each asset counts a word once). The label is the top two.
222
+ */
223
+ export function nameTopic(memberNames, allNames) {
224
+ if (documentFrequency?.names !== allNames) {
225
+ const counts = new Map();
226
+ for (const name of allNames)
227
+ for (const w of new Set(words(name)))
228
+ counts.set(w, (counts.get(w) ?? 0) + 1);
229
+ documentFrequency = { names: allNames, counts };
230
+ }
231
+ const inTopic = new Map();
232
+ for (const name of memberNames)
233
+ for (const w of new Set(words(name)))
234
+ inTopic.set(w, (inTopic.get(w) ?? 0) + 1);
235
+ const total = allNames.length;
236
+ const terms = [...inTopic]
237
+ // A word one asset uses names nothing.
238
+ .filter(([, n]) => n >= Math.min(2, memberNames.length))
239
+ .map(([w, n]) => ({
240
+ w,
241
+ score: (n / memberNames.length) * Math.log(total / (documentFrequency.counts.get(w) ?? 1)),
242
+ }))
243
+ // A word every asset of the library has says nothing about this topic.
244
+ .filter((t) => t.score > 0)
245
+ .sort((x, y) => y.score - x.score || (x.w < y.w ? -1 : 1))
246
+ .slice(0, 5)
247
+ .map((t) => t.w);
248
+ return { label: terms.slice(0, 2).join(" · ") || "unnamed", terms };
249
+ }