akm-cli 0.10.261124 → 0.10.261125-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/CHANGELOG.md +21 -0
  2. package/dist/cli.js +10 -0
  3. package/dist/commands/feedback-cli.js +12 -3
  4. package/dist/commands/health/semantic-search-check.js +12 -2
  5. package/dist/commands/health.js +8 -15
  6. package/dist/commands/lint/index.js +6 -6
  7. package/dist/commands/read/knowledge.js +8 -8
  8. package/dist/commands/read/related.js +3 -9
  9. package/dist/commands/read/show.js +17 -9
  10. package/dist/commands/sources/bundle-cli.js +1 -0
  11. package/dist/commands/sources/installed-stashes.js +36 -27
  12. package/dist/core/asset/resolve-ref.js +8 -0
  13. package/dist/core/common.js +21 -0
  14. package/dist/core/endpoint-url.js +4 -4
  15. package/dist/core/events.js +8 -2
  16. package/dist/core/file-lock.js +52 -1
  17. package/dist/core/state-db.js +47 -15
  18. package/dist/core/stdio-reader-gone.js +19 -0
  19. package/dist/core/write-source.js +2 -2
  20. package/dist/indexer/index-build.js +6 -3
  21. package/dist/indexer/index-readiness.js +30 -4
  22. package/dist/indexer/indexer.js +56 -8
  23. package/dist/indexer/search/db-search.js +7 -1
  24. package/dist/indexer/search/search-source.js +23 -25
  25. package/dist/indexer/usage/usage-events.js +13 -4
  26. package/dist/registry/origin-resolve.js +2 -3
  27. package/dist/scripts/akm-migrate-node.js +118 -72
  28. package/dist/scripts/akm-migrate.js +118 -72
  29. package/dist/setup/endpoint.js +1 -1
  30. package/dist/storage/repositories/index-connection.js +44 -41
  31. package/dist/storage/repositories/index-entries-repository.js +14 -7
  32. package/dist/storage/repositories/index-entry-schema.js +7 -5
  33. package/dist/storage/repositories/index-schema.js +23 -0
  34. package/dist/workspace/assets.js +24 -44
  35. package/dist/workspace/bundle-files.js +2 -2
  36. package/dist/workspace/documents.js +7 -12
  37. package/dist/workspace/graph-topics.js +249 -0
  38. package/dist/workspace/graph.js +68 -4
  39. package/docs/reference/workspace.md +21 -4
  40. package/package.json +1 -1
@@ -5,7 +5,7 @@ import fs from "node:fs";
5
5
  import path from "node:path";
6
6
  import { detectAdapterId } from "../core/adapter/detect-adapter.js";
7
7
  import { adapterForId } from "../core/adapter/registry.js";
8
- import { compareCodePoints, isHttpUrl } from "../core/common.js";
8
+ import { compareCodePoints, isHttpUrl, safeRealpath } from "../core/common.js";
9
9
  import { appendEvent } from "../core/events.js";
10
10
  import { classifyPathAccess, describeInaccessiblePath } from "../core/path-access.js";
11
11
  import { getDbPath } from "../core/paths.js";
@@ -14,7 +14,7 @@ import { isVerbose, warn, warnOnce, warnVerbose } from "../core/warn.js";
14
14
  import { localEmbeddingRuntime } from "../llm/embedder.js";
15
15
  import { effectiveEmbedding } from "../llm/embedders/connection.js";
16
16
  import { resolveSourcesForOrigin } from "../registry/origin-resolve.js";
17
- import { closeDatabase, openExistingDatabase, openIndexDatabase, openReadonlyExistingDatabase, } from "../storage/repositories/index-connection.js";
17
+ import { closeDatabase, deleteIndexDatabase, indexDbIsDamaged, isCorruptionError, openExistingDatabase, openIndexDatabase, openReadonlyExistingDatabase, } from "../storage/repositories/index-connection.js";
18
18
  import { deleteEntriesByBundle, deleteEntriesByDirAndBundle, deleteEntriesByDirExceptRefs, deleteEntriesByIds, deleteEntriesOutsideLibrary, deleteUsageEventsForEntries, findEntryIdByRef, getAllEntries, getEmbeddableEntryCount, getEntryCount, getIndexedBundleIdsByDir, getIndexedDirPathsByBundleId, relinkUsageEvents, upsertEntry, } from "../storage/repositories/index-entries-repository.js";
19
19
  import { rebuildFtsIfTotalsStale } from "../storage/repositories/index-fts-repository.js";
20
20
  import { clearStaleCacheEntries } from "../storage/repositories/index-llm-cache-repository.js";
@@ -254,12 +254,31 @@ export function _setAkmIndexForTests(fake) {
254
254
  export async function akmIndex(options) {
255
255
  try {
256
256
  const override = akmIndexOverride;
257
- return override ? await override(options) : await akmIndexReal(options);
257
+ return override ? await override(options) : await akmIndexRecoveringCorruption(options);
258
258
  }
259
259
  catch (error) {
260
260
  throw reclassifyIndexDbContention(error);
261
261
  }
262
262
  }
263
+ /**
264
+ * index.db is a derived cache, fully regenerable from the library, and the
265
+ * build holds the rebuild lock: damage to index.db found anywhere in a build
266
+ * (the header, the schema, an interior page hit mid-walk) deletes the file and
267
+ * builds again once. A corruption error from another database leaves it alone. An older layout is migrated in place, never dropped.
268
+ */
269
+ async function akmIndexRecoveringCorruption(options) {
270
+ try {
271
+ return await akmIndexReal(options);
272
+ }
273
+ catch (error) {
274
+ const dbPath = getDbPath();
275
+ if (!isCorruptionError(error) || !indexDbIsDamaged(dbPath))
276
+ throw error;
277
+ warn(`Index database is corrupt at ${dbPath} — rebuilding.`);
278
+ deleteIndexDatabase(dbPath);
279
+ return await akmIndexReal(options);
280
+ }
281
+ }
263
282
  let indexTransactionHookForTests;
264
283
  /**
265
284
  * TEST-ONLY. Observe the in-flight reindex transaction; `undefined` restores.
@@ -420,14 +439,14 @@ async function akmIndexReal(options) {
420
439
  });
421
440
  const t0 = Date.now();
422
441
  const dbPath = getDbPath();
423
- // The build holds the rebuild lock, so it alone may delete and rebuild a corrupt file.
424
- const db = openIndexDatabase(dbPath, { rebuildCorrupt: true });
442
+ const db = openIndexDatabase(dbPath);
425
443
  try {
426
444
  // `--full` folds into `isIncremental`: a full run drains every directory
427
445
  // through the same diff-persist path as an incremental one.
446
+ canonicalizeStoredPaths(db);
428
447
  const prevStashDir = getMeta(db, "stashDir");
429
448
  const prevBuiltAt = getMeta(db, "builtAt");
430
- const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
449
+ const isIncremental = !full && servesLibrary(prevStashDir, stashDir) && !!prevBuiltAt;
431
450
  const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
432
451
  const mode = isIncremental ? "incremental" : "full";
433
452
  onProgress({
@@ -946,6 +965,35 @@ function requiresWorkflowSourcePreflight(ctxs) {
946
965
  }
947
966
  });
948
967
  }
968
+ /**
969
+ * An index built before sources were keyed by real path stores the library under another spelling of
970
+ * its directories. Rewriting those paths in place keeps its rows, and with them their embeddings,
971
+ * which a delete and re-insert under the real spelling would drop.
972
+ */
973
+ function canonicalizeStoredPaths(db) {
974
+ const stored = [getMeta(db, "stashDir"), ...JSON.parse(getMeta(db, "stashDirs") ?? "[]")];
975
+ const moves = [...new Set(stored.filter((dir) => dir !== undefined))]
976
+ .map((dir) => ({ from: dir, to: safeRealpath(dir) }))
977
+ .filter(({ from, to }) => from !== to);
978
+ if (moves.length === 0)
979
+ return;
980
+ const rewrite = (table, column, exact) => db.prepare(`UPDATE OR IGNORE ${table} SET ${column} = ?1 || substr(${column}, ?2) WHERE substr(${column}, 1, ?3) = ?4${exact ? ` OR ${column} = ?5` : ""}`);
981
+ db.transaction(() => {
982
+ for (const { from, to } of moves) {
983
+ const prefix = `${from}${path.sep}`;
984
+ rewrite("entries", "file_path", false).run(to, from.length + 1, prefix.length, prefix);
985
+ rewrite("index_dir_state", "dir_path", true).run(to, from.length + 1, prefix.length, prefix, from);
986
+ }
987
+ setMeta(db, "stashDirs", JSON.stringify(JSON.parse(getMeta(db, "stashDirs") ?? "[]").map(safeRealpath)));
988
+ const stashDir = getMeta(db, "stashDir");
989
+ if (stashDir !== undefined)
990
+ setMeta(db, "stashDir", safeRealpath(stashDir));
991
+ })();
992
+ }
993
+ /** Whether the stored library marker names `stashDir`: the same directory, however either is spelled. */
994
+ function servesLibrary(stored, stashDir) {
995
+ return stored !== undefined && safeRealpath(stored) === safeRealpath(stashDir);
996
+ }
949
997
  /**
950
998
  * Phase 2 (sync): write all pre-generated scan records inside a single transaction.
951
999
  *
@@ -964,11 +1012,11 @@ function persistDirRecords(db, dirRecords, warnings, bundleByRoot, maxInputToken
964
1012
  const persist = () => {
965
1013
  // Entries are about to describe another library: the old watermark is not theirs, and neither are rows outside
966
1014
  // its directories. They go in this transaction, so the marker never names a library whose index serves another's.
967
- if (getMeta(db, "stashDir") !== library.stashDir) {
1015
+ if (!servesLibrary(getMeta(db, "stashDir"), library.stashDir)) {
968
1016
  deleteMeta(db, "builtAt");
969
1017
  addDeletedEntries(deletedEntries, deleteEntriesOutsideLibrary(db, library.sourceDirs, library.unresolvedBundleIds));
970
1018
  }
971
- setMeta(db, "stashDir", library.stashDir);
1019
+ setMeta(db, "stashDir", safeRealpath(library.stashDir));
972
1020
  setMeta(db, "stashDirs", JSON.stringify(library.sourceDirs));
973
1021
  for (const { dirPath, currentStashDir, files, fingerprint, stash, skip, reason, persistedRowCount, hashByFile, conceptIdByFile, indexVariant, remove, } of dirRecords) {
974
1022
  const bundle = bundleByRoot.get(path.resolve(currentStashDir));
@@ -21,7 +21,7 @@ import { embed } from "../../llm/embedder.js";
21
21
  import { effectiveEmbedding } from "../../llm/embedders/connection.js";
22
22
  import { applyEmbeddingTemplate, resolveEmbeddingProfile } from "../../llm/embedders/profile.js";
23
23
  import { normalizeEmbeddingEndpoint } from "../../llm/embedders/remote.js";
24
- import { closeDatabase } from "../../storage/repositories/index-connection.js";
24
+ import { closeDatabase, indexDbIsDamaged, isCorruptionError } from "../../storage/repositories/index-connection.js";
25
25
  import { getAllEntries, getBaseBeliefStatesForDerivedTwins, getEntriesByIds, getEntryById, getEntryRefsAndTypes, } from "../../storage/repositories/index-entries-repository.js";
26
26
  import { searchFts } from "../../storage/repositories/index-fts-repository.js";
27
27
  import { getMeta } from "../../storage/repositories/index-meta-repository.js";
@@ -128,6 +128,12 @@ export async function searchLocal(input) {
128
128
  mode,
129
129
  };
130
130
  }
131
+ catch (error) {
132
+ // A damaged page is an index that cannot serve; `akm index` rebuilds it.
133
+ if (!isCorruptionError(error) || !indexDbIsDamaged())
134
+ throw error;
135
+ return { hits: [], warnings: [...warnings, noIndexWarning()], mode: "keyword" };
136
+ }
131
137
  finally {
132
138
  closeDatabase(db);
133
139
  }
@@ -3,7 +3,7 @@
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import fs from "node:fs";
5
5
  import path from "node:path";
6
- import { isWithin, resolveStashDir } from "../../core/common.js";
6
+ import { isWithin, relativeToRoot, resolveStashDir, safeRealpath } from "../../core/common.js";
7
7
  import { bundleComponentConfig, bundleKeyForContentRoot, bundlesToSourceEntries, getSources, isBundleEnabled, loadConfig, } from "../../core/config/config.js";
8
8
  import { ConfigError } from "../../core/errors.js";
9
9
  import { getUnresolvedSourcesDir } from "../../core/paths.js";
@@ -50,22 +50,15 @@ export function resolveSourceEntries(overrideStashDir, existingConfig) {
50
50
  }
51
51
  // Explicit and environment overrides stay first. Without either override,
52
52
  // the configured default enters through the validated loop below.
53
+ // A source's path is its real path, so one directory reached through two spellings
54
+ // (a symlink, macOS `/tmp` and `/private/tmp`) keys the same index rows.
53
55
  const sources = implicitStash
54
- ? [{ path: implicitStash, writable: true, isDefault: true, type: "filesystem" }]
56
+ ? [{ path: safeRealpath(implicitStash), writable: true, isDefault: true, type: "filesystem" }]
55
57
  : [];
56
- const sourceIdentity = (dir) => {
57
- const resolved = path.resolve(dir);
58
- try {
59
- return fs.realpathSync.native(resolved);
60
- }
61
- catch {
62
- return resolved;
63
- }
64
- };
65
- const seen = new Set(implicitStash ? [sourceIdentity(implicitStash)] : []);
58
+ const seen = new Set(implicitStash ? [safeRealpath(implicitStash)] : []);
66
59
  const addSource = (dir, registryId, writable, type, adapterId, unresolved = false, isDefault = false) => {
67
60
  const resolved = path.resolve(dir);
68
- const identity = sourceIdentity(resolved);
61
+ const identity = safeRealpath(resolved);
69
62
  if (seen.has(identity)) {
70
63
  // Already in the source list — typically the primary stash injected at
71
64
  // sources[0] before this loop. Enrich that entry with whatever metadata
@@ -73,7 +66,7 @@ export function resolveSourceEntries(overrideStashDir, existingConfig) {
73
66
  // find it via registryId. Without this, the primary stash entry stays
74
67
  // identity-less and a user-named primary source ("name": "my-stash")
75
68
  // would validate but match zero entries when filtering.
76
- const existing = sources.find((source) => sourceIdentity(source.path) === identity);
69
+ const existing = sources.find((source) => safeRealpath(source.path) === identity);
77
70
  if (existing && existing.registryId === undefined) {
78
71
  if (registryId)
79
72
  existing.registryId = registryId;
@@ -92,7 +85,7 @@ export function resolveSourceEntries(overrideStashDir, existingConfig) {
92
85
  warn(`Warning: stash root "${dir}" appears to be a system directory. This may be unintentional.`);
93
86
  }
94
87
  sources.push({
95
- path: resolved,
88
+ path: identity,
96
89
  ...(registryId ? { registryId } : {}),
97
90
  ...(isDefault ? { isDefault: true } : {}),
98
91
  writable,
@@ -192,17 +185,22 @@ export function resolveEntryContentDir(entry) {
192
185
  * corrupting origin / editability / provenance decisions for the affected files.
193
186
  */
194
187
  export function findSourceForPath(filePath, sources) {
195
- const resolved = path.resolve(filePath);
196
- let best;
197
- let bestLen = -1;
198
- for (const source of sources) {
199
- const base = path.resolve(source.path);
200
- if (resolved.startsWith(base + path.sep) && base.length > bestLen) {
201
- best = source;
202
- bestLen = base.length;
188
+ const longest = (within) => {
189
+ let best;
190
+ let bestLen = -1;
191
+ for (const source of sources) {
192
+ const base = path.resolve(source.path);
193
+ if (base.length > bestLen && within(base)) {
194
+ best = source;
195
+ bestLen = base.length;
196
+ }
203
197
  }
204
- }
205
- return best;
198
+ return best;
199
+ };
200
+ const resolved = path.resolve(filePath);
201
+ // Paths from the index and the sources share one spelling; a caller's may not (a symlinked library).
202
+ return (longest((base) => resolved.startsWith(base + path.sep)) ??
203
+ longest((base) => relativeToRoot(resolved, base) !== undefined));
206
204
  }
207
205
  // ── Editability ─────────────────────────────────────────────────────────────
208
206
  /**
@@ -146,21 +146,30 @@ export function usageRetentionSince() {
146
146
  return new Date(Date.now() - USAGE_EVENT_RETENTION_DAYS * 86_400_000).toISOString();
147
147
  }
148
148
  /**
149
- * How many times each asset was shown and how many searches returned it, by durable `entry_ref`: user-attributed
150
- * events at or after `sinceIso`, in one pass. The same definition of use as the utility aggregation in the indexer,
149
+ * How many times each asset was shown, how many searches returned it, and when it was last shown, by durable
150
+ * `entry_ref`: user-attributed events at or after `sinceIso`, in one pass. The same definition of use as the utility aggregation in the indexer,
151
151
  * read live instead of as of the last index.
152
152
  */
153
153
  export function countUseByRef(db, sinceIso) {
154
154
  const rows = db
155
155
  .prepare(`SELECT entry_ref,
156
156
  SUM(CASE WHEN event_type = 'show' THEN 1 ELSE 0 END) AS shows,
157
- SUM(CASE WHEN event_type = 'search' THEN 1 ELSE 0 END) AS searches
157
+ SUM(CASE WHEN event_type = 'search' THEN 1 ELSE 0 END) AS searches,
158
+ MAX(CASE WHEN event_type = 'show' THEN datetime(created_at) END) AS last_shown
158
159
  FROM usage_events
159
160
  WHERE event_type IN ('show', 'search') AND source = 'user' AND entry_ref IS NOT NULL
160
161
  AND datetime(created_at) >= datetime(?)
161
162
  GROUP BY entry_ref`)
162
163
  .all(sinceIso);
163
- return new Map(rows.map((row) => [row.entry_ref, { shows: row.shows, searches: row.searches }]));
164
+ return new Map(rows.map((row) => [
165
+ row.entry_ref,
166
+ {
167
+ shows: row.shows,
168
+ searches: row.searches,
169
+ // SQLite's datetime() is UTC without a zone: say so, so it parses as UTC everywhere.
170
+ lastShownAt: row.last_shown ? `${row.last_shown.replace(" ", "T")}Z` : null,
171
+ },
172
+ ]));
164
173
  }
165
174
  /**
166
175
  * Count usage events of a given `event_type`.
@@ -1,8 +1,8 @@
1
1
  // This Source Code Form is subject to the terms of the Mozilla Public
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
- import path from "node:path";
5
4
  import { isBundleSlug } from "../core/asset/asset-ref.js";
5
+ import { samePath } from "../core/common.js";
6
6
  import { deriveInstallations } from "../indexer/installations.js";
7
7
  import { parseRegistryRef } from "./resolve.js";
8
8
  /**
@@ -32,8 +32,7 @@ export function resolveSourcesForLocator(locator, allSources) {
32
32
  catch {
33
33
  // Not a registry locator; continue to path matching.
34
34
  }
35
- const resolvedLocator = path.resolve(locator);
36
- return allSources.filter((source) => path.resolve(source.path) === resolvedLocator);
35
+ return allSources.filter((source) => samePath(source.path, locator));
37
36
  }
38
37
  /**
39
38
  * Check whether an origin refers to something that could be fetched remotely