akm-cli 0.9.0-rc.2 → 0.9.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CHANGELOG.md +34 -8
  2. package/dist/assets/improve-strategies/catchup.json +3 -1
  3. package/dist/assets/improve-strategies/consolidate.json +3 -1
  4. package/dist/assets/improve-strategies/frequent.json +3 -1
  5. package/dist/assets/improve-strategies/graph-refresh.json +3 -1
  6. package/dist/assets/improve-strategies/memory-focus.json +4 -1
  7. package/dist/assets/improve-strategies/proactive-maintenance.json +1 -0
  8. package/dist/assets/improve-strategies/quick.json +3 -1
  9. package/dist/assets/improve-strategies/recombine-only.json +2 -0
  10. package/dist/assets/improve-strategies/reflect-distill.json +1 -0
  11. package/dist/assets/improve-strategies/synthesize.json +2 -0
  12. package/dist/assets/tasks/core/backup.yml +2 -2
  13. package/dist/cli/config-migrate.js +554 -26
  14. package/dist/cli.js +3 -1
  15. package/dist/commands/backup-cli.js +6 -4
  16. package/dist/commands/config-cli.js +10 -3
  17. package/dist/commands/feedback-cli.js +24 -7
  18. package/dist/commands/health/checks.js +94 -15
  19. package/dist/commands/health/surfaces.js +2 -1
  20. package/dist/commands/improve/anti-collapse.js +3 -3
  21. package/dist/commands/improve/collapse-detector.js +5 -5
  22. package/dist/commands/improve/consolidate.js +123 -66
  23. package/dist/commands/improve/distill/promote-memory.js +8 -6
  24. package/dist/commands/improve/distill/quality-gate.js +1 -1
  25. package/dist/commands/improve/distill-guards.js +1 -1
  26. package/dist/commands/improve/distill-promotion-policy.js +32 -26
  27. package/dist/commands/improve/distill.js +52 -36
  28. package/dist/commands/improve/eligibility.js +9 -8
  29. package/dist/commands/improve/extract-prompt.js +2 -2
  30. package/dist/commands/improve/extract.js +43 -11
  31. package/dist/commands/improve/improve-auto-accept.js +14 -28
  32. package/dist/commands/improve/improve-cli.js +4 -2
  33. package/dist/commands/improve/improve-strategies.js +2 -1
  34. package/dist/commands/improve/improve.js +49 -8
  35. package/dist/commands/improve/loop-stages.js +11 -6
  36. package/dist/commands/improve/preparation.js +51 -17
  37. package/dist/commands/improve/procedural.js +10 -6
  38. package/dist/commands/improve/recombine.js +31 -7
  39. package/dist/commands/improve/reflect.js +24 -14
  40. package/dist/commands/improve/salience.js +5 -1
  41. package/dist/commands/improve/source-identity.js +43 -0
  42. package/dist/commands/migrate-cli.js +36 -0
  43. package/dist/commands/mv-cli.js +647 -258
  44. package/dist/commands/proposal/drain.js +16 -4
  45. package/dist/commands/proposal/proposal-cli.js +3 -2
  46. package/dist/commands/proposal/proposal.js +16 -55
  47. package/dist/commands/proposal/repository.js +664 -58
  48. package/dist/commands/read/curate.js +4 -2
  49. package/dist/commands/read/knowledge.js +4 -1
  50. package/dist/commands/read/search.js +6 -2
  51. package/dist/commands/read/show.js +8 -7
  52. package/dist/commands/sources/self-update.js +156 -112
  53. package/dist/commands/sources/sources-cli.js +7 -2
  54. package/dist/core/common.js +36 -3
  55. package/dist/core/config/config-io.js +2 -2
  56. package/dist/core/config/config-schema.js +15 -3
  57. package/dist/core/config/config.js +14 -13
  58. package/dist/core/file-lock.js +2 -2
  59. package/dist/core/migration-backup.js +816 -171
  60. package/dist/core/migration-operation.js +44 -0
  61. package/dist/core/state/migrations.js +4 -7
  62. package/dist/core/state-db.js +22 -7
  63. package/dist/core/write-source.js +86 -18
  64. package/dist/indexer/db/db.js +109 -53
  65. package/dist/indexer/index-writer-lock.js +38 -37
  66. package/dist/indexer/index-written-assets.js +73 -56
  67. package/dist/indexer/indexer.js +5 -1
  68. package/dist/indexer/search/search-source.js +2 -2
  69. package/dist/indexer/usage/usage-events.js +8 -2
  70. package/dist/integrations/agent/engine-resolution.js +14 -7
  71. package/dist/scripts/migrate-storage.js +867 -990
  72. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +405 -566
  73. package/dist/setup/detected-engines.js +7 -13
  74. package/dist/setup/engine-config.js +14 -4
  75. package/dist/setup/setup.js +1 -1
  76. package/dist/setup/steps/connection.js +1 -1
  77. package/dist/setup/steps/tasks.js +1 -1
  78. package/dist/sources/providers/git-stash.js +58 -21
  79. package/dist/sources/providers/git.js +1 -1
  80. package/dist/storage/engines/sqlite-migrations.js +138 -3
  81. package/dist/storage/repositories/events-repository.js +24 -0
  82. package/dist/storage/repositories/proposals-repository.js +14 -0
  83. package/dist/tasks/embedded.js +2 -0
  84. package/dist/workflows/db.js +24 -13
  85. package/dist/workflows/validator.js +24 -2
  86. package/docs/migration/release-notes/0.9.0.md +24 -7
  87. package/docs/migration/v0.8-to-v0.9.md +124 -19
  88. package/package.json +1 -1
@@ -0,0 +1,44 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import crypto from "node:crypto";
5
+ import fs from "node:fs";
6
+ import path from "node:path";
7
+ import { ConfigError } from "./errors.js";
8
+ import { getConfigPath, getDataDir } from "./paths.js";
9
+ let afterPendingCheckHook;
10
+ /** TEST-ONLY: run once after a clear pending-operation check. */
11
+ export function _setAfterPendingOperationCheckHookForTests(hook) {
12
+ afterPendingCheckHook = hook;
13
+ }
14
+ function installationId() {
15
+ return crypto
16
+ .createHash("sha256")
17
+ .update(path.resolve(path.dirname(getConfigPath())))
18
+ .update("\0")
19
+ .update(path.resolve(getDataDir()))
20
+ .digest("hex")
21
+ .slice(0, 24);
22
+ }
23
+ export function getMigrationOperationRoot() {
24
+ return path.join(getDataDir(), "backups", "migrations", installationId());
25
+ }
26
+ export function getMigrationRestoreJournalPath() {
27
+ return path.join(getMigrationOperationRoot(), "restore-active.json");
28
+ }
29
+ export function getMigrationApplyJournalPath() {
30
+ return path.join(getMigrationOperationRoot(), "apply-active.json");
31
+ }
32
+ export function assertNoPendingMigrationOperation() {
33
+ for (const [kind, journalPath] of [
34
+ ["restore", getMigrationRestoreJournalPath()],
35
+ ["migration apply", getMigrationApplyJournalPath()],
36
+ ]) {
37
+ if (fs.existsSync(journalPath)) {
38
+ throw new ConfigError(`AKM ${kind} recovery is pending at ${journalPath}; refusing canonical config/database access until recovery completes.`, "INVALID_CONFIG_FILE");
39
+ }
40
+ }
41
+ const hook = afterPendingCheckHook;
42
+ afterPendingCheckHook = undefined;
43
+ hook?.();
44
+ }
@@ -2,8 +2,7 @@
2
2
  // License, v. 2.0. If a copy of the MPL was not distributed with this
3
3
  // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
4
  import { runMigrations as runSqliteMigrations } from "../../storage/engines/sqlite-migrations.js";
5
- import { ensureMigrationBackup } from "../migration-backup.js";
6
- const MIGRATIONS = [
5
+ export const STATE_MIGRATIONS = [
7
6
  // ── Migration 001 — initial schema ──────────────────────────────────────────
8
7
  {
9
8
  id: "001-initial-schema",
@@ -776,10 +775,8 @@ const MIGRATIONS = [
776
775
  * Called automatically by `openStateDatabase()`.
777
776
  */
778
777
  export function runMigrations(db, options) {
779
- runSqliteMigrations(db, MIGRATIONS, {
780
- beforeMigration(migration) {
781
- if (options?.ensureCutoverBackup && migration.id === "017-improve-run-strategy")
782
- ensureMigrationBackup();
783
- },
778
+ runSqliteMigrations(db, STATE_MIGRATIONS, {
779
+ applyPending: options?.applyPending,
780
+ generationMarker: options?.generationMarker,
784
781
  });
785
782
  }
@@ -57,10 +57,13 @@
57
57
  */
58
58
  import fs from "node:fs";
59
59
  import path from "node:path";
60
+ import { openDatabase } from "../storage/database.js";
61
+ import { assertCurrentMigrationLedger, assertMigrationLedger } from "../storage/engines/sqlite-migrations.js";
60
62
  import { openManagedDatabase, withManagedDb, withManagedDbAsync } from "../storage/managed-db.js";
61
63
  import { acquireMaintenanceActivitySync } from "./maintenance-barrier.js";
62
- import { ensureMigrationBackup, getMigrationBackupDir } from "./migration-backup.js";
64
+ import { assertNoPendingMigrationOperation } from "./migration-operation.js";
63
65
  import { getDataDir } from "./paths.js";
66
+ import { runMigrations, STATE_MIGRATIONS } from "./state/migrations.js";
64
67
  // ── Path helper ──────────────────────────────────────────────────────────────
65
68
  /**
66
69
  * Default path: `<dataDir>/state.db`.
@@ -102,15 +105,28 @@ export function openStateDatabase(dbPath) {
102
105
  const canonicalPath = getStateDbPath();
103
106
  const resolvedPath = dbPath ?? canonicalPath;
104
107
  const isCanonical = path.resolve(resolvedPath) === path.resolve(canonicalPath);
108
+ if (isCanonical)
109
+ assertNoPendingMigrationOperation();
105
110
  const releaseActivity = isCanonical ? acquireMaintenanceActivitySync("state-db") : undefined;
106
111
  try {
107
- // This must precede mkdir/open: opening an absent SQLite path creates it and
108
- // would make the recovery manifest falsely record a fresh DB as pre-existing.
109
- if (isCanonical && !fs.existsSync(getMigrationBackupDir()))
110
- ensureMigrationBackup();
112
+ if (isCanonical)
113
+ assertNoPendingMigrationOperation();
114
+ const existed = fs.existsSync(resolvedPath);
115
+ if (existed) {
116
+ const preflight = openDatabase(resolvedPath, { readonly: true });
117
+ try {
118
+ if (isCanonical)
119
+ assertCurrentMigrationLedger(preflight, STATE_MIGRATIONS);
120
+ else
121
+ assertMigrationLedger(preflight, STATE_MIGRATIONS);
122
+ }
123
+ finally {
124
+ preflight.close();
125
+ }
126
+ }
111
127
  const db = openManagedDatabase({
112
128
  path: resolvedPath,
113
- init: (db) => runMigrations(db, { ensureCutoverBackup: isCanonical }),
129
+ init: (db) => runMigrations(db, { applyPending: !(isCanonical && existed) }),
114
130
  });
115
131
  if (!releaseActivity)
116
132
  return db;
@@ -164,7 +180,6 @@ export function withStateDbAsync(fn, opts) {
164
180
  // The MIGRATIONS registry + runMigrations live in ./state/migrations (the single
165
181
  // append-only ordered source of truth). Imported for internal use by
166
182
  // openStateDatabase.
167
- import { runMigrations } from "./state/migrations.js";
168
183
  // ── BEGIN IMMEDIATE transaction helper ───────────────────────────────────────
169
184
  /**
170
185
  * Run `fn` inside a `BEGIN IMMEDIATE` transaction.
@@ -13,8 +13,8 @@
13
13
  * commit behaviour — they only ever touch the filesystem. Git-backed targets
14
14
  * are committed in a SINGLE batch at the operation boundary via
15
15
  * {@link commitWriteTargetBoundary} (which delegates to `saveGitStash`). This
16
- * stages `.akm/` + sibling assets together as one complete commit instead of
17
- * one noisy, incomplete commit per asset.
16
+ * stages only operation-owned paths that still have a Git status entry as one
17
+ * complete commit instead of one noisy, incomplete commit per asset.
18
18
  *
19
19
  * This module is still the **single dispatch point** for write/delete: callers
20
20
  * (remember, import, source-add, etc.) MUST go through `writeAssetToSource` /
@@ -24,7 +24,7 @@
24
24
  */
25
25
  import fs from "node:fs";
26
26
  import path from "node:path";
27
- import { getCachePaths, parseGitRepoUrl, saveGitStash } from "../sources/providers/git.js";
27
+ import { getCachePaths, listGitChangedPaths, parseGitRepoUrl, saveGitStash } from "../sources/providers/git.js";
28
28
  import { makeAssetRef } from "./asset/asset-ref.js";
29
29
  import { resolveAssetPathFromName, TYPE_DIRS } from "./asset/asset-spec.js";
30
30
  import { isWithin, resolveStashDir } from "./common.js";
@@ -45,6 +45,30 @@ const REJECTED_WRITABLE_KINDS = new Set(["website", "npm"]);
45
45
  * stream a downstream consumer parses.
46
46
  */
47
47
  const COMMIT_MESSAGE_MAX_LENGTH = 4096;
48
+ const pendingGitPaths = new Map();
49
+ function gitTargetKey(source) {
50
+ return `${path.resolve(source.repoPath ?? source.path)}\0${path.resolve(source.path)}`;
51
+ }
52
+ /** Register a successful direct mutation for the target's exact-path boundary commit. */
53
+ export function recordWriteTargetPath(source, filePath) {
54
+ if (source.kind !== "git")
55
+ return;
56
+ const key = gitTargetKey(source);
57
+ const paths = pendingGitPaths.get(key) ?? new Set();
58
+ paths.add(path.resolve(filePath));
59
+ pendingGitPaths.set(key, paths);
60
+ }
61
+ function takeGitTargetPaths(source) {
62
+ const key = gitTargetKey(source);
63
+ const absolutePaths = pendingGitPaths.get(key);
64
+ pendingGitPaths.delete(key);
65
+ if (!absolutePaths)
66
+ return [];
67
+ const repoDir = path.resolve(source.repoPath ?? source.path);
68
+ return [...absolutePaths]
69
+ .map((filePath) => path.relative(repoDir, filePath).replaceAll(path.sep, "/"))
70
+ .filter((filePath) => filePath && filePath !== ".." && !filePath.startsWith("../"));
71
+ }
48
72
  /**
49
73
  * Sanitize a string before passing it as `git commit -m <message>`.
50
74
  *
@@ -161,6 +185,7 @@ export async function writeAssetToSource(source, config, ref, content) {
161
185
  fs.mkdirSync(path.dirname(filePath), { recursive: true });
162
186
  const normalized = content.endsWith("\n") ? content : `${content}\n`;
163
187
  fs.writeFileSync(filePath, normalized, "utf8");
188
+ recordWriteTargetPath(source, filePath);
164
189
  // Non-fatal portability advisory (review 13, D1): flag absolute host home
165
190
  // paths in the written content. These make the stash non-portable and leak
166
191
  // the local username. We warn AFTER the write so the advisory never blocks it.
@@ -185,6 +210,7 @@ export async function deleteAssetFromSource(source, config, ref) {
185
210
  throw new UsageError(`Asset "${formatRefForMessage(ref)}" not found in source "${source.name}" (expected at ${filePath}).`, "MISSING_REQUIRED_ARGUMENT");
186
211
  }
187
212
  fs.unlinkSync(filePath);
213
+ recordWriteTargetPath(source, filePath);
188
214
  return { path: filePath, ref: makeAssetRef(ref.type, ref.name, ref.origin) };
189
215
  }
190
216
  /**
@@ -196,9 +222,10 @@ export async function deleteAssetFromSource(source, config, ref) {
196
222
  * primary stash stay non-committing here — the primary stash is committed by
197
223
  * the existing improve auto-sync boundary).
198
224
  *
199
- * For a git target it delegates to `saveGitStash(name, message, writable, …)`,
200
- * which stages `.akm/` + sibling assets together (`git add -A`), commits once,
201
- * and pushes when the target is writable, has a remote, and `push !== false`.
225
+ * For a git target it delegates to `saveGitStash(name, message, writable, …)`
226
+ * with the exact paths recorded by the write/delete helpers (plus any explicit
227
+ * caller paths), commits once, and pushes when the target is writable, has a
228
+ * remote, and `push !== false`.
202
229
  *
203
230
  * The push intent honours a deprecated `options.pushOnCommit` on the source
204
231
  * config (mapped onto the batch push gate) when `push` is not explicitly set.
@@ -212,12 +239,16 @@ export function commitWriteTargetBoundary(target, message, options) {
212
239
  // gates the actual push on writable + remote, so this only ever opts *in*.
213
240
  const push = options?.push ?? (target.config.options?.pushOnCommit === true ? true : undefined);
214
241
  const writable = resolveWritable(target.config);
215
- // Commit against the already-resolved repo directory (target.source.path)
216
- // rather than re-resolving the stash by name through config. The write helper
217
- // resolved this exact path; the boundary commit must operate on the SAME
218
- // directory so the staged batch matches what was just written.
242
+ const repoDir = target.source.repoPath ?? target.source.path;
243
+ const changedPaths = new Set(listGitChangedPaths(repoDir));
244
+ const paths = [...new Set([...(options?.paths ?? []), ...takeGitTargetPaths(target.source)])]
245
+ .map((filePath) => filePath.replaceAll(path.sep, "/"))
246
+ .filter((filePath) => changedPaths.has(filePath));
247
+ // Assets may live under <repo>/content, but git synchronization always runs
248
+ // against the repository root.
219
249
  saveGitStash(undefined, message, writable, {
220
- repoDir: target.source.path,
250
+ repoDir,
251
+ paths,
221
252
  ...(push === undefined ? {} : { push }),
222
253
  });
223
254
  }
@@ -239,6 +270,32 @@ function warnIfPushOnCommit(config) {
239
270
  "akm now commits writes in a single batch at the operation boundary and pushes when the target is " +
240
271
  "writable with a remote. Remove the option or rely on sync push instead.\n");
241
272
  }
273
+ /** Enumerate enabled writable targets, deduplicated by materialized content root. */
274
+ export function resolveWritableTargets(akmConfig) {
275
+ const byRoot = new Map();
276
+ for (const runtime of resolveConfiguredSources(akmConfig)) {
277
+ if (runtime.enabled === false || !resolveWritable({ type: runtime.type, writable: runtime.writable }))
278
+ continue;
279
+ const target = adaptConfiguredSource(runtime);
280
+ const root = path.resolve(target.source.path);
281
+ const existing = byRoot.get(root);
282
+ if (!existing || target.source.name === akmConfig.defaultWriteTarget)
283
+ byRoot.set(root, target);
284
+ }
285
+ if (byRoot.size === 0) {
286
+ try {
287
+ const stashDir = resolveStashDir({ readOnly: true });
288
+ byRoot.set(path.resolve(stashDir), {
289
+ source: { kind: "filesystem", name: "stash", path: stashDir },
290
+ config: { type: "filesystem", path: stashDir, name: "stash", writable: true },
291
+ });
292
+ }
293
+ catch {
294
+ // No active working stash; configured writable targets above are complete.
295
+ }
296
+ }
297
+ return [...byRoot.values()];
298
+ }
242
299
  /**
243
300
  * Resolve the destination for a write per locked decision 3:
244
301
  *
@@ -250,8 +307,9 @@ function warnIfPushOnCommit(config) {
250
307
  * The legacy `first-writable-in-source-array-order` fallback is *not* used —
251
308
  * see plan §6 decision 3 for the rationale.
252
309
  */
253
- export function resolveWriteTarget(akmConfig, explicitTarget) {
310
+ export function resolveWriteTarget(akmConfig, explicitTarget, options = {}) {
254
311
  const configuredSources = resolveConfiguredSources(akmConfig);
312
+ const requireWritable = options.requireWritable !== false;
255
313
  // 1. Explicit --target wins.
256
314
  if (explicitTarget) {
257
315
  const match = configuredSources.find((s) => s.name === explicitTarget);
@@ -264,7 +322,7 @@ export function resolveWriteTarget(akmConfig, explicitTarget) {
264
322
  // effective writable flag (filesystem defaults to true; everything else
265
323
  // defaults to false) so unset values are interpreted correctly.
266
324
  const effectiveWritable = resolveWritable({ type: match.type, writable: match.writable });
267
- if (!effectiveWritable) {
325
+ if (requireWritable && !effectiveWritable) {
268
326
  throw new ConfigError(`source ${explicitTarget} is not writable`, "INVALID_CONFIG_FILE", `Set \`writable: true\` on the "${explicitTarget}" source in your config, or pass --target to a different source.`);
269
327
  }
270
328
  return adaptConfiguredSource(match);
@@ -279,7 +337,7 @@ export function resolveWriteTarget(akmConfig, explicitTarget) {
279
337
  // ConfigError, rather than surfacing as a generic UsageError after
280
338
  // path-building has already begun.
281
339
  const effectiveWritable = resolveWritable({ type: match.type, writable: match.writable });
282
- if (!effectiveWritable) {
340
+ if (requireWritable && !effectiveWritable) {
283
341
  throw new ConfigError(`defaultWriteTarget "${akmConfig.defaultWriteTarget}" is not writable`, "INVALID_CONFIG_FILE", `Set \`writable: true\` on the "${akmConfig.defaultWriteTarget}" source in your config, or change \`defaultWriteTarget\` to a writable source.`);
284
342
  }
285
343
  return adaptConfiguredSource(match);
@@ -366,8 +424,8 @@ export function formatRefForMessage(ref) {
366
424
  * `git` by the config loader, so this mapping is straightforward.
367
425
  */
368
426
  function adaptConfiguredSource(runtime) {
369
- const writePath = pathFromConfiguredSource(runtime);
370
- if (!writePath) {
427
+ const repoPath = pathFromConfiguredSource(runtime);
428
+ if (!repoPath) {
371
429
  throw new ConfigError(`Source "${runtime.name}" has no resolvable on-disk path; writes are unsupported for this entry.`, "INVALID_CONFIG_FILE");
372
430
  }
373
431
  // Map the runtime kind to the write helper's `kind` discriminator. Only
@@ -383,15 +441,25 @@ function adaptConfiguredSource(runtime) {
383
441
  const config = {
384
442
  type: runtime.type,
385
443
  name: runtime.name,
386
- ...(writePath !== undefined ? { path: writePath } : {}),
444
+ ...(repoPath !== undefined ? { path: repoPath } : {}),
387
445
  ...(runtime.writable !== undefined ? { writable: runtime.writable } : {}),
388
446
  ...(runtime.options ? { options: runtime.options } : {}),
389
447
  };
390
448
  return {
391
- source: { kind, name: runtime.name, path: writePath },
449
+ source: {
450
+ kind,
451
+ name: runtime.name,
452
+ path: kind === "git" ? resolveGitContentRoot(repoPath) : repoPath,
453
+ ...(kind === "git" ? { repoPath } : {}),
454
+ },
392
455
  config,
393
456
  };
394
457
  }
458
+ /** Resolve the asset root inside a git checkout while preserving root-layout repos. */
459
+ export function resolveGitContentRoot(repoPath) {
460
+ const contentPath = path.join(repoPath, "content");
461
+ return fs.existsSync(contentPath) && fs.statSync(contentPath).isDirectory() ? contentPath : repoPath;
462
+ }
395
463
  function pathFromConfiguredSource(runtime) {
396
464
  // ConfiguredSource.source is the parsed SourceSpec (filesystem|git|website|npm).
397
465
  // For writable kinds we only ever care about a local on-disk path: filesystem
@@ -351,9 +351,14 @@ export function getBaseBeliefStatesForDerivedTwins(db, twinIds) {
351
351
  * full `akm index` picks the file up as a fresh entry).
352
352
  */
353
353
  export function rekeyEntryInPlace(db, opts) {
354
- const row = db.prepare("SELECT id, entry_json, search_text FROM entries WHERE entry_key = ?").get(opts.oldEntryKey);
354
+ const row = db
355
+ .prepare("SELECT id, stash_dir, entry_json, search_text FROM entries WHERE entry_key = ?")
356
+ .get(opts.oldEntryKey);
355
357
  if (!row)
356
358
  return null;
359
+ if (opts.sourceRoot && path.resolve(row.stash_dir) !== path.resolve(opts.sourceRoot)) {
360
+ throw new Error(`Refusing to re-key entry ${opts.oldEntryKey}: source root does not match.`);
361
+ }
357
362
  // Patch the JSON payload. On corrupt entry_json still re-key key + paths so
358
363
  // the utility history survives; the next full index heals the JSON.
359
364
  let entryJson = row.entry_json;
@@ -391,7 +396,7 @@ export function rekeyEntryInPlace(db, opts) {
391
396
  // the part after the last `//` (stored origins never contain `//`, so a
392
397
  // qualified ref has exactly one — same normalization as
393
398
  // getRetrievalCounts). Legacy DBs may predate usage_events.
394
- bestEffort(() => {
399
+ try {
395
400
  // Live-asset-wins collision policy, mirroring the stale-entries eviction
396
401
  // above and mv's state.db re-key: DETACHED orphan events (entry_id NULL
397
402
  // — a deleted asset's history retained by a full rebuild) already
@@ -401,17 +406,30 @@ export function rekeyEntryInPlace(db, opts) {
401
406
  // moved asset would adopt the stranger's history: getRetrievalCounts
402
407
  // reads by entry_ref immediately, and the next full rebuild's
403
408
  // relinkUsageEvents would attach every stranger event by ref.
404
- db.prepare("DELETE FROM usage_events WHERE entry_id IS NULL AND entry_ref = ?").run(opts.newRef);
405
- db.prepare(`DELETE FROM usage_events
406
- WHERE entry_id IS NULL
407
- AND instr(entry_ref, '//') > 0
408
- AND substr(entry_ref, instr(entry_ref, '//') + 2) = ?`).run(opts.newRef);
409
- db.prepare("UPDATE usage_events SET entry_ref = ? WHERE entry_ref = ?").run(opts.newRef, opts.oldRef);
410
- db.prepare(`UPDATE usage_events
411
- SET entry_ref = substr(entry_ref, 1, instr(entry_ref, '//') + 1) || ?
412
- WHERE instr(entry_ref, '//') > 0
413
- AND substr(entry_ref, instr(entry_ref, '//') + 2) = ?`).run(opts.newRef, opts.oldRef);
414
- }, "usage_events table may be missing on legacy DBs — no usage history to re-key");
409
+ if (opts.sourceName) {
410
+ const origins = new Set([opts.sourceName]);
411
+ if (opts.sourceName === "stash" && opts.includeLegacyBare)
412
+ origins.add("local");
413
+ for (const origin of origins) {
414
+ const oldQualifiedRef = `${origin}//${opts.oldRef}`;
415
+ const newQualifiedRef = `${origin}//${opts.newRef}`;
416
+ db.prepare("DELETE FROM usage_events WHERE entry_id IS NULL AND entry_ref = ?").run(newQualifiedRef);
417
+ db.prepare("UPDATE usage_events SET entry_ref = ? WHERE entry_ref = ?").run(newQualifiedRef, oldQualifiedRef);
418
+ }
419
+ }
420
+ if (opts.includeLegacyBare || !opts.sourceName) {
421
+ db.prepare("DELETE FROM usage_events WHERE entry_id IS NULL AND entry_ref = ?").run(opts.newRef);
422
+ db.prepare("UPDATE usage_events SET entry_ref = ? WHERE entry_ref = ?").run(opts.newRef, opts.oldRef);
423
+ }
424
+ }
425
+ catch (error) {
426
+ const message = error instanceof Error ? error.message : String(error);
427
+ const missingLegacyUsageSchema = /no such table:\s*(?:main\.)?usage_events\b/i.test(message) ||
428
+ /no such column:\s*(?:usage_events\.)?(?:entry_id|entry_ref)\b/i.test(message) ||
429
+ /table\s+usage_events\s+has no column named\s+(?:entry_id|entry_ref)\b/i.test(message);
430
+ if (!missingLegacyUsageSchema)
431
+ throw error;
432
+ }
415
433
  db.prepare("INSERT OR IGNORE INTO entries_fts_dirty (entry_id) VALUES (?)").run(row.id);
416
434
  })();
417
435
  return row.id;
@@ -884,7 +902,7 @@ export function getEntitiesByEntryIds(db, entryIds) {
884
902
  }
885
903
  return result;
886
904
  }
887
- export function findEntryIdByRef(db, ref) {
905
+ export function findEntryIdByRef(db, ref, stashDir) {
888
906
  const parsed = parseAssetRef(ref);
889
907
  const nameVariants = [parsed.name];
890
908
  if (parsed.name.endsWith(".md")) {
@@ -893,10 +911,14 @@ export function findEntryIdByRef(db, ref) {
893
911
  else {
894
912
  nameVariants.push(`${parsed.name}.md`);
895
913
  }
896
- const stmt = db.prepare("SELECT id FROM entries WHERE entry_type = ? AND substr(entry_key, length(entry_key) - length(?) + 1) = ? LIMIT 1");
914
+ const stmt = db.prepare(`SELECT id FROM entries
915
+ WHERE entry_type = ?
916
+ AND substr(entry_key, length(entry_key) - length(?) + 1) = ?
917
+ ${stashDir ? "AND stash_dir = ?" : ""}
918
+ LIMIT 1`);
897
919
  for (const name of nameVariants) {
898
920
  const suffix = `${parsed.type}:${name}`;
899
- const row = stmt.get(parsed.type, suffix, suffix);
921
+ const row = stmt.get(parsed.type, suffix, suffix, ...(stashDir ? [stashDir] : []));
900
922
  if (row)
901
923
  return row.id;
902
924
  }
@@ -914,7 +936,7 @@ export function getEmbeddingCount(db) {
914
936
  return row.cnt;
915
937
  }
916
938
  export function getEntryById(db, id) {
917
- const row = db.prepare("SELECT file_path, entry_json FROM entries WHERE id = ?").get(id);
939
+ const row = db.prepare("SELECT file_path, stash_dir, entry_json FROM entries WHERE id = ?").get(id);
918
940
  if (!row)
919
941
  return undefined;
920
942
  // Guard against corrupt JSON
@@ -926,7 +948,7 @@ export function getEntryById(db, id) {
926
948
  warn(`[db] getEntryById: skipping entry id=${id} — corrupt entry_json`);
927
949
  return undefined;
928
950
  }
929
- return { filePath: row.file_path, entry };
951
+ return { filePath: row.file_path, stashDir: row.stash_dir, entry };
930
952
  }
931
953
  export function getEntriesByDir(db, dirPath) {
932
954
  const rows = db.prepare(`SELECT ${ENTRY_COLUMNS} FROM entries WHERE dir_path = ?`).all(dirPath);
@@ -1192,33 +1214,12 @@ function bareRef(ref) {
1192
1214
  return ref;
1193
1215
  }
1194
1216
  }
1195
- /**
1196
- * Count retrieval events for the given entry refs.
1197
- *
1198
- * Counts `search`, `show`, and `curate` usage events. Returns a
1199
- * Map<inputRef, count> keyed by the *input* ref strings (only those with at
1200
- * least one matching event appear). Used by the improve loop to find
1201
- * high-retrieval assets without feedback.
1202
- *
1203
- * Matching is normalization-aware: each stored `entry_ref` is reduced to its
1204
- * bare `type:name` form before comparison, so a stash-prefixed stored ref
1205
- * (`origin//type:name`) still matches a bare input ref (`type:name`) and vice
1206
- * versa. Previously the raw `entry_ref IN (...)` comparison silently dropped
1207
- * roughly half the signal whenever the two spellings disagreed.
1208
- *
1209
- * `curate` events are included: their per-item rows are written with
1210
- * entry_ref populated (see logCurateEvent), so curation is a real retrieval
1211
- * signal here. Legacy summary-only curate rows with a NULL entry_ref simply
1212
- * contribute nothing.
1213
- *
1214
- * Machine-sourced events (`source` = 'improve' or 'task') are EXCLUDED: this
1215
- * count feeds salience/ranking, and pipeline probe traffic counting as demand
1216
- * creates a self-reinforcing loop (meta-review 05 DRIFT-6). NULL sources
1217
- * (pre-column rows) count as user demand.
1218
- */
1219
- export function getRetrievalCounts(db, refs) {
1217
+ export function getRetrievalCounts(db, refs, options = {}) {
1220
1218
  if (refs.length === 0)
1221
1219
  return new Map();
1220
+ if (options.sourceName || options.stashDir) {
1221
+ return getSourceScopedRetrievalCounts(db, refs, options);
1222
+ }
1222
1223
  // Map each distinct bare form back to the input ref(s) that produced it so we
1223
1224
  // can re-key DB results (grouped by bare form) onto the caller's ref strings.
1224
1225
  const bareToInputs = new Map();
@@ -1274,6 +1275,55 @@ export function getRetrievalCounts(db, refs) {
1274
1275
  }
1275
1276
  return result;
1276
1277
  }
1278
+ function getSourceScopedRetrievalCounts(db, refs, options) {
1279
+ const bareToInputs = new Map();
1280
+ for (const ref of refs) {
1281
+ const bare = bareRef(ref);
1282
+ const inputs = bareToInputs.get(bare);
1283
+ if (inputs)
1284
+ inputs.push(ref);
1285
+ else
1286
+ bareToInputs.set(bare, [ref]);
1287
+ }
1288
+ const countsByBare = new Map();
1289
+ const bareForms = [...bareToInputs.keys()];
1290
+ const selectedRoot = options.stashDir ? path.resolve(options.stashDir) : undefined;
1291
+ for (let i = 0; i < bareForms.length; i += SQLITE_CHUNK_SIZE) {
1292
+ const chunk = bareForms.slice(i, i + SQLITE_CHUNK_SIZE);
1293
+ const placeholders = chunk.map(() => "?").join(", ");
1294
+ const rows = db
1295
+ .prepare(`SELECT ue.entry_ref, ue.entry_id, e.stash_dir
1296
+ FROM usage_events ue
1297
+ LEFT JOIN entries e ON e.id = ue.entry_id
1298
+ WHERE ue.event_type IN ('search','show','curate')
1299
+ AND ue.entry_ref IS NOT NULL
1300
+ AND (ue.source IS NULL OR ue.source NOT IN ('improve','task'))
1301
+ AND CASE
1302
+ WHEN instr(ue.entry_ref, '//') > 0
1303
+ THEN substr(ue.entry_ref, instr(ue.entry_ref, '//') + 2)
1304
+ ELSE ue.entry_ref
1305
+ END IN (${placeholders})`)
1306
+ .all(...chunk);
1307
+ for (const row of rows) {
1308
+ const bare = bareRef(row.entry_ref);
1309
+ const linkedToSelectedRoot = row.entry_id !== null && selectedRoot !== undefined && row.stash_dir !== null
1310
+ ? path.resolve(row.stash_dir) === selectedRoot
1311
+ : false;
1312
+ const detached = row.entry_id === null || selectedRoot === undefined;
1313
+ const qualifiedForSource = detached && options.sourceName !== undefined && row.entry_ref === `${options.sourceName}//${bare}`;
1314
+ const acceptedLegacyBare = detached && options.includeLegacyBare === true && row.entry_ref === bare;
1315
+ if (!linkedToSelectedRoot && !qualifiedForSource && !acceptedLegacyBare)
1316
+ continue;
1317
+ countsByBare.set(bare, (countsByBare.get(bare) ?? 0) + 1);
1318
+ }
1319
+ }
1320
+ const result = new Map();
1321
+ for (const [bare, count] of countsByBare) {
1322
+ for (const input of bareToInputs.get(bare) ?? [])
1323
+ result.set(input, count);
1324
+ }
1325
+ return result;
1326
+ }
1277
1327
  /**
1278
1328
  * Apply a MemRL reward signal to a batch of entries via exponential moving
1279
1329
  * average (EMA): next = clamp(current + lr * (reward - current), 0, 1).
@@ -1434,13 +1484,23 @@ export function applyFeedbackToUtilityScore(db, entryId, positiveCount, negative
1434
1484
  `).run(entryId, result.nextUtility, now, now, result.nextUtility, now);
1435
1485
  return result;
1436
1486
  }
1487
+ function resolveUsageEventEntryId(db, ref, options) {
1488
+ const parsed = parseAssetRef(ref);
1489
+ if (!parsed.origin) {
1490
+ return options.defaultStashDir ? findEntryIdByRef(db, ref, options.defaultStashDir) : undefined;
1491
+ }
1492
+ const source = parsed.origin === "local" || parsed.origin === "stash"
1493
+ ? options.sources?.[0]
1494
+ : options.sources?.find((candidate) => candidate.registryId === parsed.origin);
1495
+ return source ? findEntryIdByRef(db, ref, source.path) : undefined;
1496
+ }
1437
1497
  /**
1438
1498
  * Re-link detached usage_events to their current entry_ids via entry_ref.
1439
1499
  *
1440
1500
  * After a full rebuild, entry IDs change. This restores each event's link
1441
1501
  * using the stable `entry_ref` column so usage history survives a reindex.
1442
1502
  */
1443
- export function relinkUsageEvents(db) {
1503
+ export function relinkUsageEvents(db, options = {}) {
1444
1504
  bestEffort(() => {
1445
1505
  // Step 1: null out stale entry_ids (entry was deleted, re-keyed, etc).
1446
1506
  // Leaving them in place would let `recomputeUtilityScores` aggregate
@@ -1455,14 +1515,10 @@ export function relinkUsageEvents(db) {
1455
1515
  WHERE entry_id IS NOT NULL
1456
1516
  AND entry_id NOT IN (SELECT id FROM entries)
1457
1517
  `);
1458
- // Step 2: re-resolve any null entry_id from entry_ref against the current
1459
- // entries table, reusing the SAME canonical resolver the read path uses at
1460
- // insert time (`findEntryIdByRef` `parseAssetRef`). Resolving per DISTINCT
1461
- // ref keeps this O(distinct-refs) indexed lookups instead of the previous
1462
- // O(events × entries) non-indexable `substr(entry_key, …)` scan. It also
1463
- // fixes a silent correctness bug: the old suffix match compared the RAW
1464
- // `entry_ref`, so origin-qualified refs ("source//type:name") never matched
1465
- // an `entry_key` and lost their usage history on every full rebuild.
1518
+ // Step 2: re-resolve each distinct ref inside its source boundary. Qualified
1519
+ // refs require an origin→root mapping; bare legacy refs require the explicit
1520
+ // historical/default root. This keeps duplicate refs from adopting whichever
1521
+ // entries row SQLite happens to return first while retaining indexed lookups.
1466
1522
  const refs = db
1467
1523
  .prepare("SELECT DISTINCT entry_ref AS ref FROM usage_events WHERE entry_id IS NULL AND entry_ref IS NOT NULL")
1468
1524
  .all();
@@ -1471,7 +1527,7 @@ export function relinkUsageEvents(db) {
1471
1527
  for (const { ref } of refs) {
1472
1528
  let id;
1473
1529
  try {
1474
- id = findEntryIdByRef(db, ref);
1530
+ id = resolveUsageEventEntryId(db, ref, options);
1475
1531
  }
1476
1532
  catch (err) {
1477
1533
  if (err instanceof Error && err.name === "UsageError")