akm-cli 0.9.17-alpha.8 → 0.9.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/CHANGELOG.md +277 -1694
  2. package/STABILITY.md +9 -8
  3. package/dist/assets/hints/cli-hints-full.md +6 -7
  4. package/dist/assets/improve-strategies/catchup.json +0 -3
  5. package/dist/assets/improve-strategies/consolidate.json +0 -1
  6. package/dist/assets/improve-strategies/default.json +1 -2
  7. package/dist/assets/improve-strategies/proactive-maintenance.json +1 -2
  8. package/dist/assets/improve-strategies/quick.json +1 -2
  9. package/dist/assets/improve-strategies/reflect-distill.json +1 -2
  10. package/dist/assets/improve-strategies/thorough.json +0 -3
  11. package/dist/assets/prompts/consolidate-pair.md +20 -0
  12. package/dist/assets/stash-skeleton/facts/conventions/backlinks.md +17 -19
  13. package/dist/assets/stash-skeleton/facts/conventions/domains.md +2 -2
  14. package/dist/assets/templates/html/health.html +3 -5
  15. package/dist/cli/retired-commands.js +1 -1
  16. package/dist/cli/unknown-flags.js +24 -1
  17. package/dist/cli.js +46 -1
  18. package/dist/commands/health/archive-usage.js +92 -0
  19. package/dist/commands/health/data-dir-usage.js +25 -13
  20. package/dist/commands/health/html-report.js +1 -4
  21. package/dist/commands/health/improve-metrics.js +25 -37
  22. package/dist/commands/health/md-report.js +1 -6
  23. package/dist/commands/health/report-view-model.js +4 -14
  24. package/dist/commands/health/windows.js +0 -1
  25. package/dist/commands/health.js +13 -0
  26. package/dist/commands/improve/consolidate/continuity-check.js +137 -0
  27. package/dist/commands/improve/consolidate/pair-pass.js +791 -0
  28. package/dist/commands/improve/consolidate.js +38 -63
  29. package/dist/commands/improve/extract-prompt.js +1 -2
  30. package/dist/commands/improve/improve-cli.js +1 -1
  31. package/dist/commands/improve/improve-strategies.js +23 -5
  32. package/dist/commands/improve/improve.js +19 -30
  33. package/dist/commands/improve/ledger.js +3 -2
  34. package/dist/commands/improve/loop-stages.js +5 -85
  35. package/dist/commands/improve/memory/memory-belief.js +3 -1
  36. package/dist/commands/improve/memory/memory-improve.js +262 -11
  37. package/dist/commands/improve/planner.js +0 -5
  38. package/dist/commands/improve/preparation.js +20 -135
  39. package/dist/commands/improve/retrieval-scope.js +19 -4
  40. package/dist/commands/improve/salience.js +1 -14
  41. package/dist/commands/improve/stage.js +0 -1
  42. package/dist/commands/lint/base-linter.js +19 -11
  43. package/dist/commands/proposal/drain.js +8 -1
  44. package/dist/commands/proposal/proposal-cli.js +16 -2
  45. package/dist/commands/proposal/proposal-types.js +7 -0
  46. package/dist/commands/proposal/proposal.js +37 -6
  47. package/dist/commands/proposal/repository.js +613 -4
  48. package/dist/commands/proposal/validators/proposals.js +9 -0
  49. package/dist/commands/read/knowledge.js +3 -2
  50. package/dist/commands/read/show.js +0 -14
  51. package/dist/commands/sources/info.js +122 -18
  52. package/dist/commands/sources/stash-cli.js +23 -3
  53. package/dist/core/bundle-rename.js +1 -7
  54. package/dist/core/config/config-schema.js +8 -1
  55. package/dist/core/config/config.js +23 -48
  56. package/dist/core/config/engine-semantics.js +0 -2
  57. package/dist/core/config/schema/improve-processes.js +17 -42
  58. package/dist/core/config/schema/index-config.js +5 -25
  59. package/dist/core/file-change.js +13 -5
  60. package/dist/core/improve-result.js +22 -6
  61. package/dist/core/improve-types.js +0 -1
  62. package/dist/core/loopback.js +7 -12
  63. package/dist/core/parse.js +13 -16
  64. package/dist/core/state/migrations.js +15 -0
  65. package/dist/core/time.js +0 -20
  66. package/dist/indexer/db/llm-cache.js +2 -2
  67. package/dist/indexer/ensure-index.js +2 -2
  68. package/dist/indexer/index-written-assets.js +2 -3
  69. package/dist/indexer/indexer.js +18 -418
  70. package/dist/indexer/passes/metadata.js +0 -19
  71. package/dist/indexer/walk/walker.js +3 -4
  72. package/dist/llm/client.js +8 -10
  73. package/dist/llm/embedders/remote.js +1 -2
  74. package/dist/llm/feature-gate.js +0 -5
  75. package/dist/output/shapes/helpers.js +20 -4
  76. package/dist/output/text/command-format.js +9 -8
  77. package/dist/output/text/proposal-format.js +47 -1
  78. package/dist/output/text/show-format.js +0 -20
  79. package/dist/scripts/akm-migrate-node.js +923 -950
  80. package/dist/scripts/akm-migrate.js +923 -950
  81. package/dist/setup/steps/connection.js +5 -6
  82. package/dist/setup/steps/platforms.js +2 -2
  83. package/dist/sources/providers/git-stash.js +83 -4
  84. package/dist/storage/repositories/improve-ledger-repository.js +48 -7
  85. package/dist/storage/repositories/index-connection.js +5 -2
  86. package/dist/storage/repositories/index-entries-repository.js +4 -7
  87. package/dist/storage/repositories/index-entry-schema.js +4 -2
  88. package/dist/storage/repositories/index-llm-cache-repository.js +7 -26
  89. package/dist/storage/repositories/index-schema.js +55 -104
  90. package/dist/storage/repositories/proposals-repository.js +61 -0
  91. package/dist/storage/repositories/salience-repository.js +1 -19
  92. package/docs/migration/README.md +1 -1
  93. package/docs/migration/release-notes/0.9.17.md +130 -41
  94. package/docs/migration/release-notes/README.md +7 -0
  95. package/docs/reference/cli.md +27 -21
  96. package/docs/reference/configuration.md +21 -12
  97. package/docs/reference/data-and-telemetry.md +0 -1
  98. package/package.json +1 -1
  99. package/schemas/akm-config.json +0 -342
  100. package/dist/assets/improve-strategies/graph-refresh.json +0 -15
  101. package/dist/assets/prompts/contradiction-judge.md +0 -33
  102. package/dist/assets/prompts/graph-extract-system.md +0 -1
  103. package/dist/assets/prompts/graph-extract-user-prompt.md +0 -35
  104. package/dist/assets/prompts/metadata-enhance-system.md +0 -1
  105. package/dist/assets/tasks/improve/akm-graph-refresh-weekly.yml +0 -4
  106. package/dist/indexer/db/graph-db.js +0 -399
  107. package/dist/indexer/graph/graph-extraction.js +0 -809
  108. package/dist/indexer/graph/graph-related.js +0 -131
  109. package/dist/indexer/graph/graph-types.js +0 -4
  110. package/dist/llm/graph-extract.js +0 -892
  111. package/dist/llm/metadata-enhance.js +0 -95
@@ -5,68 +5,36 @@ import fs from "node:fs";
5
5
  import path from "node:path";
6
6
  import { detectAdapterId } from "../core/adapter/detect-adapter.js";
7
7
  import { adapterForId } from "../core/adapter/registry.js";
8
- import { isHttpUrl, toErrorMessage } from "../core/common.js";
9
- import { concurrentMap } from "../core/concurrent.js";
10
- import { ConfigError } from "../core/errors.js";
11
- import { defaultConcurrencyForEndpoint } from "../core/loopback.js";
8
+ import { isHttpUrl } from "../core/common.js";
12
9
  import { classifyPathAccess, describeInaccessiblePath } from "../core/path-access.js";
13
10
  import { getDbPath } from "../core/paths.js";
14
11
  import { SCRIPT_EXTENSIONS } from "../core/recognition-util.js";
15
12
  import { withStateDb } from "../core/state-db.js";
16
13
  import { isVerbose, warn, warnOnce, warnVerbose } from "../core/warn.js";
17
- import { assertRunnerCredentials } from "../integrations/agent/runner-dispatch.js";
18
- import { isLlmFeatureEnabled } from "../llm/feature-gate.js";
19
- import { resolveIndexPassExecution } from "../llm/index-passes.js";
20
14
  import { resolveSourcesForOrigin } from "../registry/origin-resolve.js";
21
15
  import { closeDatabase, openExistingDatabase, openIndexDatabase, openReadonlyExistingDatabase, } from "../storage/repositories/index-connection.js";
22
16
  import { deleteEntriesByBundle, deleteEntriesByDirAndBundle, deleteEntriesByDirExceptRefs, deleteEntriesByIds, deleteUsageEventsByEntryIds, findEntryIdByRef, getAllEntries, getEmbeddableEntryCount, getEntryCount, getIndexedBundleIdsByDir, getIndexedDirPathsByBundleId, relinkUsageEvents, upsertEntry, } from "../storage/repositories/index-entries-repository.js";
23
- import { clearStaleCacheEntries, computeBodyHash, getLlmCacheEntry, upsertLlmCacheEntry, } from "../storage/repositories/index-llm-cache-repository.js";
17
+ import { clearStaleCacheEntries } from "../storage/repositories/index-llm-cache-repository.js";
24
18
  import { deleteIndexDirState, deleteMeta, getIndexDirState, getMeta, setMeta, upsertIndexDirState, } from "../storage/repositories/index-meta-repository.js";
25
19
  import { VACUUM_PENDING_META } from "../storage/repositories/index-schema.js";
26
20
  import { upsertUtilityScore } from "../storage/repositories/index-utility-repository.js";
27
21
  import { getEmbeddingCount } from "../storage/repositories/index-vec-repository.js";
28
22
  import { INDEX_DB_VACUUMED_EVENT, readFreelistInfo, vacuumIfReclaimable } from "../storage/state-db-integrity.js";
29
23
  import { assertIndexedWorkflowSourceIdentity, WorkflowSourceIdentityError } from "../workflows/source-files.js";
30
- import { deleteStoredGraph } from "./db/graph-db.js";
31
24
  import { reclassifyIndexDbContention } from "./index-db-contention.js";
32
25
  import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
33
26
  import { indexedPathMatchesOwner, resolveAdapterConceptOwner, } from "./lookup/adapter-concept-owner.js";
34
27
  import { generateEmbeddingsForDb } from "./materialize-embeddings.js";
35
28
  import { canUseIncrementalSkip, computeDirFingerprint, getCachedDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
36
- import { isEnrichmentComplete, isWorkflowSkipWarning, withFileSize, } from "./passes/metadata.js";
29
+ import { isWorkflowSkipWarning, withFileSize } from "./passes/metadata.js";
37
30
  import { drainDirDocuments } from "./scan/drain-dir.js";
38
31
  import { purgeOldUsageEvents, USAGE_EVENT_RETENTION_DAYS } from "./usage/usage-events.js";
39
32
  import { walkStashFlatWithStatus } from "./walk/walker.js";
40
- function collectLoweringNotices(target, notices) {
41
- const keys = new Set(target.map((notice) => JSON.stringify(notice)));
42
- for (const notice of notices) {
43
- const key = JSON.stringify(notice);
44
- if (keys.has(key))
45
- continue;
46
- keys.add(key);
47
- target.push(notice);
48
- }
49
- }
50
33
  function throwIfAborted(signal) {
51
34
  if (signal?.aborted) {
52
35
  throw signal.reason instanceof Error ? signal.reason : new Error("index interrupted");
53
36
  }
54
37
  }
55
- export function getDefaultLlmConcurrency(llmConfig) {
56
- if (typeof llmConfig?.concurrency === "number")
57
- return llmConfig.concurrency;
58
- // ONE classifier decides the local-vs-remote default (`core/loopback.ts`'s
59
- // `defaultConcurrencyForEndpoint`), shared with the embedding pool
60
- // (`resolveEmbeddingConcurrency`, `src/llm/embedders/remote.ts`) and the
61
- // workflow engine's frozen concurrency default.
62
- //
63
- // The explicit-override branch above only fires for callers that put
64
- // `concurrency` on the connection themselves — `engines.<name>.concurrency`
65
- // is a valid schema field but `resolveLlmEngineUse` does NOT copy it into
66
- // the resolved connection, so on the enrichment path the auto-derived 1/2
67
- // is what runs (see docs/architecture/internals/indexing.md).
68
- return defaultConcurrencyForEndpoint(llmConfig?.endpoint);
69
- }
70
38
  // ── Source ownership ─────────────────────────────────────────────────────────
71
39
  /** Each source's durable bundle component (`deriveInstallations`: one per source, source order). */
72
40
  function sourceComponents(sources) {
@@ -102,8 +70,8 @@ function parseStoredSourceOwners(raw) {
102
70
  }
103
71
  }
104
72
  /**
105
- * Sources removed (or moved) since the last complete run. Their entries and
106
- * graph rows are purged by {@link applyRemovedSources} once the walk completes.
73
+ * Sources removed (or moved) since the last complete run. Their entries are
74
+ * purged by {@link applyRemovedSources} once the walk completes.
107
75
  */
108
76
  function findRemovedSources(db, sources) {
109
77
  const currentByBundle = new Map(sourceOwners(sources).map((owner) => [owner.bundleId, owner]));
@@ -116,12 +84,9 @@ function findRemovedSources(db, sources) {
116
84
  }
117
85
  function applyRemovedSources(db, sources, removedSources, isIncremental) {
118
86
  const owners = sourceOwners(sources);
119
- const currentRoots = new Set(owners.map((owner) => owner.sourceRoot));
120
87
  for (const removed of removedSources) {
121
88
  if (removed.removeBundleEntries)
122
89
  deleteEntriesByBundle(db, removed.bundleId);
123
- if (!currentRoots.has(removed.sourceRoot))
124
- deleteStoredGraph(db, removed.sourceRoot);
125
90
  }
126
91
  // A full run re-drains every configured source, so any other bundle's rows
127
92
  // are stale even when no stored owner names them.
@@ -371,9 +336,9 @@ function detectAndPersistBundleAdapters(allSourceEntries, config, mutateConfig,
371
336
  }
372
337
  async function akmIndexReal(options) {
373
338
  // R-022: `dryRun` only ever gated the `--clean` stale-entry removal pass
374
- // (see `runCleanPass` below) — every other phase (walk, LLM enrichment,
375
- // embeddings, FTS, the adapter-detection config write) ran for real
376
- // regardless, so `akm index --dry-run` alone silently performed a full,
339
+ // (see `runCleanPass` below) — every other phase (walk, embeddings, FTS,
340
+ // the adapter-detection config write) ran for real regardless, so
341
+ // `akm index --dry-run` alone silently performed a full,
377
342
  // real index. The flag's own docs (`IndexOptions.dryRun` above, and the
378
343
  // CLI help in stash-cli.ts) already scope it to `--clean`; reject the
379
344
  // combination that was never implemented instead of quietly doing
@@ -433,8 +398,6 @@ async function akmIndexReal(options) {
433
398
  message: `Resolved ${sourceDirs.length} stash source${sourceDirs.length === 1 ? "" : "s"}.`,
434
399
  });
435
400
  const t0 = Date.now();
436
- const enrichmentExecution = resolveIndexPassExecution("enrichment", config);
437
- const loweringNotices = [...enrichmentExecution.notices];
438
401
  const dbPath = getDbPath();
439
402
  const db = openIndexDatabase(dbPath);
440
403
  try {
@@ -452,7 +415,6 @@ async function akmIndexReal(options) {
452
415
  sourcesCount: sourceDirs.length,
453
416
  semanticSearchMode: config.semanticSearchMode,
454
417
  embeddingProvider: getEmbeddingProvider(config.embedding),
455
- llmEnabled: !!enrichmentExecution.runner,
456
418
  }),
457
419
  });
458
420
  const removedSources = findRemovedSources(db, sources);
@@ -461,17 +423,10 @@ async function akmIndexReal(options) {
461
423
  const tWalkStart = Date.now();
462
424
  // Walk every source and pre-generate all metadata outside any transaction…
463
425
  const { dirRecords, scannedDirs, skippedDirs, generatedCount, warnings, complete } = await scanSourceDirs(db, sources, isIncremental, builtAtMs, removedSources.length > 0, onProgress, !clean);
464
- // …fail on a missing enrichment credential before anything is persisted…
465
- const bundleByRoot = buildIndexedSourceOwners(sources);
466
- const runner = enrichmentExecution.runner;
467
- if (runner &&
468
- isLlmFeatureEnabled(config, "metadata_enhance") &&
469
- dirRecordsNeedMetadataDispatch(db, dirRecords, bundleByRoot)) {
470
- assertRunnerCredentials(runner);
471
- }
472
426
  // …then write it in one transaction: `item_ref = <bundle>//<conceptId>`
473
427
  // plus canonical component/adapter provenance per source root.
474
- const { dirsNeedingLlm } = persistDirRecords(db, dirRecords, warnings, bundleByRoot);
428
+ const bundleByRoot = buildIndexedSourceOwners(sources);
429
+ persistDirRecords(db, dirRecords, warnings, bundleByRoot);
475
430
  onProgress({
476
431
  phase: "scan",
477
432
  message: `Scanned ${scannedDirs} ${scannedDirs === 1 ? "directory" : "directories"} and skipped ${skippedDirs}.`,
@@ -488,15 +443,9 @@ async function akmIndexReal(options) {
488
443
  }
489
444
  }
490
445
  const tWalkEnd = Date.now();
491
- // ── LLM enrichment for the directories that need it ───────────────────
492
- throwIfAborted(signal);
493
- await enhanceDirsWithLlm(db, config, enrichmentExecution, dirsNeedingLlm, onProgress, signal, (notices) => collectLoweringNotices(loweringNotices, notices));
494
- onProgress({
495
- phase: "llm",
496
- message: enrichmentExecution.runner
497
- ? `LLM enhancement reviewed ${dirsNeedingLlm.length} ${dirsNeedingLlm.length === 1 ? "directory" : "directories"}.`
498
- : "LLM enhancement disabled.",
499
- });
446
+ // Metadata enhancement (the LLM pass that used to run here) is retired
447
+ // (RS-D, 0.9.17-alpha.9) — see CHANGELOG. `timing.llmMs` stays (always
448
+ // ~0 now) so the JSON shape is unchanged.
500
449
  const tLlmEnd = Date.now();
501
450
  if (complete)
502
451
  applyRemovedSources(db, sources, removedSources, isIncremental);
@@ -553,7 +502,6 @@ async function akmIndexReal(options) {
553
502
  directoriesSkipped: skippedDirs,
554
503
  scanComplete: complete,
555
504
  ...(warnings.length > 0 ? { warnings } : {}),
556
- ...(loweringNotices.length > 0 ? { notices: Object.freeze([...loweringNotices]) } : {}),
557
505
  ...(Object.keys(persistedAdapters).length > 0 ? { configUpdated: { detectedAdapters: persistedAdapters } } : {}),
558
506
  verification,
559
507
  timing: {
@@ -582,52 +530,6 @@ function buildIndexedSourceOwners(sources) {
582
530
  return [path.resolve(source.path), { bundleId: id, componentId: id, adapterId: adapter }];
583
531
  }));
584
532
  }
585
- /**
586
- * The metadata-enrichment cache's body hash (`llm_enrichment_cache`, keyed by
587
- * item ref + this hash): the entry file's text, else its name and description.
588
- */
589
- function enrichmentBodyHash(entry, fileContent) {
590
- return computeBodyHash(fileContent ?? `${entry.name}\n${entry.description ?? ""}`);
591
- }
592
- /** Read-only mirror of the enrichment cache gate used before entry persistence. */
593
- function dirRecordsNeedMetadataDispatch(db, records, ownersByRoot) {
594
- for (const record of records) {
595
- if (record.skip || record.remove || !record.stash)
596
- continue;
597
- const owner = ownersByRoot.get(path.resolve(record.currentStashDir));
598
- if (!owner)
599
- throw new Error(`Missing bundle provenance for indexed source ${record.currentStashDir}`);
600
- for (const entry of record.stash.entries) {
601
- if (entry.quality !== "generated" || isEnrichmentComplete(entry))
602
- continue;
603
- const entryFile = entry.filename ? path.join(record.dirPath, entry.filename) : undefined;
604
- if (!entryFile)
605
- continue;
606
- const adapterConceptId = record.conceptIdByFile?.get(entryFile);
607
- if (!adapterConceptId)
608
- continue;
609
- let fileContent;
610
- try {
611
- fileContent = fs.readFileSync(entryFile, "utf8");
612
- }
613
- catch {
614
- // The dispatch path uses the same deterministic metadata fallback.
615
- }
616
- const bodyHash = enrichmentBodyHash(entry, fileContent);
617
- const cacheKey = deriveEntryProvenance(owner, entry.type, entry.name, adapterConceptId).itemRef;
618
- const cached = getLlmCacheEntry(db, cacheKey, bodyHash);
619
- if (!cached)
620
- return true;
621
- try {
622
- JSON.parse(cached.resultJson);
623
- }
624
- catch {
625
- return true;
626
- }
627
- }
628
- }
629
- return false;
630
- }
631
533
  function removalsFirst(records) {
632
534
  return [...records.filter((record) => record.remove), ...records.filter((record) => !record.remove)];
633
535
  }
@@ -1003,20 +905,15 @@ function requiresWorkflowSourcePreflight(ctxs) {
1003
905
  }
1004
906
  });
1005
907
  }
1006
- /**
1007
- * Phase 2 (sync): write all pre-generated scan records inside a single
1008
- * transaction, returning the directories that still need LLM enrichment.
1009
- */
908
+ /** Phase 2 (sync): write all pre-generated scan records inside a single transaction. */
1010
909
  function persistDirRecords(db, dirRecords, warnings, bundleByRoot) {
1011
- const dirsNeedingLlm = [];
1012
910
  // Per-source dedup: the same logical asset can appear more than once within
1013
911
  // one owning source, where source order still makes the first occurrence win.
1014
912
  // The owner is part of the key so identical concepts in different bundles
1015
913
  // remain distinct indexed rows.
1016
914
  const indexedAssetIdentities = new Set();
1017
915
  const deletedUsageEntryIds = new Set();
1018
- const findPersisted = db.prepare("SELECT id, content_hash, file_path, adapter_id, json_extract(document_json, '$.quality') AS quality " +
1019
- "FROM entries WHERE item_ref = ?");
916
+ const findPersisted = db.prepare("SELECT id, content_hash, file_path, adapter_id FROM entries WHERE item_ref = ?");
1020
917
  const insertTransaction = db.transaction(() => {
1021
918
  for (const { dirPath, currentStashDir, files, fingerprint, stash, skip, reason, persistedRowCount, hashByFile, conceptIdByFile, indexVariant, remove, } of dirRecords) {
1022
919
  const bundle = bundleByRoot.get(path.resolve(currentStashDir));
@@ -1055,15 +952,14 @@ function persistDirRecords(db, dirRecords, warnings, bundleByRoot) {
1055
952
  // re-reads its siblings), but on an incremental run a sibling whose
1056
953
  // content hash, path and adapter are unchanged since the last drain
1057
954
  // under the same adapter variant is already persisted exactly as this
1058
- // drain would persist it — including any LLM enrichment layered onto its
1059
- // row — so it is neither rewritten nor re-enriched. `--full` re-persists
955
+ // drain would persist it — including any metadata layered onto its row
956
+ // — so it is neither rewritten nor re-persisted. `--full` re-persists
1060
957
  // every entry.
1061
958
  const sameVariant = reason?.kind !== "full-rebuild" &&
1062
959
  indexVariant !== undefined &&
1063
960
  getIndexDirState(db, dirPath)?.indexVariant === indexVariant;
1064
961
  let persistedRows = 0;
1065
962
  let dedupedRows = 0;
1066
- const entriesToEnrich = [];
1067
963
  if (stash) {
1068
964
  const ownerIdentity = bundle.bundleId;
1069
965
  for (const entry of stash.entries) {
@@ -1100,19 +996,9 @@ function persistDirRecords(db, dirRecords, warnings, bundleByRoot) {
1100
996
  previous.file_path === entryPath &&
1101
997
  previous.adapter_id === bundle.adapterId;
1102
998
  if (unchanged) {
1103
- // An unchanged row that was never enriched still wants the LLM
1104
- // pass (the cache decides whether a call is needed).
1105
- if (entry.quality === "generated" && previous.quality !== "enriched")
1106
- entriesToEnrich.push(entry);
1107
999
  continue;
1108
1000
  }
1109
1001
  upsertEntry(db, entryPath, withFileSize(entry, entryPath), provenance, contentHash);
1110
- if (entry.quality === "generated")
1111
- entriesToEnrich.push(entry);
1112
- }
1113
- // Only "generated" entries (never user-curated ones) are enriched.
1114
- if (entriesToEnrich.length > 0) {
1115
- dirsNeedingLlm.push({ dirPath, files, currentStashDir, stash: { entries: entriesToEnrich } });
1116
1002
  }
1117
1003
  }
1118
1004
  // Prune the departed rows: everything under this dir NOT re-upserted above
@@ -1161,193 +1047,12 @@ function persistDirRecords(db, dirRecords, warnings, bundleByRoot) {
1161
1047
  });
1162
1048
  insertTransaction();
1163
1049
  deleteUsageEventsByEntryIds([...deletedUsageEntryIds]);
1164
- return { dirsNeedingLlm };
1165
- }
1166
- function indexedProvenanceForFile(db, filePath) {
1167
- const row = db
1168
- .prepare("SELECT item_ref AS itemRef, bundle_id AS bundleId, component_id AS componentId, " +
1169
- "concept_id AS conceptId, adapter_id AS adapterId FROM entries WHERE file_path = ? LIMIT 1")
1170
- .get(filePath);
1171
- if (!row?.itemRef || !row.bundleId || !row.componentId || !row.conceptId || !row.adapterId) {
1172
- throw new Error(`Missing indexed provenance for ${filePath}`);
1173
- }
1174
- return {
1175
- itemRef: row.itemRef,
1176
- bundleId: row.bundleId,
1177
- componentId: row.componentId,
1178
- conceptId: row.conceptId,
1179
- adapterId: row.adapterId,
1180
- };
1181
- }
1182
- async function enhanceDirsWithLlm(db, config, execution, dirsNeedingLlm, onProgress, signal, onNotices) {
1183
- // The invocation owns one frozen symbolic selection. Summary reporting and
1184
- // every enrichment dispatch consume this same snapshot.
1185
- const llmRunner = execution.runner;
1186
- if (!llmRunner || dirsNeedingLlm.length === 0)
1187
- return;
1188
- // Aggregate per-entry failures so a misconfigured LLM endpoint surfaces
1189
- // as a single visible warning instead of silently degrading every entry
1190
- // and leaving the user wondering why nothing got enhanced.
1191
- const summary = { attempted: 0, succeeded: 0, skipped: 0, failureSamples: [] };
1192
- let completedDirs = 0;
1193
- let completedEntries = 0;
1194
- const totalDirs = dirsNeedingLlm.length;
1195
- const totalEntries = dirsNeedingLlm.reduce((sum, { stash }) => sum + stash.entries.filter((e) => e.quality === "generated" && !isEnrichmentComplete(e)).length, 0);
1196
- // P3 — wall-clock budget for the enrichment pass. Defaults to the resolved
1197
- // engine's timeoutMs (or 10 minutes if not set). Users can extend it via
1198
- // `index.enrichment.timeoutMs` (or `index.defaults.timeoutMs`, or the
1199
- // engine's own `engines.<name>.timeoutMs`) — no separate knob needed.
1200
- const enrichDeadline = createEnrichmentDeadline(llmRunner.timeoutMs, totalEntries);
1201
- const enrichSignal = AbortSignal.any([signal, enrichDeadline].filter((s) => s !== undefined));
1202
- if (totalEntries > 0) {
1203
- onProgress({
1204
- phase: "llm",
1205
- message: `LLM enhancement starting for ${totalEntries} entr${totalEntries === 1 ? "y" : "ies"} ` +
1206
- `across ${totalDirs} director${totalDirs === 1 ? "y" : "ies"} (concurrency ${getDefaultLlmConcurrency(llmRunner.connection)}).`,
1207
- processed: 0,
1208
- total: totalEntries,
1209
- });
1210
- }
1211
- let currentDirLabel;
1212
- let configFailure;
1213
- let lastProgressAt = Date.now();
1214
- let heartbeatTimer;
1215
- if (totalEntries > 0) {
1216
- heartbeatTimer = setInterval(() => {
1217
- if (Date.now() - lastProgressAt < 15000)
1218
- return;
1219
- onProgress({
1220
- phase: "llm",
1221
- message: `Still enriching ${completedEntries}/${totalEntries} entr${totalEntries === 1 ? "y" : "ies"}` +
1222
- (currentDirLabel ? `; waiting on ${currentDirLabel}` : "") +
1223
- ".",
1224
- processed: completedEntries,
1225
- total: totalEntries,
1226
- });
1227
- lastProgressAt = Date.now();
1228
- }, 15000);
1229
- }
1230
- try {
1231
- await concurrentMap(dirsNeedingLlm, async ({ dirPath, files, currentStashDir, stash: originalStash }) => {
1232
- if (enrichSignal.aborted)
1233
- return undefined;
1234
- // Only enhance generated entries; user-provided overrides should not
1235
- // be overwritten. Skip entries that are already fully enriched
1236
- // (description + tags + searchHints).
1237
- const entriesToEnhance = originalStash.entries.filter((e) => {
1238
- if (e.quality !== "generated")
1239
- return false;
1240
- if (isEnrichmentComplete(e)) {
1241
- warnVerbose(`[akm] skipping LLM enrichment for "${e.name}" — entry already complete`);
1242
- return false;
1243
- }
1244
- return true;
1245
- });
1246
- if (entriesToEnhance.length === 0)
1247
- return undefined;
1248
- currentDirLabel = path.relative(currentStashDir, dirPath) || ".";
1249
- onProgress({
1250
- phase: "llm",
1251
- message: `Enhancing ${currentDirLabel} ` +
1252
- `(${entriesToEnhance.length} entr${entriesToEnhance.length === 1 ? "y" : "ies"}).`,
1253
- processed: completedEntries,
1254
- total: totalEntries,
1255
- });
1256
- lastProgressAt = Date.now();
1257
- const entryPathOf = (entry) => entry.filename ? path.join(dirPath, entry.filename) : files[0] || dirPath;
1258
- const itemRefs = entriesToEnhance.map((entry) => indexedProvenanceForFile(db, entryPathOf(entry)).itemRef);
1259
- let enhanced;
1260
- try {
1261
- enhanced = await enhanceEntriesWithLlm({
1262
- llmRunner,
1263
- entries: entriesToEnhance,
1264
- itemRefs,
1265
- files,
1266
- db,
1267
- config,
1268
- summary,
1269
- signal: enrichSignal,
1270
- onNotices,
1271
- onEntryDone: (event) => {
1272
- completedEntries++;
1273
- lastProgressAt = Date.now();
1274
- onProgress({
1275
- phase: "llm",
1276
- message: `Enhanced ${completedEntries}/${totalEntries} entr${totalEntries === 1 ? "y" : "ies"}; ` +
1277
- `${completedDirs}/${totalDirs} director${totalDirs === 1 ? "y" : "ies"} complete` +
1278
- (event.entryName ? `; current ${event.entryName}` : "") +
1279
- (currentDirLabel ? ` in ${currentDirLabel}` : "") +
1280
- (event.outcome === "cache-hit" ? " (cache hit)" : ""),
1281
- processed: completedEntries,
1282
- total: totalEntries,
1283
- });
1284
- },
1285
- });
1286
- }
1287
- catch (err) {
1288
- if (err instanceof ConfigError) {
1289
- configFailure ??= err;
1290
- return undefined;
1291
- }
1292
- throw err;
1293
- }
1294
- // Re-upsert the enhanced entries in a single transaction so a crash
1295
- // cannot leave half the entries updated and the rest stale.
1296
- db.transaction(() => {
1297
- for (const entry of enhanced) {
1298
- const entryPath = entryPathOf(entry);
1299
- const provenance = indexedProvenanceForFile(db, entryPath);
1300
- upsertEntry(db, entryPath, withFileSize(entry, entryPath), provenance);
1301
- }
1302
- })();
1303
- completedDirs++;
1304
- lastProgressAt = Date.now();
1305
- onProgress({
1306
- phase: "llm",
1307
- message: `Completed ${completedDirs}/${totalDirs} director${totalDirs === 1 ? "y" : "ies"}; ` +
1308
- `${completedEntries}/${totalEntries} entr${totalEntries === 1 ? "y" : "ies"} processed.`,
1309
- processed: completedEntries,
1310
- total: totalEntries,
1311
- });
1312
- return undefined;
1313
- },
1314
- // Defaults: 2 for remote LLM APIs, 1 for local model servers (LM
1315
- // Studio, Ollama run one inference at a time — parallel requests cause
1316
- // "Model reloaded" / 500 errors). No config override reaches this path:
1317
- // `resolveLlmEngineUse` does not forward `engines.<name>.concurrency`.
1318
- getDefaultLlmConcurrency(llmRunner.connection));
1319
- if (configFailure)
1320
- throw configFailure;
1321
- }
1322
- finally {
1323
- if (heartbeatTimer)
1324
- clearInterval(heartbeatTimer);
1325
- }
1326
- if (enrichDeadline?.aborted) {
1327
- warn("[akm] LLM enrichment budget exceeded. Re-run `akm index` to continue. Increase index.enrichment.timeoutMs for a larger budget.");
1328
- }
1329
- // Gate-closed (`skipped`) entries are not failures — exclude them so a
1330
- // deliberately disabled feature never surfaces as an enrichment error.
1331
- const failed = summary.attempted - summary.succeeded - summary.skipped;
1332
- if (failed > 0 && summary.succeeded === 0) {
1333
- const sample = summary.failureSamples.length ? ` Example: ${summary.failureSamples[0]}` : "";
1334
- warn(`LLM enhancement failed for all ${failed} attempted entries — index built without LLM enrichment.` +
1335
- ` Check llm.endpoint and llm.model in your config.${sample}`);
1336
- }
1337
- else if (failed > 0) {
1338
- const sample = summary.failureSamples.length ? ` Examples: ${summary.failureSamples.join("; ")}` : "";
1339
- warn(`LLM enhancement failed for ${failed}/${summary.attempted} entries — they were left un-enhanced.${sample}`);
1340
- }
1341
- }
1342
- export function createEnrichmentDeadline(timeoutMs, totalEntries) {
1343
- const perEntryTimeoutMs = timeoutMs === undefined ? 10 * 60 * 1000 : timeoutMs;
1344
- return perEntryTimeoutMs === null ? undefined : AbortSignal.timeout(perEntryTimeoutMs * Math.max(totalEntries, 1));
1345
1050
  }
1346
1051
  // ── Helpers ─────────────────────────────────────────────────────────────────
1347
1052
  function buildIndexSummaryMessage(options) {
1348
1053
  const stashSourceLabel = options.sourcesCount === 1 ? "stash source" : "stash sources";
1349
1054
  const semanticDetail = options.semanticSearchMode === "off" ? "disabled" : `${options.embeddingProvider} embeddings`;
1350
- return `Starting ${options.mode} index (${options.sourcesCount} ${stashSourceLabel}, semantic search: ${semanticDetail}, LLM: ${options.llmEnabled ? "enabled" : "disabled"}).`;
1055
+ return `Starting ${options.mode} index (${options.sourcesCount} ${stashSourceLabel}, semantic search: ${semanticDetail}).`;
1351
1056
  }
1352
1057
  function getEmbeddingProvider(embedding) {
1353
1058
  return isHttpUrl(embedding?.endpoint) ? "remote" : "local";
@@ -1381,111 +1086,6 @@ function verifyIndexState(db, config, embeddableEntries, embeddingResult) {
1381
1086
  ? "Check your embedding endpoint and credentials, then retry `akm index --full --verbose`."
1382
1087
  : "Retry `akm index --full --verbose`. If it still fails, confirm local model downloads are permitted and see docs/reference/configuration.md for local embedding dependency setup.");
1383
1088
  }
1384
- /**
1385
- * Enrich each entry through the LLM. `llm_enrichment_cache` (keyed by the
1386
- * canonical item ref + file body hash) skips the call when the body is
1387
- * unchanged.
1388
- */
1389
- async function enhanceEntriesWithLlm(args) {
1390
- const { llmRunner, entries, itemRefs, files, db, config, summary, signal, onEntryDone, onNotices } = args;
1391
- const { enhanceMetadata } = await import("../llm/metadata-enhance.js");
1392
- // failureSamples is bounded to 3 items, so a linear scan is cheaper than a
1393
- // parallel Set for membership checks (#177 review).
1394
- const recordFailure = (message) => {
1395
- if (summary.failureSamples.length < 3 && !summary.failureSamples.includes(message)) {
1396
- summary.failureSamples.push(message);
1397
- }
1398
- };
1399
- // The non-empty enrichment fields replace the entry's, and the entry is
1400
- // marked enriched so later runs skip it (P2).
1401
- const withEnrichment = (entry, metadata) => ({
1402
- ...entry,
1403
- ...(metadata.description ? { description: metadata.description } : {}),
1404
- ...(metadata.searchHints?.length ? { searchHints: metadata.searchHints } : {}),
1405
- ...(metadata.tags?.length ? { tags: metadata.tags } : {}),
1406
- quality: "enriched",
1407
- });
1408
- let configFailure;
1409
- const results = await concurrentMap(entries, async (entry, idx) => {
1410
- if (signal.aborted)
1411
- return entry;
1412
- summary.attempted++;
1413
- try {
1414
- const entryFile = entry.filename
1415
- ? (files.find((f) => path.basename(f) === entry.filename) ?? files[0])
1416
- : files[0];
1417
- let fileContent;
1418
- if (entryFile) {
1419
- try {
1420
- fileContent = fs.readFileSync(entryFile, "utf8");
1421
- }
1422
- catch {
1423
- warn(`Could not read file for LLM enrichment: ${entry.filename ?? entry.name}`);
1424
- }
1425
- }
1426
- const bodyHash = enrichmentBodyHash(entry, fileContent);
1427
- const cacheKey = itemRefs[idx];
1428
- if (!cacheKey)
1429
- throw new Error(`Missing canonical item ref for enrichment entry ${entry.name}.`);
1430
- const cached = getLlmCacheEntry(db, cacheKey, bodyHash);
1431
- if (cached) {
1432
- try {
1433
- const updated = withEnrichment(entry, JSON.parse(cached.resultJson));
1434
- summary.succeeded++;
1435
- onEntryDone({ entryName: entry.name, outcome: "cache-hit" });
1436
- return updated;
1437
- }
1438
- catch {
1439
- warn(`LLM enrichment cache entry corrupt for ${entry.name}; re-running enrichment`);
1440
- }
1441
- }
1442
- const outcome = await enhanceMetadata(llmRunner, entry, fileContent, signal, config, onNotices);
1443
- if (outcome.status !== "enriched") {
1444
- // Not a genuine LLM success: the gate was closed (`skipped`) or the
1445
- // call errored/timed out (`failed`). Do NOT mark the entry enriched
1446
- // and do NOT write the LLM cache — caching here would poison the
1447
- // entry into a permanent enrichment skip even though nothing was
1448
- // enhanced. Surface failures honestly; stay silent on gated-off skips.
1449
- if (outcome.status === "failed") {
1450
- recordFailure(outcome.error ?? "metadata enrichment failed");
1451
- onEntryDone({ entryName: entry.name, outcome: "failed" });
1452
- }
1453
- else {
1454
- summary.skipped++;
1455
- onEntryDone({ entryName: entry.name, outcome: "skipped" });
1456
- }
1457
- return entry;
1458
- }
1459
- // An empty-but-successful response is still cached: the LLM was paid
1460
- // for this body_hash and produced no improvements, so re-running would
1461
- // only re-pay for the same no-op.
1462
- const improvements = outcome.metadata;
1463
- upsertLlmCacheEntry(db, cacheKey, bodyHash, JSON.stringify({
1464
- description: improvements.description,
1465
- searchHints: improvements.searchHints,
1466
- tags: improvements.tags,
1467
- }));
1468
- summary.succeeded++;
1469
- onEntryDone({ entryName: entry.name, outcome: "llm" });
1470
- return withEnrichment(entry, improvements);
1471
- }
1472
- catch (err) {
1473
- if (err instanceof ConfigError) {
1474
- configFailure ??= err;
1475
- return entry;
1476
- }
1477
- recordFailure(toErrorMessage(err));
1478
- onEntryDone({ entryName: entry.name, outcome: "failed" });
1479
- return entry;
1480
- }
1481
- },
1482
- // Defaults: 2 for remote LLM APIs, 1 for local model servers. No config
1483
- // override reaches this path (see getDefaultLlmConcurrency).
1484
- getDefaultLlmConcurrency(llmRunner.connection));
1485
- if (configFailure)
1486
- throw configFailure;
1487
- return results.map((result, i) => result ?? entries[i]);
1488
- }
1489
1089
  // ── lookup ─────────────────────────────────────────────────────────────────
1490
1090
  import { makeBundleRef } from "../core/asset/asset-ref.js";
1491
1091
  import { conceptIdFromTypeName } from "../core/asset/resolve-ref.js";
@@ -743,25 +743,6 @@ function mergeAliases(existing, generated) {
743
743
  const merged = normalizeTerms([...(existing ?? []), ...generated]);
744
744
  return merged.length > 0 ? merged : undefined;
745
745
  }
746
- // ── Enrichment Completeness ─────────────────────────────────────────────────
747
- /**
748
- * Returns `true` when a stash entry already has enough LLM-quality metadata
749
- * that calling the LLM would produce no meaningful improvement.
750
- *
751
- * An entry is considered complete when ALL of the following hold:
752
- * - `description` is a non-empty string
753
- * - `tags` is a non-empty array
754
- * - `searchHints` is a non-empty array
755
- *
756
- * This predicate is used by `enhanceDirsWithLlm` to skip the LLM call for
757
- * entries that were previously enriched and already carry all three fields.
758
- */
759
- export function isEnrichmentComplete(entry) {
760
- const hasDescription = typeof entry.description === "string" && entry.description.trim().length > 0;
761
- const hasTags = Array.isArray(entry.tags) && entry.tags.length > 0;
762
- const hasSearchHints = Array.isArray(entry.searchHints) && entry.searchHints.length > 0;
763
- return hasDescription && hasTags && hasSearchHints;
764
- }
765
746
  // ── Native Markdown search projection ──────────────────────────────────────
766
747
  /**
767
748
  * Maximum native Markdown prose carried by the low-weight `content` field.
@@ -169,10 +169,9 @@ function isInsideGitRepo(dir) {
169
169
  /**
170
170
  * Recursively collect every `.md` file under `root`.
171
171
  *
172
- * Shared by graph-extraction and memory-inference so the walk logic lives in
173
- * exactly one place. Mirrors the `complete` tracking of the other walkers in
174
- * this file: a directory that cannot be read makes the result incomplete
175
- * instead of silently looking like a clean, empty scan.
172
+ * Used by memory-inference. Mirrors the `complete` tracking of the other
173
+ * walkers in this file: a directory that cannot be read makes the result
174
+ * incomplete instead of silently looking like a clean, empty scan.
176
175
  */
177
176
  export function walkMarkdownFiles(root) {
178
177
  const files = [];