akm-cli 0.9.0-beta.5 → 0.9.0-beta.51

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/CHANGELOG.md +711 -0
  2. package/README.md +12 -4
  3. package/dist/akm +38 -0
  4. package/dist/akm-migrate-storage +38 -0
  5. package/dist/assets/profiles/default.json +9 -4
  6. package/dist/assets/profiles/frequent.json +1 -1
  7. package/dist/assets/profiles/memory-focus.json +1 -1
  8. package/dist/assets/profiles/quick.json +1 -1
  9. package/dist/assets/profiles/synthesize.json +15 -0
  10. package/dist/assets/profiles/thorough.json +1 -1
  11. package/dist/assets/prompts/consolidate-system.md +23 -0
  12. package/dist/assets/prompts/contradiction-judge.md +33 -0
  13. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  14. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  15. package/dist/assets/prompts/extract-session.md +6 -2
  16. package/dist/assets/prompts/graph-extract-system.md +1 -0
  17. package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
  18. package/dist/assets/prompts/memory-infer-system.md +1 -0
  19. package/dist/assets/prompts/memory-infer-user.md +5 -0
  20. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  21. package/dist/assets/prompts/procedural-system.md +44 -0
  22. package/dist/assets/prompts/recombine-system.md +40 -0
  23. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  24. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  25. package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
  26. package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
  27. package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
  28. package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
  29. package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
  30. package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
  31. package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
  32. package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
  33. package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
  34. package/dist/assets/templates/html/health.html +281 -111
  35. package/dist/assets/wiki/ingest-workflow-template.md +38 -10
  36. package/dist/cli/parse-args.js +46 -1
  37. package/dist/cli/shared.js +28 -0
  38. package/dist/cli.js +27 -11
  39. package/dist/commands/agent/agent-dispatch.js +2 -2
  40. package/dist/commands/agent/agent-support.js +0 -7
  41. package/dist/commands/agent/contribute-cli.js +17 -4
  42. package/dist/commands/config-cli.js +18 -2
  43. package/dist/commands/env/child-env.js +47 -0
  44. package/dist/commands/env/env-cli.js +33 -26
  45. package/dist/commands/env/secret-cli.js +36 -22
  46. package/dist/commands/feedback-cli.js +15 -6
  47. package/dist/commands/graph/graph-cli.js +5 -13
  48. package/dist/commands/graph/graph.js +76 -72
  49. package/dist/commands/health/checks.js +49 -1
  50. package/dist/commands/health/html-report.js +422 -80
  51. package/dist/commands/health.js +386 -9
  52. package/dist/commands/improve/calibration.js +161 -0
  53. package/dist/commands/improve/consolidate/chunking.js +141 -0
  54. package/dist/commands/improve/consolidate/eligibility.js +81 -0
  55. package/dist/commands/improve/consolidate/merge.js +145 -0
  56. package/dist/commands/improve/consolidate/sanitize.js +231 -0
  57. package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
  58. package/dist/commands/improve/consolidate.js +635 -660
  59. package/dist/commands/improve/dedup.js +482 -0
  60. package/dist/commands/improve/distill.js +159 -69
  61. package/dist/commands/improve/eligibility.js +434 -0
  62. package/dist/commands/improve/encoding-salience.js +205 -0
  63. package/dist/commands/improve/extract-cli.js +124 -2
  64. package/dist/commands/improve/extract-prompt.js +39 -2
  65. package/dist/commands/improve/extract-watch.js +140 -0
  66. package/dist/commands/improve/extract.js +389 -40
  67. package/dist/commands/improve/feedback-valence.js +54 -0
  68. package/dist/commands/improve/homeostatic.js +467 -0
  69. package/dist/commands/improve/improve-auto-accept.js +138 -7
  70. package/dist/commands/improve/improve-cli.js +36 -61
  71. package/dist/commands/improve/improve-profiles.js +14 -0
  72. package/dist/commands/improve/improve-result-file.js +14 -25
  73. package/dist/commands/improve/improve-session.js +58 -0
  74. package/dist/commands/improve/improve.js +485 -2498
  75. package/dist/commands/improve/locks.js +154 -0
  76. package/dist/commands/improve/loop-stages.js +1083 -0
  77. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  78. package/dist/commands/improve/outcome-loop.js +256 -0
  79. package/dist/commands/improve/preparation.js +1966 -0
  80. package/dist/commands/improve/proactive-maintenance.js +115 -0
  81. package/dist/commands/improve/procedural.js +418 -0
  82. package/dist/commands/improve/recombine.js +850 -0
  83. package/dist/commands/improve/reflect-noise.js +0 -0
  84. package/dist/commands/improve/reflect.js +183 -40
  85. package/dist/commands/improve/salience.js +438 -0
  86. package/dist/commands/improve/triage.js +93 -0
  87. package/dist/commands/lint/agent-linter.js +19 -24
  88. package/dist/commands/lint/base-linter.js +173 -60
  89. package/dist/commands/lint/command-linter.js +19 -24
  90. package/dist/commands/lint/env-key-rules.js +38 -1
  91. package/dist/commands/lint/fact-linter.js +39 -0
  92. package/dist/commands/lint/index.js +31 -13
  93. package/dist/commands/lint/memory-linter.js +1 -1
  94. package/dist/commands/lint/registry.js +7 -2
  95. package/dist/commands/lint/task-linter.js +3 -3
  96. package/dist/commands/lint/workflow-linter.js +26 -1
  97. package/dist/commands/proposal/drain-policies.js +5 -0
  98. package/dist/commands/proposal/drain.js +43 -50
  99. package/dist/commands/proposal/proposal-cli.js +21 -31
  100. package/dist/commands/proposal/proposal.js +5 -0
  101. package/dist/commands/proposal/propose.js +7 -2
  102. package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
  103. package/dist/commands/proposal/validators/proposals.js +189 -63
  104. package/dist/commands/read/curate.js +414 -94
  105. package/dist/commands/read/knowledge.js +6 -3
  106. package/dist/commands/read/search-cli.js +9 -4
  107. package/dist/commands/read/search.js +10 -6
  108. package/dist/commands/read/show.js +86 -7
  109. package/dist/commands/sources/init.js +49 -17
  110. package/dist/commands/sources/installed-stashes.js +11 -3
  111. package/dist/commands/sources/schema-repair.js +43 -45
  112. package/dist/commands/sources/self-update.js +2 -2
  113. package/dist/commands/sources/source-add.js +7 -3
  114. package/dist/commands/sources/stash-cli.js +28 -40
  115. package/dist/commands/sources/stash-skeleton.js +23 -8
  116. package/dist/commands/tasks/tasks-cli.js +19 -27
  117. package/dist/commands/tasks/tasks.js +39 -11
  118. package/dist/commands/wiki-cli.js +21 -35
  119. package/dist/core/asset/asset-registry.js +3 -1
  120. package/dist/core/asset/asset-spec.js +18 -2
  121. package/dist/core/asset/frontmatter.js +166 -167
  122. package/dist/core/asset/markdown.js +8 -0
  123. package/dist/core/authoring-rules.js +92 -0
  124. package/dist/core/common.js +0 -5
  125. package/dist/core/config/config-migration.js +12 -11
  126. package/dist/core/config/config-schema.js +340 -56
  127. package/dist/core/config/config-types.js +3 -3
  128. package/dist/core/config/config.js +28 -7
  129. package/dist/core/events.js +3 -7
  130. package/dist/core/improve-types.js +11 -8
  131. package/dist/core/logs-db.js +10 -66
  132. package/dist/core/parse.js +36 -16
  133. package/dist/core/paths.js +3 -0
  134. package/dist/core/standards/resolve-standards-context.js +87 -0
  135. package/dist/core/standards/resolve-stash-standards.js +99 -0
  136. package/dist/core/standards/resolve-type-conventions.js +66 -0
  137. package/dist/core/state/migrations.js +714 -0
  138. package/dist/core/state-db.js +525 -474
  139. package/dist/indexer/db/db.js +439 -247
  140. package/dist/indexer/db/graph-db.js +129 -86
  141. package/dist/indexer/ensure-index.js +152 -17
  142. package/dist/indexer/graph/graph-boost.js +51 -41
  143. package/dist/indexer/graph/graph-extraction.js +218 -4
  144. package/dist/indexer/index-writer-lock.js +99 -0
  145. package/dist/indexer/indexer.js +123 -221
  146. package/dist/indexer/passes/dir-staleness.js +114 -0
  147. package/dist/indexer/passes/memory-inference.js +13 -5
  148. package/dist/indexer/passes/staleness-detect.js +2 -5
  149. package/dist/indexer/search/db-search.js +19 -6
  150. package/dist/indexer/search/ranking-contributors.js +22 -0
  151. package/dist/indexer/search/ranking.js +4 -0
  152. package/dist/indexer/search/search-source.js +17 -18
  153. package/dist/indexer/search/semantic-status.js +4 -0
  154. package/dist/indexer/walk/matchers.js +9 -0
  155. package/dist/integrations/agent/config.js +6 -53
  156. package/dist/integrations/agent/index.js +2 -18
  157. package/dist/integrations/agent/prompts.js +75 -9
  158. package/dist/integrations/agent/runner-dispatch.js +59 -0
  159. package/dist/integrations/harnesses/claude/session-log.js +11 -1
  160. package/dist/integrations/harnesses/index.js +2 -3
  161. package/dist/integrations/harnesses/opencode/session-log.js +173 -3
  162. package/dist/integrations/harnesses/opencode-sdk/index.js +2 -2
  163. package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +0 -2
  164. package/dist/integrations/session-logs/index.js +16 -0
  165. package/dist/llm/client.js +45 -15
  166. package/dist/llm/embedder.js +42 -3
  167. package/dist/llm/embedders/deterministic.js +66 -0
  168. package/dist/llm/embedders/local.js +66 -2
  169. package/dist/llm/feature-gate.js +8 -4
  170. package/dist/llm/graph-extract.js +67 -44
  171. package/dist/llm/memory-infer-impl.js +138 -0
  172. package/dist/llm/memory-infer.js +1 -127
  173. package/dist/llm/metadata-enhance.js +44 -31
  174. package/dist/llm/structured-call.js +49 -0
  175. package/dist/migrate-storage-node.mjs +8 -0
  176. package/dist/output/context.js +5 -5
  177. package/dist/output/renderers.js +74 -2
  178. package/dist/output/shapes/curate.js +14 -2
  179. package/dist/output/shapes/passthrough.js +0 -1
  180. package/dist/output/text/helpers.js +16 -1
  181. package/dist/registry/providers/skills-sh.js +21 -147
  182. package/dist/registry/providers/static-index.js +15 -157
  183. package/dist/registry/resolve.js +22 -9
  184. package/dist/runtime.js +25 -1
  185. package/dist/scripts/migrate-storage.js +2617 -1961
  186. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +759 -510
  187. package/dist/setup/setup.js +29 -8
  188. package/dist/sources/include.js +6 -2
  189. package/dist/sources/providers/filesystem.js +0 -1
  190. package/dist/sources/providers/git-install.js +210 -0
  191. package/dist/sources/providers/git-provider.js +234 -0
  192. package/dist/sources/providers/git-stash.js +248 -0
  193. package/dist/sources/providers/git.js +10 -661
  194. package/dist/sources/providers/npm.js +2 -6
  195. package/dist/sources/providers/provider-utils.js +13 -7
  196. package/dist/sources/providers/sync-from-ref.js +9 -1
  197. package/dist/sources/providers/tar-utils.js +16 -8
  198. package/dist/sources/providers/website.js +9 -5
  199. package/dist/sources/website-ingest.js +187 -29
  200. package/dist/sources/wiki-fetchers/registry.js +53 -0
  201. package/dist/sources/wiki-fetchers/youtube.js +239 -0
  202. package/dist/storage/database.js +45 -10
  203. package/dist/storage/managed-db.js +82 -0
  204. package/dist/storage/repositories/registry-cache.js +92 -0
  205. package/dist/storage/sqlite-pragmas.js +146 -0
  206. package/dist/tasks/backends/cron.js +1 -1
  207. package/dist/tasks/backends/launchd.js +1 -1
  208. package/dist/tasks/backends/schtasks.js +1 -1
  209. package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
  210. package/dist/tasks/runner.js +5 -13
  211. package/dist/text-import-hook.mjs +0 -0
  212. package/dist/wiki/wiki.js +37 -0
  213. package/dist/workflows/db.js +3 -4
  214. package/dist/workflows/runtime/runs.js +1 -117
  215. package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
  216. package/dist/workflows/validate-summary.js +2 -7
  217. package/docs/data-and-telemetry.md +3 -2
  218. package/docs/migration/release-notes/0.9.0.md +39 -0
  219. package/package.json +13 -11
  220. package/dist/commands/db-cli.js +0 -23
  221. package/dist/indexer/db/db-backup.js +0 -376
@@ -241,13 +241,18 @@ export function collectGraphRelatedHit(context, filePath) {
241
241
  * Find graph files that share entities with the given file.
242
242
  *
243
243
  * Implementation: SQL self-join on graph_file_entities, scoped by stash_root,
244
- * grouped by entry_id, ordered by shared-entity count desc. Touches ~50-200
244
+ * grouped by file_path, ordered by shared-entity count desc. Touches ~50-200
245
245
  * rows instead of loading the entire snapshot into memory. Cold-call latency
246
246
  * drops from ~30-60ms (full snapshot parse) to ~2-5ms on typical stashes.
247
247
  *
248
+ * #624-P1: the graph tables are keyed on (stash_root, file_path, body_hash) —
249
+ * NOT entries.id — so candidates are identified by file_path (the unique index
250
+ * idx_graph_files_path guarantees one graph_files row per path).
251
+ *
248
252
  * The returned `ref` field carries the canonical asset ref (`type:name`)
249
- * resolved from entries.entry_key when the entry is indexed. Callers should
250
- * fall back to formatting `path` when `ref` is undefined (orphan graph row).
253
+ * resolved from entries.entry_key when the file is indexed. Callers should
254
+ * fall back to formatting `path` when `ref` is undefined (graph row with no
255
+ * matching entries row).
251
256
  */
252
257
  export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
253
258
  if (!db) {
@@ -255,113 +260,118 @@ export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
255
260
  // callers pass a handle), so degrade to empty rather than reopening.
256
261
  return [];
257
262
  }
258
- // Resolve target's entry_id from the stash_root + file_path. The graph rows
259
- // are keyed on entry_id; without it we can't run the join.
260
- let targetEntryId;
263
+ // Confirm the target file has a graph row; without it there is nothing to
264
+ // relate. (Identity is file_path within the stash one row per path.)
261
265
  try {
262
266
  const row = db
263
- .prepare("SELECT entry_id FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
267
+ .prepare("SELECT 1 AS present FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
264
268
  .get(stashRoot, filePath);
265
- targetEntryId = row?.entry_id;
269
+ if (row === undefined)
270
+ return [];
266
271
  }
267
272
  catch {
268
273
  return [];
269
274
  }
270
- if (targetEntryId == null)
271
- return [];
272
275
  const effectiveLimit = Math.max(1, limit);
273
- // Shared-entity count per candidate entry_id.
276
+ // Shared-entity count per candidate file_path. The target's entities are the
277
+ // rows for `filePath`; candidates are any OTHER file_path in the stash that
278
+ // shares a normalized entity.
274
279
  let candidateRows;
275
280
  try {
276
281
  candidateRows = db
277
- .prepare(`SELECT gf.entry_id AS entry_id,
278
- gf.file_path AS file_path,
282
+ .prepare(`SELECT gf.file_path AS file_path,
279
283
  gf.file_type AS file_type,
280
284
  COUNT(*) AS shared
281
285
  FROM graph_file_entities target
282
286
  JOIN graph_file_entities e
283
287
  ON e.stash_root = target.stash_root
284
288
  AND e.entity_norm = target.entity_norm
285
- AND e.entry_id != target.entry_id
289
+ AND e.file_path != target.file_path
286
290
  JOIN graph_files gf
287
- ON gf.entry_id = e.entry_id
288
- WHERE target.entry_id = ?
291
+ ON gf.stash_root = e.stash_root
292
+ AND gf.file_path = e.file_path
293
+ AND gf.body_hash = e.body_hash
294
+ WHERE target.file_path = ?
289
295
  AND target.stash_root = ?
290
- GROUP BY gf.entry_id
296
+ GROUP BY gf.file_path
291
297
  ORDER BY shared DESC, gf.file_path ASC
292
298
  LIMIT ?`)
293
- .all(targetEntryId, stashRoot, effectiveLimit);
299
+ .all(filePath, stashRoot, effectiveLimit);
294
300
  }
295
301
  catch {
296
302
  return [];
297
303
  }
298
304
  if (candidateRows.length === 0)
299
305
  return [];
300
- const candidateIds = candidateRows.map((r) => r.entry_id);
301
- const placeholders = candidateIds.map(() => "?").join(",");
306
+ const candidatePaths = candidateRows.map((r) => r.file_path);
307
+ const placeholders = candidatePaths.map(() => "?").join(",");
302
308
  // Pull the shared entity names (joined by normalized casing) for display.
303
309
  const sharedRows = db
304
- .prepare(`SELECT e.entry_id AS entry_id, e.entity AS entity
310
+ .prepare(`SELECT e.file_path AS file_path, e.entity AS entity
305
311
  FROM graph_file_entities e
306
312
  JOIN graph_file_entities target
307
313
  ON target.stash_root = e.stash_root
308
314
  AND target.entity_norm = e.entity_norm
309
- WHERE e.entry_id IN (${placeholders})
310
- AND target.entry_id = ?
315
+ WHERE e.file_path IN (${placeholders})
316
+ AND e.stash_root = ?
317
+ AND target.file_path = ?
311
318
  AND target.stash_root = ?`)
312
- .all(...candidateIds, targetEntryId, stashRoot);
313
- const sharedByEntry = new Map();
319
+ .all(...candidatePaths, stashRoot, filePath, stashRoot);
320
+ const sharedByPath = new Map();
314
321
  for (const row of sharedRows) {
315
- let bucket = sharedByEntry.get(row.entry_id);
322
+ let bucket = sharedByPath.get(row.file_path);
316
323
  if (!bucket) {
317
324
  bucket = new Set();
318
- sharedByEntry.set(row.entry_id, bucket);
325
+ sharedByPath.set(row.file_path, bucket);
319
326
  }
320
327
  bucket.add(row.entity);
321
328
  }
322
329
  // Relation count for each candidate (relations where either endpoint
323
330
  // matches one of the shared entities).
324
- const relationCountByEntry = new Map();
331
+ const relationCountByPath = new Map();
325
332
  const relationRows = db
326
- .prepare(`SELECT entry_id, from_entity, to_entity
333
+ .prepare(`SELECT file_path, from_entity, to_entity
327
334
  FROM graph_file_relations
328
- WHERE entry_id IN (${placeholders})`)
329
- .all(...candidateIds);
335
+ WHERE file_path IN (${placeholders})
336
+ AND stash_root = ?`)
337
+ .all(...candidatePaths, stashRoot);
330
338
  for (const row of relationRows) {
331
- const shared = sharedByEntry.get(row.entry_id);
339
+ const shared = sharedByPath.get(row.file_path);
332
340
  if (!shared)
333
341
  continue;
334
342
  if (shared.has(row.from_entity) || shared.has(row.to_entity)) {
335
- relationCountByEntry.set(row.entry_id, (relationCountByEntry.get(row.entry_id) ?? 0) + 1);
343
+ relationCountByPath.set(row.file_path, (relationCountByPath.get(row.file_path) ?? 0) + 1);
336
344
  }
337
345
  }
338
346
  // Optional: ref lookup via entries.entry_key. entry_key is stored as
339
347
  // `${stash_dir}:${type}:${name}` — strip the stash-dir prefix to get the
340
- // user-facing `type:name`.
341
- const refByEntryId = new Map();
348
+ // user-facing `type:name`. Resolve by (stash_dir, file_path) now that the
349
+ // graph rows are no longer keyed on entries.id.
350
+ const refByPath = new Map();
342
351
  try {
343
352
  const entryRows = db
344
- .prepare(`SELECT id, entry_key, stash_dir FROM entries WHERE id IN (${placeholders})`)
345
- .all(...candidateIds);
353
+ .prepare(`SELECT entry_key, stash_dir, file_path FROM entries
354
+ WHERE file_path IN (${placeholders}) AND stash_dir = ?`)
355
+ .all(...candidatePaths, stashRoot);
346
356
  for (const row of entryRows) {
347
357
  const ref = stripStashPrefix(row.entry_key, row.stash_dir);
348
358
  if (ref)
349
- refByEntryId.set(row.id, ref);
359
+ refByPath.set(row.file_path, ref);
350
360
  }
351
361
  }
352
362
  catch {
353
363
  /* ignore — refs are best-effort */
354
364
  }
355
365
  return candidateRows.map((row) => {
356
- const sharedSet = sharedByEntry.get(row.entry_id) ?? new Set();
366
+ const sharedSet = sharedByPath.get(row.file_path) ?? new Set();
357
367
  const sharedEntities = [...sharedSet].sort((a, b) => a.localeCompare(b));
358
- const ref = refByEntryId.get(row.entry_id);
368
+ const ref = refByPath.get(row.file_path);
359
369
  return {
360
370
  ...(ref ? { ref } : {}),
361
371
  path: row.file_path,
362
372
  type: row.file_type,
363
373
  sharedEntities,
364
- relationCount: relationCountByEntry.get(row.entry_id) ?? 0,
374
+ relationCount: relationCountByPath.get(row.file_path) ?? 0,
365
375
  };
366
376
  });
367
377
  }
@@ -40,13 +40,14 @@ import path from "node:path";
40
40
  import { TYPE_DIRS } from "../../core/asset/asset-spec.js";
41
41
  import { parseFrontmatter } from "../../core/asset/frontmatter.js";
42
42
  import { concurrentMap } from "../../core/concurrent.js";
43
- import { getIndexPassConfig, resolveBatchSize } from "../../core/config/config.js";
43
+ import { getIndexPassConfig, loadConfig, resolveBatchSize } from "../../core/config/config.js";
44
+ import { rethrowIfTestIsolationError } from "../../core/errors.js";
44
45
  import { warn, warnVerbose } from "../../core/warn.js";
45
46
  import { isProcessEnabled } from "../../llm/feature-gate.js";
46
47
  import * as graphExtract from "../../llm/graph-extract.js";
47
48
  import { resolveIndexPassLLM } from "../../llm/index-passes.js";
48
49
  import { computeBodyHash, GRAPH_SCHEMA_VERSION, getLlmCacheEntriesByRefs, getLlmCacheEntry, upsertLlmCacheEntry, } from "../db/db.js";
49
- import { loadStoredGraphSnapshot, replaceStoredGraph } from "../db/graph-db.js";
50
+ import { drainExtractionQueue, loadStoredGraphSnapshot, replaceStoredGraph } from "../db/graph-db.js";
50
51
  import { walkMarkdownFiles } from "../walk/walker.js";
51
52
  import { deduplicateGraph } from "./graph-dedup.js";
52
53
  /** Schema version for the persisted artifact — bumps trigger a full rebuild. */
@@ -92,6 +93,12 @@ function computeGraphQualityTelemetry(consideredFiles, extractedFiles, entityCou
92
93
  };
93
94
  }
94
95
  export const DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES = ["memory", "knowledge"];
96
+ /**
97
+ * Max number of lazy-extraction queue rows drained per pass (#624-P3). Bounds
98
+ * per-run work so a large backlog is spread across runs rather than processed
99
+ * all at once. Generous default — the queue is normally near-empty.
100
+ */
101
+ const GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT = 100;
95
102
  const SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES = new Set([
96
103
  "memory",
97
104
  "knowledge",
@@ -291,8 +298,29 @@ export async function runGraphExtractionPass(ctx) {
291
298
  warnVerbose("graph extraction: skipped because no primary stash source is available.");
292
299
  return { ...EMPTY_RESULT };
293
300
  }
301
+ // #624-P3: drain the lazy-extraction queue BEFORE the ranked sweep, highest
302
+ // priority first. Queued paths are extracted individually (per-file merge,
303
+ // other files untouched) so they are processed even when they fall outside
304
+ // the normal candidate set. Default (empty queue) is a byte-identical no-op:
305
+ // drainExtractionQueue returns [] and the loop body never runs.
306
+ if (db) {
307
+ const drained = drainExtractionQueue(db, primary.path, GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT);
308
+ for (const queued of drained) {
309
+ if (signal?.aborted)
310
+ break;
311
+ await extractGraphForSingleFile(db, primary.path, queued.filePath, queued.bodyHash, { config, signal });
312
+ }
313
+ }
294
314
  const includeTypes = getGraphExtractionIncludeTypes(config);
295
- const eligible = collectEligibleFiles(primary.path, includeTypes).filter((candidate) => !options.candidatePaths || options.candidatePaths.has(candidate.absPath));
315
+ let eligible = collectEligibleFiles(primary.path, includeTypes).filter((candidate) => !options.candidatePaths || options.candidatePaths.has(candidate.absPath));
316
+ // P2 (#624): when topN is set and a DB is available, rank the (already
317
+ // candidate-filtered) eligible set by utility_scores DESC and keep only the
318
+ // top-N. Default (topN unset) is byte-identical to today — no ranking query
319
+ // is issued and the eligible set is untouched. Ranking composes WITH the
320
+ // candidatePaths filter: scoped-then-ranked-then-sliced.
321
+ if (db && options.topN != null && options.topN >= 0) {
322
+ eligible = rankCandidatesByUtility(db, eligible, primary.path).slice(0, options.topN);
323
+ }
296
324
  const considered = eligible.length;
297
325
  if (considered === 0) {
298
326
  const scoped = options.candidatePaths ? ` matching ${options.candidatePaths.size} candidate path(s)` : "";
@@ -344,6 +372,7 @@ export async function runGraphExtractionPass(ctx) {
344
372
  failureCount: 0,
345
373
  htmlErrorCount: 0,
346
374
  retryAttempts: 0,
375
+ nonArrayBatchFailures: 0,
347
376
  };
348
377
  const canReusePreviousGraph = previousGraph.telemetry?.extractorId === extractorId;
349
378
  const runtimeTelemetry = {
@@ -597,6 +626,7 @@ export async function runGraphExtractionPass(ctx) {
597
626
  telemetry.failureCount = runtimeTelemetry.failureCount ?? 0;
598
627
  telemetry.htmlErrorCount = runtimeTelemetry.htmlErrorCount ?? 0;
599
628
  telemetry.retryAttempts = runtimeTelemetry.retryAttempts ?? 0;
629
+ telemetry.nonArrayBatchFailures = runtimeTelemetry.nonArrayBatchFailures ?? 0;
600
630
  const qualityConsidered = mergedNodes.length;
601
631
  const qualityExtracted = mergedNodes.filter((node) => node.status === "extracted" && node.entities.length > 0).length;
602
632
  const quality = computeGraphQualityTelemetry(qualityConsidered, qualityExtracted, deduped.entities.length, deduped.relations.length);
@@ -628,6 +658,178 @@ export async function runGraphExtractionPass(ctx) {
628
658
  warnings,
629
659
  };
630
660
  }
661
+ /**
662
+ * Infer the asset type (`memory`, `knowledge`, …) for a path from the stash
663
+ * directory segment it lives under. Returns the matching include-type, or
664
+ * `undefined` when the path is not under a known graph-eligible type dir.
665
+ */
666
+ function inferGraphTypeForPath(stashRoot, absPath) {
667
+ const rel = path.relative(stashRoot, absPath);
668
+ const firstSeg = rel.split(path.sep)[0];
669
+ if (!firstSeg)
670
+ return undefined;
671
+ for (const type of SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES) {
672
+ if (TYPE_DIRS[type] === firstSeg)
673
+ return type;
674
+ }
675
+ return undefined;
676
+ }
677
+ /**
678
+ * #624-P3 — extract graph data for a SINGLE file and merge it into the stored
679
+ * graph WITHOUT clobbering other files' rows.
680
+ *
681
+ * Re-reads the body from disk at call time (the queued body_hash is NOT trusted
682
+ * blindly — the file may have been deleted or changed since enqueue) and skips
683
+ * silently (returns `false`) when the file is gone or empty. Resolves the LLM
684
+ * via {@link resolveIndexPassLLM} (model-available guard: returns `false` when
685
+ * no provider is configured) UNLESS `opts.llmOverride` is supplied, in which
686
+ * case the override is the extractor seam (used by tests and by callers that
687
+ * already hold a resolved model).
688
+ *
689
+ * Returns `true` when a graph row was written for the file, `false` on any
690
+ * skip (missing file, empty body, unknown type, no model, or extraction error).
691
+ */
692
+ export async function extractGraphForSingleFile(db, stashRoot, filePath, bodyHash, opts) {
693
+ try {
694
+ // Re-read from disk — never trust a stale queued body.
695
+ let raw;
696
+ try {
697
+ raw = fs.readFileSync(filePath, "utf8");
698
+ }
699
+ catch {
700
+ return false; // file gone / unreadable → silent skip
701
+ }
702
+ const parsed = parseFrontmatter(raw);
703
+ const body = parsed.content.trim();
704
+ if (!body)
705
+ return false;
706
+ const type = inferGraphTypeForPath(stashRoot, filePath) ?? "memory";
707
+ const effectiveHash = bodyHash ?? computeBodyHash(body);
708
+ // Extract — via the injected seam, or the real per-asset path.
709
+ let extraction;
710
+ if (opts?.llmOverride) {
711
+ const out = await opts.llmOverride(body);
712
+ extraction = {
713
+ entities: out.entities,
714
+ relations: out.relations,
715
+ ...(out.confidence !== undefined ? { confidence: out.confidence } : {}),
716
+ };
717
+ }
718
+ else {
719
+ const config = opts?.config ?? loadConfig();
720
+ if (!isProcessEnabled("index", "graph_extraction", config))
721
+ return false;
722
+ const llmConfig = resolveIndexPassLLM("graph", config);
723
+ if (!llmConfig)
724
+ return false; // model-available guard
725
+ const result = await graphExtract.extractGraphFromBody(llmConfig, body, opts?.signal, config);
726
+ extraction = {
727
+ entities: result.entities,
728
+ relations: result.relations,
729
+ ...(result.confidence !== undefined ? { confidence: result.confidence } : {}),
730
+ };
731
+ }
732
+ const entities = [...new Set(extraction.entities.map((e) => e.trim()).filter(Boolean))];
733
+ const relations = extraction.relations
734
+ .map((r) => ({
735
+ from: r.from.trim(),
736
+ to: r.to.trim(),
737
+ ...(r.type ? { type: r.type.trim() } : {}),
738
+ ...(normalizeConfidence(r.confidence) !== undefined ? { confidence: normalizeConfidence(r.confidence) } : {}),
739
+ }))
740
+ .filter((r) => r.from && r.to);
741
+ const node = {
742
+ path: filePath,
743
+ type,
744
+ bodyHash: effectiveHash,
745
+ entities,
746
+ relations,
747
+ ...(normalizeConfidence(extraction.confidence) !== undefined
748
+ ? { confidence: normalizeConfidence(extraction.confidence) }
749
+ : {}),
750
+ status: entities.length > 0 ? "extracted" : "empty",
751
+ reason: entities.length > 0 ? "none" : "no_graph_content",
752
+ extractionRunId: crypto.randomUUID(),
753
+ };
754
+ // Merge with the previously-stored nodes, scoping the refresh to JUST this
755
+ // path so other files' rows are preserved (and graph_meta counts refresh).
756
+ const previousGraph = loadGraphFile(stashRoot, db);
757
+ const candidatePaths = new Set([filePath]);
758
+ const mergedNodes = mergeGraphNodes(previousGraph.files, [node], candidatePaths);
759
+ const assetRefs = mergedNodes.map((n) => n.path);
760
+ const deduped = deduplicateGraph(mergedNodes.map((n) => ({ entities: n.entities, relations: n.relations })), assetRefs);
761
+ const qualityExtracted = mergedNodes.filter((n) => n.status === "extracted" && n.entities.length > 0).length;
762
+ const quality = computeGraphQualityTelemetry(mergedNodes.length, qualityExtracted, deduped.entities.length, deduped.relations.length);
763
+ const graph = {
764
+ schemaVersion: GRAPH_FILE_SCHEMA_VERSION,
765
+ generatedAt: new Date().toISOString(),
766
+ stashRoot,
767
+ files: mergedNodes,
768
+ entities: deduped.entities,
769
+ relations: deduped.relations,
770
+ quality,
771
+ ...(previousGraph.telemetry ? { telemetry: previousGraph.telemetry } : {}),
772
+ };
773
+ return writeGraphFile(stashRoot, graph, db);
774
+ }
775
+ catch (err) {
776
+ rethrowIfTestIsolationError(err);
777
+ return false;
778
+ }
779
+ }
780
+ // ── Eligible-file detection ─────────────────────────────────────────────────
781
+ /**
782
+ * Rank eligible graph-extraction candidates by their entry `utility_scores`,
783
+ * highest first, for the incremental high-signal-first sweep (P2 of #624).
784
+ *
785
+ * The join is READ-ONLY (`entries.file_path = candidate.absPath`, then
786
+ * `entries.id -> utility_scores.entry_id`) and does NOT re-couple the graph
787
+ * rows to `entries`. It reads the GLOBAL `utility_scores` table (not the
788
+ * per-scope `utility_scores_scoped`), so ranking is corpus-wide; `stashRoot`
789
+ * is accepted for call-site symmetry/future scoping but is not used to filter
790
+ * (the global table has no `stash_root` column).
791
+ *
792
+ * Candidates with no matching `entries` row, or an entry with no
793
+ * `utility_scores` row, get an effective utility of 0 (LEFT JOIN + COALESCE)
794
+ * and sort LAST — they are deprioritized, never dropped, so a `topN >= total`
795
+ * slice still includes them and they remain reachable on later runs.
796
+ *
797
+ * Ties (equal utility) break by `file_path` ASC for deterministic output.
798
+ * Returns a NEW array; the input is not mutated. SQLite's ~999 bound-parameter
799
+ * cap is respected by chunking the `IN (...)` lookup at 500.
800
+ *
801
+ * Exported for direct unit testing.
802
+ */
803
+ export function rankCandidatesByUtility(db, candidates, _stashRoot) {
804
+ // Cannot rank without a DB → return the input unranked rather than throw.
805
+ // Keeps the DB-less code path (reuse-from-memory) working when topN is set.
806
+ if (!db || candidates.length === 0)
807
+ return candidates;
808
+ const utilityByPath = new Map();
809
+ const CHUNK = 500;
810
+ for (let start = 0; start < candidates.length; start += CHUNK) {
811
+ const chunk = candidates.slice(start, start + CHUNK);
812
+ const paths = chunk.map((c) => c.absPath);
813
+ const placeholders = paths.map(() => "?").join(", ");
814
+ const rows = db
815
+ .prepare(`SELECT e.file_path AS file_path, COALESCE(MAX(u.utility), 0) AS utility
816
+ FROM entries e
817
+ LEFT JOIN utility_scores u ON u.entry_id = e.id
818
+ WHERE e.file_path IN (${placeholders})
819
+ GROUP BY e.file_path`)
820
+ .all(...paths);
821
+ for (const row of rows) {
822
+ utilityByPath.set(row.file_path, row.utility ?? 0);
823
+ }
824
+ }
825
+ return [...candidates].sort((a, b) => {
826
+ const ua = utilityByPath.get(a.absPath) ?? 0;
827
+ const ub = utilityByPath.get(b.absPath) ?? 0;
828
+ if (ub !== ua)
829
+ return ub - ua; // utility DESC
830
+ return a.absPath < b.absPath ? -1 : a.absPath > b.absPath ? 1 : 0; // tie-break: path ASC
831
+ });
832
+ }
631
833
  /**
632
834
  * Scan the primary stash for `memory:` and `knowledge:` markdown files
633
835
  * suitable for graph extraction. The directory layout convention is the
@@ -635,10 +837,14 @@ export async function runGraphExtractionPass(ctx) {
635
837
  *
636
838
  * Inferred-child memories (frontmatter `inferred: true`) are skipped — they
637
839
  * are already derived summaries, with no additional internal graph structure worth
638
- * extracting.
840
+ * extracting. Session-capture telemetry checkpoints are skipped too (see below).
639
841
  *
640
842
  * Exported for direct unit testing.
641
843
  */
844
+ // #632 — canonical session-capture name shape, mirrored from
845
+ // isSessionCaptureMemoryName (src/commands/improve/consolidate/eligibility.ts).
846
+ // Inlined to avoid an improve-layer import from the indexer layer.
847
+ const SESSION_CAPTURE_NAME_RE = /-(session|checkpoint)-\d{8}/;
642
848
  export function collectEligibleFiles(stashRoot, includeTypes = [...DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES]) {
643
849
  const out = [];
644
850
  for (const rawType of includeTypes) {
@@ -664,6 +870,14 @@ export function collectEligibleFiles(stashRoot, includeTypes = [...DEFAULT_GRAPH
664
870
  // graph to extract from a single-fact body.
665
871
  if (type === "memory" && parsed.data.inferred === true)
666
872
  continue;
873
+ // #632 — skip session-capture telemetry checkpoints (named
874
+ // `<harness>-(session|checkpoint)-<YYYYMMDD>-<id>`). Their bodies are
875
+ // pipeline bookkeeping; graph-extracting them lifts metadata fields as
876
+ // entities that then dominate clustering as bland stash-wide buckets.
877
+ // Mirrors isSessionCaptureMemoryName (consolidate/eligibility.ts); inlined
878
+ // here to keep the indexer layer free of an improve-layer import.
879
+ if (type === "memory" && SESSION_CAPTURE_NAME_RE.test(path.basename(filePath, ".md")))
880
+ continue;
667
881
  const body = parsed.content.trim();
668
882
  if (!body)
669
883
  continue;
@@ -0,0 +1,99 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import fs from "node:fs";
5
+ import path from "node:path";
6
+ import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../core/file-lock.js";
7
+ import { getDbPath, getIndexWriterLockPath } from "../core/paths.js";
8
+ const INDEX_WRITER_LOCK_STALE_AFTER_MS = 12 * 60 * 60 * 1000;
9
+ const INDEX_WRITER_WAIT_MS = 100;
10
+ const heldLocks = new Map();
11
+ function buildPayload(purpose, pid = process.pid) {
12
+ return JSON.stringify({
13
+ pid,
14
+ purpose,
15
+ dbPath: getDbPath(),
16
+ startedAt: new Date().toISOString(),
17
+ });
18
+ }
19
+ function delay(ms) {
20
+ return new Promise((resolve) => setTimeout(resolve, ms));
21
+ }
22
+ function throwIfAborted(signal) {
23
+ if (!signal?.aborted)
24
+ return;
25
+ throw signal.reason instanceof Error ? signal.reason : new Error("index writer wait aborted");
26
+ }
27
+ function releaseHeldLock(lockPath) {
28
+ const held = heldLocks.get(lockPath);
29
+ if (!held)
30
+ return;
31
+ held.depth -= 1;
32
+ if (held.depth > 0)
33
+ return;
34
+ heldLocks.delete(lockPath);
35
+ process.off("exit", held.exitHandler);
36
+ releaseLockIfOwned(lockPath, process.pid);
37
+ }
38
+ function retainHeldLock(lockPath) {
39
+ const existing = heldLocks.get(lockPath);
40
+ if (existing) {
41
+ existing.depth += 1;
42
+ return { lockPath, release: () => releaseHeldLock(lockPath) };
43
+ }
44
+ const exitHandler = () => releaseLockIfOwned(lockPath, process.pid);
45
+ process.on("exit", exitHandler);
46
+ heldLocks.set(lockPath, { depth: 1, exitHandler });
47
+ return { lockPath, release: () => releaseHeldLock(lockPath) };
48
+ }
49
+ function detachHeldLock(lockPath) {
50
+ const held = heldLocks.get(lockPath);
51
+ if (!held)
52
+ return;
53
+ heldLocks.delete(lockPath);
54
+ process.off("exit", held.exitHandler);
55
+ }
56
+ export async function acquireIndexWriterLease(options) {
57
+ const mode = options.mode ?? "wait";
58
+ const lockPath = getIndexWriterLockPath();
59
+ fs.mkdirSync(path.dirname(lockPath), { recursive: true });
60
+ if (heldLocks.has(lockPath)) {
61
+ return retainHeldLock(lockPath);
62
+ }
63
+ while (true) {
64
+ throwIfAborted(options.signal);
65
+ if (tryAcquireLockSync(lockPath, buildPayload(options.purpose))) {
66
+ return retainHeldLock(lockPath);
67
+ }
68
+ const probe = probeLock(lockPath, { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
69
+ if (probe.state === "held" && probe.holderPid === process.pid) {
70
+ return retainHeldLock(lockPath);
71
+ }
72
+ if (probe.state === "stale") {
73
+ releaseLock(lockPath);
74
+ continue;
75
+ }
76
+ if (mode === "try")
77
+ return undefined;
78
+ await delay(INDEX_WRITER_WAIT_MS);
79
+ }
80
+ }
81
+ export async function withIndexWriterLease(options, run) {
82
+ const lease = await acquireIndexWriterLease(options);
83
+ if (!lease) {
84
+ throw new Error(`index writer lease unavailable for ${options.purpose}`);
85
+ }
86
+ try {
87
+ return await run();
88
+ }
89
+ finally {
90
+ lease.release();
91
+ }
92
+ }
93
+ export function handoffIndexWriterLeaseToPid(lease, pid, purpose) {
94
+ fs.writeFileSync(lease.lockPath, buildPayload(purpose, pid), "utf8");
95
+ detachHeldLock(lease.lockPath);
96
+ }
97
+ export function probeIndexWriterLease() {
98
+ return probeLock(getIndexWriterLockPath(), { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
99
+ }