akm-cli 0.9.7 → 0.9.8-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/CHANGELOG.md +310 -0
  2. package/dist/commands/env/env.js +2 -21
  3. package/dist/commands/health/checks.js +0 -25
  4. package/dist/commands/health/data-dir-usage.js +165 -0
  5. package/dist/commands/health/improve-metrics.js +8 -34
  6. package/dist/commands/health/windows.js +0 -4
  7. package/dist/commands/health.js +18 -37
  8. package/dist/commands/improve/consolidate/eligibility.js +11 -5
  9. package/dist/commands/improve/extract.js +36 -32
  10. package/dist/commands/improve/memory/memory-belief.js +15 -5
  11. package/dist/commands/improve/memory/memory-contradiction-detect.js +16 -25
  12. package/dist/commands/improve/memory/memory-improve.js +7 -19
  13. package/dist/commands/improve/preparation.js +52 -36
  14. package/dist/commands/improve/reflect.js +13 -17
  15. package/dist/commands/lint/base-linter.js +42 -2
  16. package/dist/commands/lint/index.js +4 -5
  17. package/dist/commands/migrate/config-extra-params.js +61 -0
  18. package/dist/commands/migrate/dead-residue.js +113 -0
  19. package/dist/commands/migrate/stale-txn.js +49 -0
  20. package/dist/commands/migrate-cli.js +42 -1
  21. package/dist/commands/proposal/proposal.js +1 -21
  22. package/dist/commands/proposal/repository.js +0 -4
  23. package/dist/commands/proposal/validators/proposal-quality-validators.js +0 -6
  24. package/dist/commands/read/curate.js +4 -18
  25. package/dist/commands/read/show.js +2 -1
  26. package/dist/commands/sources/installed-stashes.js +5 -1
  27. package/dist/commands/sources/self-update.js +38 -1
  28. package/dist/commands/sources/sources-cli.js +17 -1
  29. package/dist/commands/tasks/tasks.js +0 -8
  30. package/dist/commands/url-checker.js +20 -28
  31. package/dist/commands/workflow-cli.js +11 -61
  32. package/dist/core/adapter/adapters/agent-skills-adapter.js +2 -6
  33. package/dist/core/adapter/adapters/akm-adapter.js +2 -3
  34. package/dist/core/adapter/adapters/akm-lint.js +2 -20
  35. package/dist/core/adapter/adapters/akm-task-adapter.js +2 -6
  36. package/dist/core/adapter/adapters/akm-workflow-adapter.js +2 -6
  37. package/dist/core/adapter/adapters/dotenv-adapter.js +2 -22
  38. package/dist/core/adapter/adapters/generic-files-adapter.js +3 -9
  39. package/dist/core/adapter/adapters/llm-wiki-adapter.js +2 -6
  40. package/dist/core/adapter/adapters/okf-adapter.js +7 -18
  41. package/dist/core/adapter/adapters/shared.js +2 -0
  42. package/dist/core/adapter/adapters/tool-dir-shared.js +3 -9
  43. package/dist/core/adapter/adapters/website-snapshot-adapter.js +2 -6
  44. package/dist/core/adapter/validate-context.js +1 -3
  45. package/dist/core/asset/asset-placement.js +1 -3
  46. package/dist/core/asset/frontmatter.js +96 -0
  47. package/dist/core/common.js +66 -2
  48. package/dist/core/config/config-io.js +21 -8
  49. package/dist/core/config/config-walker.js +6 -10
  50. package/dist/core/config/config.js +11 -8
  51. package/dist/core/extra-params.js +17 -13
  52. package/dist/core/improve-result.js +1 -3
  53. package/dist/core/json-schema.js +9 -11
  54. package/dist/core/state/migrations.js +74 -2
  55. package/dist/core/state-db.js +2 -1
  56. package/dist/core/warn.js +15 -0
  57. package/dist/execution/executable-identity.js +1 -3
  58. package/dist/execution/guarded-source.js +1 -6
  59. package/dist/indexer/bundle-identity-guard.js +6 -1
  60. package/dist/indexer/db/graph-db.js +139 -154
  61. package/dist/indexer/ensure-index.js +11 -19
  62. package/dist/indexer/graph/graph-boost.js +23 -34
  63. package/dist/indexer/graph/graph-extraction.js +12 -2
  64. package/dist/indexer/indexer.js +43 -22
  65. package/dist/indexer/lookup/adapter-concept-owner.js +12 -9
  66. package/dist/indexer/passes/dir-staleness.js +30 -9
  67. package/dist/indexer/passes/memory-inference.js +7 -2
  68. package/dist/indexer/scan/drain-dir.js +2 -4
  69. package/dist/indexer/search/db-search.js +3 -3
  70. package/dist/indexer/search/fts-query.js +10 -15
  71. package/dist/indexer/search/search-source.js +0 -13
  72. package/dist/indexer/usage/usage-events.js +9 -1
  73. package/dist/indexer/walk/walker.js +11 -6
  74. package/dist/integrations/harnesses/claude/result-extractor.js +30 -15
  75. package/dist/integrations/harnesses/codex/result-extractor.js +43 -6
  76. package/dist/integrations/harnesses/copilot/result-extractor.js +39 -12
  77. package/dist/integrations/harnesses/gemini/result-extractor.js +40 -12
  78. package/dist/integrations/harnesses/index.js +0 -4
  79. package/dist/integrations/harnesses/openhands/result-extractor.js +51 -19
  80. package/dist/integrations/harnesses/pi/result-extractor.js +45 -12
  81. package/dist/integrations/lockfile.js +0 -14
  82. package/dist/integrations/session-logs/index.js +0 -81
  83. package/dist/llm/client.js +0 -12
  84. package/dist/llm/memory-infer.js +1 -3
  85. package/dist/llm/usage-telemetry.js +1 -3
  86. package/dist/registry/create-provider-registry.js +4 -0
  87. package/dist/registry/factory.js +4 -0
  88. package/dist/registry/resolve.js +23 -8
  89. package/dist/runtime.js +0 -12
  90. package/dist/scripts/akm-migrate-node.js +179 -248
  91. package/dist/scripts/akm-migrate.js +179 -248
  92. package/dist/setup/setup.js +6 -12
  93. package/dist/sources/providers/git-install.js +7 -2
  94. package/dist/sources/providers/tar-utils.js +1 -7
  95. package/dist/sources/snapshot-fetchers/content-extract.js +0 -17
  96. package/dist/storage/managed-db.js +20 -7
  97. package/dist/storage/repositories/events-repository.js +0 -81
  98. package/dist/storage/repositories/index-entries-repository.js +24 -28
  99. package/dist/storage/repositories/index-fts-repository.js +24 -30
  100. package/dist/storage/repositories/index-meta-repository.js +6 -4
  101. package/dist/storage/repositories/index-schema.js +16 -1
  102. package/dist/storage/repositories/index-utility-repository.js +0 -57
  103. package/dist/storage/repositories/index-vec-repository.js +25 -27
  104. package/dist/storage/repositories/proposals-repository.js +4 -1
  105. package/dist/storage/repositories/task-history-repository.js +9 -3
  106. package/dist/tasks/backends/launchd.js +8 -18
  107. package/dist/tasks/resolve-akm-bin.js +15 -0
  108. package/dist/tasks/run/task-history.js +21 -31
  109. package/dist/tasks/scheduler-sync.js +1 -6
  110. package/dist/tasks/source/bounded-document.js +1 -14
  111. package/dist/tasks/source/task-to-v3.js +10 -2
  112. package/dist/workflows/exec/child-workflow.js +1 -1
  113. package/dist/workflows/exec/native-executor.js +2 -2
  114. package/dist/workflows/exec/step-work.js +5 -17
  115. package/dist/workflows/exec/worktree.js +40 -6
  116. package/dist/workflows/freeze/task-bindings.js +2 -4
  117. package/dist/workflows/ir/compile.js +3 -14
  118. package/dist/workflows/ir/schema-v4.js +4 -6
  119. package/dist/workflows/ir/schema.js +2 -5
  120. package/dist/workflows/parser.js +23 -26
  121. package/dist/workflows/source-files.js +8 -13
  122. package/dist/workflows/source-ir/schema.js +1 -19
  123. package/docs/migration/v0.9.0-troubleshooting.md +12 -2
  124. package/docs/migration/v0.9.1-to-v0.9.2.md +18 -4
  125. package/docs/reference/cli.md +3 -1
  126. package/docs/reference/data-and-telemetry.md +1 -0
  127. package/docs/reference/tasks.md +11 -0
  128. package/docs/reference/workflow-schema.md +24 -0
  129. package/package.json +2 -9
  130. package/dist/workflows/source-ir/compare.js +0 -17
@@ -259,25 +259,20 @@ export function acknowledgeExtractionQueueEntry(db, stashRoot, filePath, bodyHas
259
259
  export function loadGraphFilesOnly(stashPath, db) {
260
260
  try {
261
261
  return withReadableGraphDb(db, (readDb) => {
262
- try {
263
- const rows = readDb
264
- .prepare(`SELECT file_path, file_type, body_hash, confidence, status, reason
265
- FROM graph_files
266
- WHERE stash_root = ?
267
- ORDER BY file_order`)
268
- .all(stashPath);
269
- return rows.map((row) => ({
270
- path: row.file_path,
271
- type: row.file_type,
272
- bodyHash: row.body_hash,
273
- ...(typeof row.confidence === "number" ? { confidence: row.confidence } : {}),
274
- ...(row.status ? { status: row.status } : {}),
275
- ...(row.reason ? { reason: row.reason } : {}),
276
- }));
277
- }
278
- catch {
279
- return [];
280
- }
262
+ const rows = readDb
263
+ .prepare(`SELECT file_path, file_type, body_hash, confidence, status, reason
264
+ FROM graph_files
265
+ WHERE stash_root = ?
266
+ ORDER BY file_order`)
267
+ .all(stashPath);
268
+ return rows.map((row) => ({
269
+ path: row.file_path,
270
+ type: row.file_type,
271
+ bodyHash: row.body_hash,
272
+ ...(typeof row.confidence === "number" ? { confidence: row.confidence } : {}),
273
+ ...(row.status ? { status: row.status } : {}),
274
+ ...(row.reason ? { reason: row.reason } : {}),
275
+ }));
281
276
  });
282
277
  }
283
278
  catch (err) {
@@ -292,66 +287,61 @@ export function loadGraphFilesOnly(stashPath, db) {
292
287
  export function loadStoredGraphMeta(stashPath, db) {
293
288
  try {
294
289
  return withReadableGraphDb(db, (readDb) => {
295
- try {
296
- const row = readDb
297
- .prepare(`SELECT
298
- stash_root,
299
- schema_version,
300
- generated_at,
301
- considered_files,
302
- extracted_files,
303
- entity_count,
304
- relation_count,
305
- extraction_coverage,
306
- density,
307
- extractor_id,
308
- extraction_run_id,
309
- model,
310
- prompt_version,
311
- batch_size,
312
- cache_hits,
313
- cache_misses,
314
- truncation_count,
315
- failure_count
316
- FROM graph_meta
317
- WHERE stash_root = ?`)
318
- .get(stashPath);
319
- if (!row)
320
- return null;
321
- return {
322
- stashPath: row.stash_root,
323
- graphPath: getDbPath(),
324
- schemaVersion: row.schema_version,
325
- generatedAt: row.generated_at,
326
- quality: {
327
- consideredFiles: row.considered_files,
328
- extractedFiles: row.extracted_files,
329
- entityCount: row.entity_count,
330
- relationCount: row.relation_count,
331
- extractionCoverage: row.extraction_coverage,
332
- density: row.density,
333
- },
334
- telemetry: {
335
- ...(row.extractor_id ? { extractorId: row.extractor_id } : {}),
336
- ...(row.extraction_run_id ? { extractionRunId: row.extraction_run_id } : {}),
337
- ...(row.model ? { model: row.model } : {}),
338
- ...(row.prompt_version ? { promptVersion: row.prompt_version } : {}),
339
- ...(typeof row.batch_size === "number" ? { batchSize: row.batch_size } : {}),
340
- cacheHits: row.cache_hits,
341
- cacheMisses: row.cache_misses,
342
- truncationCount: row.truncation_count,
343
- failureCount: row.failure_count,
344
- // `retry_attempts` is not persisted to the graph-meta table (it is
345
- // surfaced from the run's emitted telemetry into `akm health`, not
346
- // from the reuse cache). Default to 0 so the loaded shape satisfies
347
- // GraphExtractionTelemetry.
348
- retryAttempts: 0,
349
- },
350
- };
351
- }
352
- catch {
290
+ const row = readDb
291
+ .prepare(`SELECT
292
+ stash_root,
293
+ schema_version,
294
+ generated_at,
295
+ considered_files,
296
+ extracted_files,
297
+ entity_count,
298
+ relation_count,
299
+ extraction_coverage,
300
+ density,
301
+ extractor_id,
302
+ extraction_run_id,
303
+ model,
304
+ prompt_version,
305
+ batch_size,
306
+ cache_hits,
307
+ cache_misses,
308
+ truncation_count,
309
+ failure_count
310
+ FROM graph_meta
311
+ WHERE stash_root = ?`)
312
+ .get(stashPath);
313
+ if (!row)
353
314
  return null;
354
- }
315
+ return {
316
+ stashPath: row.stash_root,
317
+ graphPath: getDbPath(),
318
+ schemaVersion: row.schema_version,
319
+ generatedAt: row.generated_at,
320
+ quality: {
321
+ consideredFiles: row.considered_files,
322
+ extractedFiles: row.extracted_files,
323
+ entityCount: row.entity_count,
324
+ relationCount: row.relation_count,
325
+ extractionCoverage: row.extraction_coverage,
326
+ density: row.density,
327
+ },
328
+ telemetry: {
329
+ ...(row.extractor_id ? { extractorId: row.extractor_id } : {}),
330
+ ...(row.extraction_run_id ? { extractionRunId: row.extraction_run_id } : {}),
331
+ ...(row.model ? { model: row.model } : {}),
332
+ ...(row.prompt_version ? { promptVersion: row.prompt_version } : {}),
333
+ ...(typeof row.batch_size === "number" ? { batchSize: row.batch_size } : {}),
334
+ cacheHits: row.cache_hits,
335
+ cacheMisses: row.cache_misses,
336
+ truncationCount: row.truncation_count,
337
+ failureCount: row.failure_count,
338
+ // `retry_attempts` is not persisted to the graph-meta table (it is
339
+ // surfaced from the run's emitted telemetry into `akm health`, not
340
+ // from the reuse cache). Default to 0 so the loaded shape satisfies
341
+ // GraphExtractionTelemetry.
342
+ retryAttempts: 0,
343
+ },
344
+ };
355
345
  });
356
346
  }
357
347
  catch (err) {
@@ -368,85 +358,80 @@ export function loadStoredGraphSnapshot(stashPath, db) {
368
358
  const meta = loadStoredGraphMeta(stashPath, readDb);
369
359
  if (!meta)
370
360
  return null;
371
- try {
372
- const fileRows = readDb
373
- .prepare(`SELECT file_path, file_type, body_hash, confidence, status, reason, extraction_run_id
374
- FROM graph_files
375
- WHERE stash_root = ?
376
- ORDER BY file_order`)
377
- .all(stashPath);
378
- const entityRows = readDb
379
- .prepare(`SELECT gf.file_path AS file_path, gfe.entity AS entity
380
- FROM graph_file_entities gfe
381
- JOIN graph_files gf
382
- ON gf.stash_root = gfe.stash_root
383
- AND gf.file_path = gfe.file_path
384
- AND gf.body_hash = gfe.body_hash
385
- WHERE gf.stash_root = ?
386
- ORDER BY gf.file_order, gfe.entity_order`)
387
- .all(stashPath);
388
- const relationRows = readDb
389
- .prepare(`SELECT gf.file_path AS file_path,
390
- gfr.from_entity AS from_entity,
391
- gfr.to_entity AS to_entity,
392
- gfr.relation_type AS relation_type,
393
- gfr.confidence AS confidence
394
- FROM graph_file_relations gfr
395
- JOIN graph_files gf
396
- ON gf.stash_root = gfr.stash_root
397
- AND gf.file_path = gfr.file_path
398
- AND gf.body_hash = gfr.body_hash
399
- WHERE gf.stash_root = ?
400
- ORDER BY gf.file_order, gfr.relation_order`)
401
- .all(stashPath);
402
- const entitiesByPath = new Map();
403
- for (const row of entityRows) {
404
- const bucket = entitiesByPath.get(row.file_path);
405
- if (bucket)
406
- bucket.push(row.entity);
407
- else
408
- entitiesByPath.set(row.file_path, [row.entity]);
409
- }
410
- const relationsByPath = new Map();
411
- for (const row of relationRows) {
412
- const relation = {
413
- from: row.from_entity,
414
- to: row.to_entity,
415
- ...(row.relation_type ? { type: row.relation_type } : {}),
416
- ...(typeof row.confidence === "number" ? { confidence: row.confidence } : {}),
417
- };
418
- const bucket = relationsByPath.get(row.file_path);
419
- if (bucket)
420
- bucket.push(relation);
421
- else
422
- relationsByPath.set(row.file_path, [relation]);
423
- }
424
- const files = fileRows.map((row) => ({
425
- path: row.file_path,
426
- type: row.file_type,
427
- ...(row.body_hash ? { bodyHash: row.body_hash } : {}),
428
- entities: entitiesByPath.get(row.file_path) ?? [],
429
- relations: relationsByPath.get(row.file_path) ?? [],
361
+ const fileRows = readDb
362
+ .prepare(`SELECT file_path, file_type, body_hash, confidence, status, reason, extraction_run_id
363
+ FROM graph_files
364
+ WHERE stash_root = ?
365
+ ORDER BY file_order`)
366
+ .all(stashPath);
367
+ const entityRows = readDb
368
+ .prepare(`SELECT gf.file_path AS file_path, gfe.entity AS entity
369
+ FROM graph_file_entities gfe
370
+ JOIN graph_files gf
371
+ ON gf.stash_root = gfe.stash_root
372
+ AND gf.file_path = gfe.file_path
373
+ AND gf.body_hash = gfe.body_hash
374
+ WHERE gf.stash_root = ?
375
+ ORDER BY gf.file_order, gfe.entity_order`)
376
+ .all(stashPath);
377
+ const relationRows = readDb
378
+ .prepare(`SELECT gf.file_path AS file_path,
379
+ gfr.from_entity AS from_entity,
380
+ gfr.to_entity AS to_entity,
381
+ gfr.relation_type AS relation_type,
382
+ gfr.confidence AS confidence
383
+ FROM graph_file_relations gfr
384
+ JOIN graph_files gf
385
+ ON gf.stash_root = gfr.stash_root
386
+ AND gf.file_path = gfr.file_path
387
+ AND gf.body_hash = gfr.body_hash
388
+ WHERE gf.stash_root = ?
389
+ ORDER BY gf.file_order, gfr.relation_order`)
390
+ .all(stashPath);
391
+ const entitiesByPath = new Map();
392
+ for (const row of entityRows) {
393
+ const bucket = entitiesByPath.get(row.file_path);
394
+ if (bucket)
395
+ bucket.push(row.entity);
396
+ else
397
+ entitiesByPath.set(row.file_path, [row.entity]);
398
+ }
399
+ const relationsByPath = new Map();
400
+ for (const row of relationRows) {
401
+ const relation = {
402
+ from: row.from_entity,
403
+ to: row.to_entity,
404
+ ...(row.relation_type ? { type: row.relation_type } : {}),
430
405
  ...(typeof row.confidence === "number" ? { confidence: row.confidence } : {}),
431
- ...(row.status ? { status: row.status } : {}),
432
- ...(row.reason ? { reason: row.reason } : {}),
433
- ...(row.extraction_run_id ? { extractionRunId: row.extraction_run_id } : {}),
434
- }));
435
- return {
436
- stashPath: meta.stashPath,
437
- graphPath: meta.graphPath,
438
- schemaVersion: meta.schemaVersion,
439
- generatedAt: meta.generatedAt,
440
- ...(meta.quality ? { quality: meta.quality } : {}),
441
- ...(meta.telemetry ? { telemetry: meta.telemetry } : {}),
442
- files,
443
- entities: uniqueSorted(files.flatMap((file) => file.entities)),
444
- relations: files.flatMap((file) => file.relations),
445
406
  };
407
+ const bucket = relationsByPath.get(row.file_path);
408
+ if (bucket)
409
+ bucket.push(relation);
410
+ else
411
+ relationsByPath.set(row.file_path, [relation]);
446
412
  }
447
- catch {
448
- return null;
449
- }
413
+ const files = fileRows.map((row) => ({
414
+ path: row.file_path,
415
+ type: row.file_type,
416
+ ...(row.body_hash ? { bodyHash: row.body_hash } : {}),
417
+ entities: entitiesByPath.get(row.file_path) ?? [],
418
+ relations: relationsByPath.get(row.file_path) ?? [],
419
+ ...(typeof row.confidence === "number" ? { confidence: row.confidence } : {}),
420
+ ...(row.status ? { status: row.status } : {}),
421
+ ...(row.reason ? { reason: row.reason } : {}),
422
+ ...(row.extraction_run_id ? { extractionRunId: row.extraction_run_id } : {}),
423
+ }));
424
+ return {
425
+ stashPath: meta.stashPath,
426
+ graphPath: meta.graphPath,
427
+ schemaVersion: meta.schemaVersion,
428
+ generatedAt: meta.generatedAt,
429
+ ...(meta.quality ? { quality: meta.quality } : {}),
430
+ ...(meta.telemetry ? { telemetry: meta.telemetry } : {}),
431
+ files,
432
+ entities: uniqueSorted(files.flatMap((file) => file.entities)),
433
+ relations: files.flatMap((file) => file.relations),
434
+ };
450
435
  });
451
436
  }
452
437
  catch (err) {
@@ -24,7 +24,6 @@ import path from "node:path";
24
24
  import { placementSpecList } from "../core/asset/asset-placement.js";
25
25
  import { classifyPathAccess } from "../core/path-access.js";
26
26
  import { getDbPath } from "../core/paths.js";
27
- import { warn } from "../core/warn.js";
28
27
  import { assertIndexPathReadable, closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
29
28
  import { getEntryCount, getIndexedFilePaths } from "../storage/repositories/index-entries-repository.js";
30
29
  import { getMeta } from "../storage/repositories/index-meta-repository.js";
@@ -188,22 +187,14 @@ function indexCanServeStash(stashDir) {
188
187
  }
189
188
  }
190
189
  async function runInlineReindex(stashDir, options = {}) {
191
- try {
192
- const { akmIndex } = await import("./indexer.js");
193
- await akmIndex({
194
- stashDir,
195
- implicit: true,
196
- ...(options.signal ? { signal: options.signal } : {}),
197
- ...(options.hydrateSources === false ? { hydrateSources: false } : {}),
198
- });
199
- return true;
200
- }
201
- catch (error) {
202
- if (options.signal?.aborted)
203
- throw error;
204
- warn("Auto-index failed, proceeding with existing index:", error instanceof Error ? error.message : String(error));
205
- return false;
206
- }
190
+ const { akmIndex } = await import("./indexer.js");
191
+ await akmIndex({
192
+ stashDir,
193
+ implicit: true,
194
+ ...(options.signal ? { signal: options.signal } : {}),
195
+ ...(options.hydrateSources === false ? { hydrateSources: false } : {}),
196
+ });
197
+ return true;
207
198
  }
208
199
  /**
209
200
  * Ensure the local index exists and can serve the caller.
@@ -217,8 +208,9 @@ async function runInlineReindex(stashDir, options = {}) {
217
208
  * trigger and waits for it. Use this for callers like `improve` whose
218
209
  * planning logic depends on a current `entries` table in the same process.
219
210
  *
220
- * Returns `true` only when an inline index run succeeds.
221
- * A rebuild attempt that fails (throws) resolves to `false`.
211
+ * Returns `true` only when an inline index run succeeds; `false` when no
212
+ * rebuild was needed. A rebuild attempt that fails propagates its error to
213
+ * the caller rather than silently proceeding with a stale/missing index.
222
214
  */
223
215
  export async function ensureIndex(stashDir, options = {}) {
224
216
  // §11.5: warn (once) if the configured bundle ids drifted from the persisted
@@ -257,45 +257,34 @@ export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
257
257
  }
258
258
  // Confirm the target file has a graph row; without it there is nothing to
259
259
  // relate. (Identity is file_path within the stash — one row per path.)
260
- try {
261
- const row = db
262
- .prepare("SELECT 1 AS present FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
263
- .get(stashRoot, filePath);
264
- if (row === undefined)
265
- return [];
266
- }
267
- catch {
260
+ const row = db
261
+ .prepare("SELECT 1 AS present FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
262
+ .get(stashRoot, filePath);
263
+ if (row === undefined)
268
264
  return [];
269
- }
270
265
  const effectiveLimit = Math.max(1, limit);
271
266
  // Shared-entity count per candidate file_path. The target's entities are the
272
267
  // rows for `filePath`; candidates are any OTHER file_path in the stash that
273
268
  // shares a normalized entity.
274
- let candidateRows;
275
- try {
276
- candidateRows = db
277
- .prepare(`SELECT gf.file_path AS file_path,
278
- gf.file_type AS file_type,
279
- COUNT(*) AS shared
280
- FROM graph_file_entities target
281
- JOIN graph_file_entities e
282
- ON e.stash_root = target.stash_root
283
- AND e.entity_norm = target.entity_norm
284
- AND e.file_path != target.file_path
285
- JOIN graph_files gf
286
- ON gf.stash_root = e.stash_root
287
- AND gf.file_path = e.file_path
288
- AND gf.body_hash = e.body_hash
289
- WHERE target.file_path = ?
290
- AND target.stash_root = ?
291
- GROUP BY gf.file_path
292
- ORDER BY shared DESC, gf.file_path ASC
293
- LIMIT ?`)
294
- .all(filePath, stashRoot, effectiveLimit);
295
- }
296
- catch {
297
- return [];
298
- }
269
+ const candidateRows = db
270
+ .prepare(`SELECT gf.file_path AS file_path,
271
+ gf.file_type AS file_type,
272
+ COUNT(*) AS shared
273
+ FROM graph_file_entities target
274
+ JOIN graph_file_entities e
275
+ ON e.stash_root = target.stash_root
276
+ AND e.entity_norm = target.entity_norm
277
+ AND e.file_path != target.file_path
278
+ JOIN graph_files gf
279
+ ON gf.stash_root = e.stash_root
280
+ AND gf.file_path = e.file_path
281
+ AND gf.body_hash = e.body_hash
282
+ WHERE target.file_path = ?
283
+ AND target.stash_root = ?
284
+ GROUP BY gf.file_path
285
+ ORDER BY shared DESC, gf.file_path ASC
286
+ LIMIT ?`)
287
+ .all(filePath, stashRoot, effectiveLimit);
299
288
  if (candidateRows.length === 0)
300
289
  return [];
301
290
  const candidatePaths = candidateRows.map((r) => r.file_path);
@@ -971,6 +971,11 @@ async function extractGraphForSingleFileRevision(db, stashRoot, filePath, opts)
971
971
  if (err instanceof ConfigError)
972
972
  throw err;
973
973
  rethrowIfTestIsolationError(err);
974
+ // A genuine extraction/write failure, distinct from the deliberate
975
+ // "nothing to do" skips above (missing file, empty body, no model). Warn
976
+ // so it is visible instead of looking identical to a no-op skip; the
977
+ // entry stays queued and is retried on the next pass.
978
+ warn(`graph extraction: failed to extract graph for ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
974
979
  return { written: false };
975
980
  }
976
981
  finally {
@@ -1057,12 +1062,17 @@ export function collectEligibleFiles(stashRoot, includeTypes = [...DEFAULT_GRAPH
1057
1062
  const dir = path.join(stashRoot, stashDir);
1058
1063
  if (!fs.existsSync(dir))
1059
1064
  continue;
1060
- for (const filePath of walkMarkdownFiles(dir)) {
1065
+ const walked = walkMarkdownFiles(dir);
1066
+ if (!walked.complete) {
1067
+ warn(`graph extraction: directory scan under ${dir} is incomplete — some files may be missing`);
1068
+ }
1069
+ for (const filePath of walked.files) {
1061
1070
  let raw;
1062
1071
  try {
1063
1072
  raw = fs.readFileSync(filePath, "utf8");
1064
1073
  }
1065
- catch {
1074
+ catch (err) {
1075
+ warn(`graph extraction: failed to read candidate file ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
1066
1076
  continue;
1067
1077
  }
1068
1078
  const parsed = parseFrontmatter(raw);
@@ -30,7 +30,7 @@ import { deleteStoredGraph } from "./db/graph-db.js";
30
30
  import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
31
31
  import { indexedPathMatchesOwner, resolveAdapterConceptOwner, } from "./lookup/adapter-concept-owner.js";
32
32
  import { generateEmbeddingsForDb } from "./materialize-embeddings.js";
33
- import { canUseIncrementalSkip, computeDirFingerprint, getCachedZeroRowDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
33
+ import { canUseIncrementalSkip, computeDirFingerprint, getCachedDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
34
34
  import { isEnrichmentComplete, isWorkflowSkipWarning } from "./passes/metadata.js";
35
35
  import { drainDirDocuments } from "./scan/drain-dir.js";
36
36
  import { buildSearchText } from "./search/search-fields.js";
@@ -208,7 +208,7 @@ async function runEmbeddingPhase(ctx) {
208
208
  * verify event.
209
209
  */
210
210
  async function runFinalizePhase(ctx, deferredUpdateTransaction) {
211
- const { db, config, sources, sourceDirs, isIncremental, stashDir, signal, onProgress } = ctx;
211
+ const { db, config, sources, sourceDirs, stashDir, signal, onProgress } = ctx;
212
212
  ctx.timing.tFinalizeStart = Date.now();
213
213
  // `upsertEntry` and every canonical delete own their FTS projection. This is
214
214
  // an observation point, not a second materialization pass.
@@ -360,6 +360,19 @@ export function _setIndexTransactionHookForTests(hook) {
360
360
  function indexTransactionHook(point) {
361
361
  indexTransactionHookForTests?.(point);
362
362
  }
363
+ let drainObserverForTests;
364
+ /**
365
+ * TEST-ONLY. Observe every directory that actually reaches
366
+ * `drainDirDocuments` — the per-file read/sha256-hash/frontmatter-parse step
367
+ * (#900) — with the directory path and its walked file count. `undefined`
368
+ * restores. A directory the pre-drain gate (`getCachedDirState`) skips never
369
+ * fires this observer, so it is the
370
+ * seam #900's own tests use to assert an unchanged directory's files are
371
+ * never read on a no-op incremental run.
372
+ */
373
+ export function _setDrainObserverForTests(observer) {
374
+ drainObserverForTests = observer;
375
+ }
363
376
  /**
364
377
  * Detect an adapter for every resolvable source that does not declare one, and
365
378
  * persist each detection into `config.json`.
@@ -883,17 +896,19 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
883
896
  // Incremental freshness gate shared by both branches: consult the persisted
884
897
  // dir state and record either a skip (unchanged + eligible for incremental
885
898
  // skip) or a scan record carrying the candidate stash.
886
- const recordFreshnessDecision = (dirPath, currentStashDir, stateFiles, stash, hashByFile, conceptIdByFile, indexVariant, forceScan, pruneMissing) => {
887
- const previousState = getDirIndexState(db, dirPath, stateFiles, builtAtMs, indexVariant);
899
+ const recordFreshnessDecision = (dirPath, currentStashDir, stateFiles, fingerprint, stash, hashByFile, conceptIdByFile, indexVariant, forceScan, pruneMissing) => {
900
+ const previousState = getDirIndexState(db, dirPath, stateFiles, builtAtMs, indexVariant, fingerprint);
888
901
  if (isIncremental && !forceScan && !previousState.stale && canUseIncrementalSkip(previousState, priorDirsChanged)) {
889
902
  skippedDirs++;
890
903
  dirRecords.push({
891
904
  dirPath,
892
905
  currentStashDir,
893
906
  files: stateFiles,
907
+ fingerprint,
894
908
  stash: null,
895
909
  skip: true,
896
910
  reason: previousState.reason,
911
+ persistedRowCount: previousState.persistedRowCount,
897
912
  indexVariant,
898
913
  });
899
914
  reportDirDecision("skip", dirPath, currentStashDir, previousState.reason, previousState.persistedRowCount);
@@ -906,6 +921,7 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
906
921
  dirPath,
907
922
  currentStashDir,
908
923
  files: stateFiles,
924
+ fingerprint,
909
925
  stash,
910
926
  skip: false,
911
927
  reason,
@@ -959,10 +975,13 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
959
975
  reportDirDecision("skip", dirPath, currentStashDir, reason);
960
976
  continue;
961
977
  }
962
- const cachedZeroRowState = isIncremental &&
978
+ // #900: decide from stat data alone whether the directory can be skipped,
979
+ // before drainDirDocuments reads, hashes, and parses every file.
980
+ const fingerprint = computeDirFingerprint(dirPath, indexableFiles, indexVariant);
981
+ const cachedState = isIncremental &&
963
982
  !forceScan &&
964
- getCachedZeroRowDirState(db, dirPath, indexableFiles, builtAtMs, priorDirsChanged, indexVariant);
965
- if (cachedZeroRowState) {
983
+ getCachedDirState(db, dirPath, indexableFiles, builtAtMs, priorDirsChanged, indexVariant, fingerprint);
984
+ if (cachedState) {
966
985
  skippedDirs++;
967
986
  dirRecords.push({
968
987
  dirPath,
@@ -970,15 +989,16 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
970
989
  files: indexableFiles,
971
990
  stash: null,
972
991
  skip: true,
973
- reason: cachedZeroRowState.reason,
992
+ reason: cachedState.reason,
974
993
  indexVariant,
975
994
  });
976
- reportDirDecision("skip", dirPath, currentStashDir, cachedZeroRowState.reason, cachedZeroRowState.persistedRowCount);
995
+ reportDirDecision("skip", dirPath, currentStashDir, cachedState.reason, cachedState.persistedRowCount);
977
996
  continue;
978
997
  }
979
998
  // F4a M-core-2 (the flip): drain the dir's `IndexDocument` stream via the
980
999
  // component's dispatched `adapter.recognize` (broken workflows dropped-with-
981
1000
  // warning at the drain layer) and reconstruct the durable `IndexDocument`s.
1001
+ drainObserverForTests?.(dirPath, ctxs.length);
982
1002
  const drained = drainDirDocuments(adapter, component, ctxs);
983
1003
  if (drained.warnings.length)
984
1004
  warnings.push(...drained.warnings);
@@ -992,7 +1012,7 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
992
1012
  if (generated.entries.length > 0) {
993
1013
  generatedCount += generated.entries.length;
994
1014
  }
995
- recordFreshnessDecision(dirPath, currentStashDir, staleFiles, stash, drained.hashByFile, drained.conceptIdByFile, indexVariant, forceScan, walkComplete);
1015
+ recordFreshnessDecision(dirPath, currentStashDir, staleFiles, fingerprint, stash, drained.hashByFile, drained.conceptIdByFile, indexVariant, forceScan, walkComplete);
996
1016
  }
997
1017
  }
998
1018
  return {
@@ -1089,7 +1109,7 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
1089
1109
  // tests/integration/indexer/reindex-generation-atomicity.test.ts.
1090
1110
  indexTransactionHook("full-delete-applied");
1091
1111
  }
1092
- for (const { dirPath, currentStashDir, files, stash, skip, reason, hashByFile, conceptIdByFile, indexVariant, remove, pruneMissing, } of dirRecords) {
1112
+ for (const { dirPath, currentStashDir, files, fingerprint, stash, skip, reason, persistedRowCount, hashByFile, conceptIdByFile, indexVariant, remove, pruneMissing, } of dirRecords) {
1093
1113
  const bundle = bundleByRoot.get(path.resolve(currentStashDir));
1094
1114
  if (!bundle)
1095
1115
  throw new Error(`Missing bundle provenance for indexed source ${currentStashDir}`);
@@ -1102,14 +1122,11 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
1102
1122
  continue;
1103
1123
  }
1104
1124
  if (skip) {
1105
- if (reason?.kind === "unchanged") {
1106
- const fingerprint = computeDirFingerprint(dirPath, files, indexVariant);
1107
- upsertIndexDirState(db, {
1108
- dirPath,
1109
- fileSetHash: fingerprint.fileSetHash,
1110
- fileMtimeMaxMs: fingerprint.fileMtimeMaxMs,
1111
- reason: reason.kind,
1112
- });
1125
+ // "unchanged" is the post-drain verdict: re-persist so the row carries
1126
+ // row_count and the gate skips this directory before draining next
1127
+ // time. "unchanged-precheck" already matched the stored row.
1128
+ if (reason?.kind === "unchanged" && fingerprint) {
1129
+ upsertIndexDirState(db, { dirPath, ...fingerprint, reason: reason.kind, rowCount: persistedRowCount });
1113
1130
  }
1114
1131
  continue;
1115
1132
  }
@@ -1165,7 +1182,7 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
1165
1182
  if (pruneMissing !== false) {
1166
1183
  addEntryIds(deletedUsageEntryIds, deleteEntriesByDirExceptRefs(db, dirPath, bundle.bundleId, keptItemRefs, { cleanupUsageEvents: false }));
1167
1184
  }
1168
- const fingerprint = computeDirFingerprint(dirPath, files, indexVariant);
1185
+ const persistedFingerprint = fingerprint ?? computeDirFingerprint(dirPath, files, indexVariant);
1169
1186
  const persistedReason = persistedRows === 0
1170
1187
  ? inferZeroRowReason(stash, reason, warnings, dirPath, dedupedRows)
1171
1188
  : reason?.kind === "full-rebuild"
@@ -1173,9 +1190,13 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
1173
1190
  : (reason?.kind ?? "updated");
1174
1191
  upsertIndexDirState(db, {
1175
1192
  dirPath,
1176
- fileSetHash: fingerprint.fileSetHash,
1177
- fileMtimeMaxMs: fingerprint.fileMtimeMaxMs,
1193
+ ...persistedFingerprint,
1178
1194
  reason: persistedReason,
1195
+ // A directory that lost rows to per-source dedup depends on the
1196
+ // directories persisted before it, not only on its own files, so it
1197
+ // must keep draining every run (as it did before the gate) until a
1198
+ // drain persists it without dedup. NULL keeps the gate closed.
1199
+ rowCount: dedupedRows === 0 ? persistedRows : undefined,
1179
1200
  });
1180
1201
  if (persistedRows === 0) {
1181
1202
  // Warn only when the dir had files that *could* produce entries (.md or