akm-cli 0.9.0-beta.1 → 0.9.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CHANGELOG.md +469 -0
  2. package/dist/assets/templates/html/default.html +78 -0
  3. package/dist/assets/templates/html/health.html +732 -0
  4. package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
  5. package/dist/cli/shared.js +21 -5
  6. package/dist/cli.js +47 -5
  7. package/dist/commands/config-cli.js +0 -10
  8. package/dist/commands/feedback-cli.js +42 -37
  9. package/dist/commands/graph/graph.js +75 -71
  10. package/dist/commands/health/checks.js +48 -0
  11. package/dist/commands/health/html-report.js +666 -0
  12. package/dist/commands/health.js +186 -13
  13. package/dist/commands/improve/consolidate.js +39 -6
  14. package/dist/commands/improve/distill.js +26 -5
  15. package/dist/commands/improve/extract-prompt.js +1 -1
  16. package/dist/commands/improve/extract.js +52 -8
  17. package/dist/commands/improve/improve-auto-accept.js +33 -1
  18. package/dist/commands/improve/improve-cli.js +7 -0
  19. package/dist/commands/improve/improve-profiles.js +4 -0
  20. package/dist/commands/improve/improve.js +874 -447
  21. package/dist/commands/improve/proactive-maintenance.js +113 -0
  22. package/dist/commands/improve/reflect-noise.js +0 -0
  23. package/dist/commands/improve/reflect.js +31 -0
  24. package/dist/commands/proposal/drain.js +73 -6
  25. package/dist/commands/proposal/proposal-cli.js +22 -10
  26. package/dist/commands/proposal/proposal.js +17 -1
  27. package/dist/commands/proposal/validators/proposals.js +365 -329
  28. package/dist/commands/read/curate.js +17 -0
  29. package/dist/commands/remember.js +6 -2
  30. package/dist/commands/sources/stash-cli.js +10 -2
  31. package/dist/commands/tasks/tasks.js +32 -8
  32. package/dist/core/config/config-schema.js +30 -0
  33. package/dist/core/logs-db.js +304 -0
  34. package/dist/core/paths.js +3 -0
  35. package/dist/core/state-db.js +152 -14
  36. package/dist/indexer/db/db.js +99 -13
  37. package/dist/indexer/ensure-index.js +152 -17
  38. package/dist/indexer/index-writer-lock.js +99 -0
  39. package/dist/indexer/indexer.js +114 -111
  40. package/dist/indexer/passes/memory-inference.js +61 -22
  41. package/dist/integrations/harnesses/claude/session-log.js +17 -5
  42. package/dist/llm/client.js +38 -4
  43. package/dist/llm/usage-persist.js +77 -0
  44. package/dist/llm/usage-telemetry.js +103 -0
  45. package/dist/output/context.js +3 -2
  46. package/dist/output/html-render.js +73 -0
  47. package/dist/output/shapes/helpers.js +17 -1
  48. package/dist/output/text/helpers.js +69 -1
  49. package/dist/scripts/migrate-storage.js +153 -25
  50. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +21 -2
  51. package/dist/sources/providers/tar-utils.js +16 -8
  52. package/dist/tasks/backends/cron.js +46 -9
  53. package/dist/tasks/runner.js +99 -16
  54. package/dist/workflows/db.js +4 -0
  55. package/package.json +3 -2
  56. package/dist/commands/config-edit.js +0 -344
@@ -0,0 +1,99 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ import fs from "node:fs";
5
+ import path from "node:path";
6
+ import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../core/file-lock.js";
7
+ import { getDbPath, getIndexWriterLockPath } from "../core/paths.js";
8
+ const INDEX_WRITER_LOCK_STALE_AFTER_MS = 12 * 60 * 60 * 1000;
9
+ const INDEX_WRITER_WAIT_MS = 100;
10
+ const heldLocks = new Map();
11
+ function buildPayload(purpose, pid = process.pid) {
12
+ return JSON.stringify({
13
+ pid,
14
+ purpose,
15
+ dbPath: getDbPath(),
16
+ startedAt: new Date().toISOString(),
17
+ });
18
+ }
19
+ function delay(ms) {
20
+ return new Promise((resolve) => setTimeout(resolve, ms));
21
+ }
22
+ function throwIfAborted(signal) {
23
+ if (!signal?.aborted)
24
+ return;
25
+ throw signal.reason instanceof Error ? signal.reason : new Error("index writer wait aborted");
26
+ }
27
+ function releaseHeldLock(lockPath) {
28
+ const held = heldLocks.get(lockPath);
29
+ if (!held)
30
+ return;
31
+ held.depth -= 1;
32
+ if (held.depth > 0)
33
+ return;
34
+ heldLocks.delete(lockPath);
35
+ process.off("exit", held.exitHandler);
36
+ releaseLockIfOwned(lockPath, process.pid);
37
+ }
38
+ function retainHeldLock(lockPath) {
39
+ const existing = heldLocks.get(lockPath);
40
+ if (existing) {
41
+ existing.depth += 1;
42
+ return { lockPath, release: () => releaseHeldLock(lockPath) };
43
+ }
44
+ const exitHandler = () => releaseLockIfOwned(lockPath, process.pid);
45
+ process.on("exit", exitHandler);
46
+ heldLocks.set(lockPath, { depth: 1, exitHandler });
47
+ return { lockPath, release: () => releaseHeldLock(lockPath) };
48
+ }
49
+ function detachHeldLock(lockPath) {
50
+ const held = heldLocks.get(lockPath);
51
+ if (!held)
52
+ return;
53
+ heldLocks.delete(lockPath);
54
+ process.off("exit", held.exitHandler);
55
+ }
56
+ export async function acquireIndexWriterLease(options) {
57
+ const mode = options.mode ?? "wait";
58
+ const lockPath = getIndexWriterLockPath();
59
+ fs.mkdirSync(path.dirname(lockPath), { recursive: true });
60
+ if (heldLocks.has(lockPath)) {
61
+ return retainHeldLock(lockPath);
62
+ }
63
+ while (true) {
64
+ throwIfAborted(options.signal);
65
+ if (tryAcquireLockSync(lockPath, buildPayload(options.purpose))) {
66
+ return retainHeldLock(lockPath);
67
+ }
68
+ const probe = probeLock(lockPath, { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
69
+ if (probe.state === "held" && probe.holderPid === process.pid) {
70
+ return retainHeldLock(lockPath);
71
+ }
72
+ if (probe.state === "stale") {
73
+ releaseLock(lockPath);
74
+ continue;
75
+ }
76
+ if (mode === "try")
77
+ return undefined;
78
+ await delay(INDEX_WRITER_WAIT_MS);
79
+ }
80
+ }
81
+ export async function withIndexWriterLease(options, run) {
82
+ const lease = await acquireIndexWriterLease(options);
83
+ if (!lease) {
84
+ throw new Error(`index writer lease unavailable for ${options.purpose}`);
85
+ }
86
+ try {
87
+ return await run();
88
+ }
89
+ finally {
90
+ lease.release();
91
+ }
92
+ }
93
+ export function handoffIndexWriterLeaseToPid(lease, pid, purpose) {
94
+ fs.writeFileSync(lease.lockPath, buildPayload(purpose, pid), "utf8");
95
+ detachHeldLock(lease.lockPath);
96
+ }
97
+ export function probeIndexWriterLease() {
98
+ return probeLock(getIndexWriterLockPath(), { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
99
+ }
@@ -12,6 +12,7 @@ import { resolveIndexPassLLM } from "../llm/index-passes.js";
12
12
  import { takeWorkflowDocument } from "../workflows/runtime/document-cache.js";
13
13
  import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddableEntryCount, getEmbeddingCount, getEntriesByDir, getEntryCount, getIndexDirState, getMeta, isVecAvailable, openDatabase, openExistingDatabase, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db/db.js";
14
14
  import { deleteStoredGraph } from "./db/graph-db.js";
15
+ import { withIndexWriterLease } from "./index-writer-lock.js";
15
16
  import { applyCuratedFrontmatter, applyWikiFrontmatter, generateMetadataFlat, isEnrichmentComplete, isWorkflowSkipWarning, loadStashFile, shouldIndexStashFile, } from "./passes/metadata.js";
16
17
  import { buildSearchText } from "./search/search-fields.js";
17
18
  import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
@@ -225,119 +226,121 @@ function runCleanPass(db, dryRun) {
225
226
  }
226
227
  // ── Indexer ──────────────────────────────────────────────────────────────────
227
228
  export async function akmIndex(options) {
228
- const stashDir = options?.stashDir || resolveStashDir();
229
- const onProgress = options?.onProgress ?? (() => { });
230
- const signal = options?.signal;
231
- const reEnrich = options?.reEnrich === true;
232
- const full = options?.full === true;
233
- const clean = options?.clean === true;
234
- const dryRun = options?.dryRun === true;
235
- // Load config and resolve all stash sources
236
- const { loadConfig } = await import("../core/config/config.js");
237
- const config = loadConfig();
238
- // One-time, read-only guard: warn if the writable stash still holds an
239
- // un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
240
- // entirely, so an unmigrated vault's `.env` data would silently never be
241
- // indexed. Non-destructive only stats, never reads/writes/deletes.
242
- const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
243
- warnOnUnmigratedVaults(stashDir);
244
- // Ensure git stash caches are extracted before resolving stash dirs,
245
- // so their content directories exist on disk for the walker to discover.
246
- const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
247
- await ensureSourceCaches(config, { force: full });
248
- const allSourceEntries = resolveSourceEntries(stashDir, config);
249
- const allSourceDirs = allSourceEntries.map((s) => s.path);
250
- const t0 = Date.now();
251
- // Open database — pass embedding dimension from config if available
252
- const dbPath = getDbPath();
253
- const embeddingDim = config.embedding?.dimension;
254
- const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
255
- try {
256
- // Determine incremental vs full mode
257
- const prevStashDir = getMeta(db, "stashDir");
258
- const prevBuiltAt = getMeta(db, "builtAt");
259
- const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
260
- const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
261
- // Assemble the run context
262
- const ctx = {
263
- db,
264
- config,
265
- sources: allSourceEntries,
266
- sourceDirs: allSourceDirs,
267
- full,
268
- reEnrich,
269
- stashDir,
270
- onProgress,
271
- signal,
272
- timing: {
273
- t0,
274
- tWalkStart: t0,
275
- tWalkEnd: t0,
276
- tLlmEnd: t0,
277
- tFtsEnd: t0,
278
- tEmbedEnd: t0,
279
- },
280
- isIncremental,
281
- builtAtMs,
282
- hadRemovedSources: false,
283
- scannedDirs: 0,
284
- skippedDirs: 0,
285
- generatedCount: 0,
286
- walkWarnings: [],
287
- dirsNeedingLlm: [],
288
- embeddingResult: null,
289
- graphExtractionResult: null,
290
- };
291
- onProgress({
292
- phase: "summary",
293
- message: buildIndexSummaryMessage({
229
+ return withIndexWriterLease({ purpose: "akm-index", signal: options?.signal }, async () => {
230
+ const stashDir = options?.stashDir || resolveStashDir();
231
+ const onProgress = options?.onProgress ?? (() => { });
232
+ const signal = options?.signal;
233
+ const reEnrich = options?.reEnrich === true;
234
+ const full = options?.full === true;
235
+ const clean = options?.clean === true;
236
+ const dryRun = options?.dryRun === true;
237
+ // Load config and resolve all stash sources
238
+ const { loadConfig } = await import("../core/config/config.js");
239
+ const config = loadConfig();
240
+ // One-time, read-only guard: warn if the writable stash still holds an
241
+ // un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
242
+ // entirely, so an unmigrated vault's `.env` data would silently never be
243
+ // indexed. Non-destructive only stats, never reads/writes/deletes.
244
+ const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
245
+ warnOnUnmigratedVaults(stashDir);
246
+ // Ensure git stash caches are extracted before resolving stash dirs,
247
+ // so their content directories exist on disk for the walker to discover.
248
+ const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
249
+ await ensureSourceCaches(config, { force: full });
250
+ const allSourceEntries = resolveSourceEntries(stashDir, config);
251
+ const allSourceDirs = allSourceEntries.map((s) => s.path);
252
+ const t0 = Date.now();
253
+ // Open database — pass embedding dimension from config if available
254
+ const dbPath = getDbPath();
255
+ const embeddingDim = config.embedding?.dimension;
256
+ const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
257
+ try {
258
+ // Determine incremental vs full mode
259
+ const prevStashDir = getMeta(db, "stashDir");
260
+ const prevBuiltAt = getMeta(db, "builtAt");
261
+ const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
262
+ const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
263
+ // Assemble the run context
264
+ const ctx = {
265
+ db,
266
+ config,
267
+ sources: allSourceEntries,
268
+ sourceDirs: allSourceDirs,
269
+ full,
270
+ reEnrich,
271
+ stashDir,
272
+ onProgress,
273
+ signal,
274
+ timing: {
275
+ t0,
276
+ tWalkStart: t0,
277
+ tWalkEnd: t0,
278
+ tLlmEnd: t0,
279
+ tFtsEnd: t0,
280
+ tEmbedEnd: t0,
281
+ },
282
+ isIncremental,
283
+ builtAtMs,
284
+ hadRemovedSources: false,
285
+ scannedDirs: 0,
286
+ skippedDirs: 0,
287
+ generatedCount: 0,
288
+ walkWarnings: [],
289
+ dirsNeedingLlm: [],
290
+ embeddingResult: null,
291
+ graphExtractionResult: null,
292
+ };
293
+ onProgress({
294
+ phase: "summary",
295
+ message: buildIndexSummaryMessage({
296
+ mode: isIncremental ? "incremental" : "full",
297
+ sourcesCount: allSourceDirs.length,
298
+ semanticSearchMode: config.semanticSearchMode,
299
+ embeddingProvider: getEmbeddingProvider(config.embedding),
300
+ llmEnabled: !!resolveIndexPassLLM("enrichment", config),
301
+ vecAvailable: isVecAvailable(db),
302
+ }),
303
+ });
304
+ // ── Phase sequence ───────────────────────────────────────────────────────
305
+ await runSourceCachePhase(ctx);
306
+ await runWalkPhase(ctx);
307
+ await runEmbeddingPhase(ctx);
308
+ await runFinalizePhase(ctx);
309
+ // ────────────────────────────────────────────────────────────────────────
310
+ const { _verification: verification, _totalEntries: totalEntries } = ctx;
311
+ const { timing } = ctx;
312
+ // ── Clean pass ───────────────────────────────────────────────────────────
313
+ // After the normal index completes, remove entries whose source files no
314
+ // longer exist on disk. Remote entries (empty file_path) are skipped.
315
+ let cleanResult;
316
+ if (clean) {
317
+ cleanResult = runCleanPass(db, dryRun);
318
+ }
319
+ // ────────────────────────────────────────────────────────────────────────
320
+ return {
321
+ stashDir,
322
+ totalEntries,
323
+ generatedMetadata: ctx.generatedCount,
324
+ indexPath: dbPath,
294
325
  mode: isIncremental ? "incremental" : "full",
295
- sourcesCount: allSourceDirs.length,
296
- semanticSearchMode: config.semanticSearchMode,
297
- embeddingProvider: getEmbeddingProvider(config.embedding),
298
- llmEnabled: !!resolveIndexPassLLM("enrichment", config),
299
- vecAvailable: isVecAvailable(db),
300
- }),
301
- });
302
- // ── Phase sequence ───────────────────────────────────────────────────────
303
- await runSourceCachePhase(ctx);
304
- await runWalkPhase(ctx);
305
- await runEmbeddingPhase(ctx);
306
- await runFinalizePhase(ctx);
307
- // ────────────────────────────────────────────────────────────────────────
308
- const { _verification: verification, _totalEntries: totalEntries } = ctx;
309
- const { timing } = ctx;
310
- // ── Clean pass ───────────────────────────────────────────────────────────
311
- // After the normal index completes, remove entries whose source files no
312
- // longer exist on disk. Remote entries (empty file_path) are skipped.
313
- let cleanResult;
314
- if (clean) {
315
- cleanResult = runCleanPass(db, dryRun);
326
+ directoriesScanned: ctx.scannedDirs,
327
+ directoriesSkipped: ctx.skippedDirs,
328
+ ...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
329
+ verification,
330
+ timing: {
331
+ totalMs: Date.now() - timing.t0,
332
+ walkMs: timing.tWalkEnd - timing.tWalkStart,
333
+ llmMs: timing.tLlmEnd - timing.tWalkEnd,
334
+ embedMs: timing.tEmbedEnd - timing.tLlmEnd,
335
+ ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
336
+ },
337
+ ...(cleanResult !== undefined ? { clean: cleanResult } : {}),
338
+ };
316
339
  }
317
- // ────────────────────────────────────────────────────────────────────────
318
- return {
319
- stashDir,
320
- totalEntries,
321
- generatedMetadata: ctx.generatedCount,
322
- indexPath: dbPath,
323
- mode: isIncremental ? "incremental" : "full",
324
- directoriesScanned: ctx.scannedDirs,
325
- directoriesSkipped: ctx.skippedDirs,
326
- ...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
327
- verification,
328
- timing: {
329
- totalMs: Date.now() - timing.t0,
330
- walkMs: timing.tWalkEnd - timing.tWalkStart,
331
- llmMs: timing.tLlmEnd - timing.tWalkEnd,
332
- embedMs: timing.tEmbedEnd - timing.tLlmEnd,
333
- ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
334
- },
335
- ...(cleanResult !== undefined ? { clean: cleanResult } : {}),
336
- };
337
- }
338
- finally {
339
- closeDatabase(db);
340
- }
340
+ finally {
341
+ closeDatabase(db);
342
+ }
343
+ });
341
344
  }
342
345
  // ── Extracted helpers for indexing ────────────────────────────────────────────
343
346
  async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, doFullDelete = false, onProgress) {
@@ -119,6 +119,26 @@ export async function runMemoryInferencePass(ctx) {
119
119
  // 2026-05-26).
120
120
  if (signal?.aborted)
121
121
  return { aborted: true };
122
+ // Pre-check (#588): when `<parent>.derived.md` is already on disk the
123
+ // inference is by definition complete — the parent only looks pending
124
+ // because `markParentProcessed` never ran (process killed between the
125
+ // child write and the mark) or the child was created externally (e.g.
126
+ // consolidation). Skip the LLM/cache call entirely and mark the parent
127
+ // so it never re-pends. Before this check, production measurements
128
+ // showed ~55% of the pass's LLM budget re-deriving such parents only to
129
+ // discover the existing child after the fact.
130
+ if (fs.existsSync(derivedChildPath(record))) {
131
+ markParentProcessed(record);
132
+ return {
133
+ skipped: false,
134
+ splitParent: false,
135
+ written: 0,
136
+ fromCache: false,
137
+ retryAttempts: 0,
138
+ childExists: true,
139
+ precheck: true,
140
+ };
141
+ }
122
142
  // Incremental cache: skip LLM call when body hash is unchanged and
123
143
  // --re-enrich was not requested. The cache ref is the absolute file path.
124
144
  const validate = (raw) => {
@@ -171,23 +191,30 @@ export async function runMemoryInferencePass(ctx) {
171
191
  return { skipped: false, splitParent: true, written: writeOutcome.written, fromCache, retryAttempts };
172
192
  }
173
193
  // LLM produced a valid derived draft but no file was written — either
174
- // because `<parent>.derived.md` already exists on disk or
175
- // `writeAssetToSource` threw. Categorise as `childExists` so the
176
- // attempt is accounted for in health metrics rather than vanishing
177
- // into the freshAttempts denominator.
194
+ // because `<parent>.derived.md` appeared on disk after the pre-check
195
+ // above (a rare mid-flight race) or `writeAssetToSource` threw.
196
+ // Categorise as `childExists` so the consumed attempt is accounted for
197
+ // in health metrics rather than vanishing into the freshAttempts
198
+ // denominator.
178
199
  //
179
- // When the child already exists on disk the inference is, by definition,
180
- // already complete — so mark the parent processed here too (#550).
181
- // Without this, `isPendingMemory()` re-queues the same parent every run
182
- // (the `written > 0` path was previously the only site that marks it),
183
- // causing permanent re-queueing and wasted LLM calls. A genuine write
184
- // *failure* (`writeAssetToSource` threw) must NOT mark the parent it
185
- // should be retried next run — so we key off the explicit `childExists`
186
- // outcome rather than the conflated `written === 0`.
200
+ // When the child exists the inference is, by definition, complete — so
201
+ // mark the parent processed here too (#550), otherwise
202
+ // `isPendingMemory()` re-queues the same parent every run. A genuine
203
+ // write *failure* (`writeAssetToSource` threw) must NOT mark the parent
204
+ // it should be retried next run so we key off the explicit
205
+ // `childExists` outcome rather than the conflated `written === 0`.
187
206
  if (writeOutcome.childExists) {
188
207
  markParentProcessed(record);
189
208
  }
190
- return { skipped: false, splitParent: false, written: 0, fromCache, retryAttempts, childExists: true };
209
+ return {
210
+ skipped: false,
211
+ splitParent: false,
212
+ written: 0,
213
+ fromCache,
214
+ retryAttempts,
215
+ childExists: true,
216
+ precheck: false,
217
+ };
191
218
  },
192
219
  // Default concurrency of 4 for cloud APIs. Set `llm.concurrency: 1`
193
220
  // in config.json for local model servers (LM Studio, Ollama).
@@ -224,11 +251,16 @@ export async function runMemoryInferencePass(ctx) {
224
251
  result.writtenFacts += res.written;
225
252
  }
226
253
  else if ("childExists" in res && res.childExists) {
227
- // LLM call was consumed but the derived file already existed (or the
228
- // write threw). Track separately so this category is observable in
229
- // health output and stops bleeding into the freshAttempts denominator.
254
+ // Derived child already on disk. Track separately so this category is
255
+ // observable in health output and stops bleeding into the
256
+ // freshAttempts denominator. Pre-check skips (#588) are the routine
257
+ // self-healing path — no LLM attempt was consumed and the parent has
258
+ // been marked processed — so only the rare post-LLM case (mid-flight
259
+ // race or write failure) warrants a per-ref warning.
230
260
  result.skippedChildExists += 1;
231
- warn(`memory inference: derived child for ${pending[i]?.ref ?? "<unknown>"} already existed or write failed; counted as skippedChildExists`);
261
+ if (!res.precheck) {
262
+ warn(`memory inference: derived child for ${pending[i]?.ref ?? "<unknown>"} already existed or write failed; counted as skippedChildExists`);
263
+ }
232
264
  }
233
265
  else {
234
266
  // The per-record state machine should cover every outcome. A hit here
@@ -324,6 +356,14 @@ function toMemoryName(memoriesDir, filePath) {
324
356
  // user has organised under memories/.
325
357
  return rel.replace(/\\/g, "/").replace(/\.md$/i, "");
326
358
  }
359
+ /**
360
+ * Absolute path of the derived child for a parent memory. Single source of
361
+ * truth for the `<parent>.derived.md` naming convention — used both by the
362
+ * pre-LLM existence check (#588) and the write path.
363
+ */
364
+ function derivedChildPath(parent) {
365
+ return path.join(parent.stashRoot, "memories", `${parent.name}.derived.md`);
366
+ }
327
367
  async function writeDerivedMemory(parent, derived) {
328
368
  const writeTarget = {
329
369
  kind: "filesystem",
@@ -338,11 +378,10 @@ async function writeDerivedMemory(parent, derived) {
338
378
  };
339
379
  const childName = `${parent.name}.derived`;
340
380
  const childRefStr = `memory:${childName}`;
341
- const childPath = path.join(parent.stashRoot, "memories", `${childName}.md`);
342
- if (fs.existsSync(childPath)) {
343
- // The derived child is already on disk inference for this parent is
344
- // complete. Report `childExists` so the caller marks the parent processed
345
- // (#550) instead of re-queueing it forever.
381
+ if (fs.existsSync(derivedChildPath(parent))) {
382
+ // The derived child appeared on disk after the caller's pre-check (#588)
383
+ // a rare mid-flight race. Report `childExists` so the caller marks the
384
+ // parent processed (#550) instead of re-queueing it forever.
346
385
  return { written: 0, childExists: true };
347
386
  }
348
387
  try {
@@ -5,7 +5,19 @@ import fs from "node:fs";
5
5
  import os from "node:os";
6
6
  import path from "node:path";
7
7
  import { extractInlineRefMentions } from "../../session-logs/inline-refs.js";
8
- const CLAUDE_PROJECTS_DIR = path.join(os.homedir(), ".claude", "projects");
8
+ /**
9
+ * Root directory holding Claude Code's per-project JSONL session logs.
10
+ *
11
+ * Resolved per call (not memoized at module load) so the `AKM_CLAUDE_PROJECTS_DIR`
12
+ * override can be set after import. The override exists so tests — and the
13
+ * isolated-storage sandbox — can point the scan at an empty fixture directory
14
+ * instead of the real `~/.claude/projects`, which on an actively-used machine
15
+ * holds many large session files and would make `akm health` (which scans it
16
+ * synchronously) slow and non-hermetic.
17
+ */
18
+ function claudeProjectsDir() {
19
+ return process.env.AKM_CLAUDE_PROJECTS_DIR ?? path.join(os.homedir(), ".claude", "projects");
20
+ }
9
21
  /**
10
22
  * Parse a single Claude Code JSONL event into a normalized {@link SessionEvent}.
11
23
  * Returns `undefined` for events that don't carry textual content (file
@@ -93,11 +105,11 @@ export class ClaudeCodeProvider {
93
105
  // HARNESS_BY_ID.get("claude").runtimeId.
94
106
  name = "claude-code";
95
107
  isAvailable() {
96
- return fs.existsSync(CLAUDE_PROJECTS_DIR);
108
+ return fs.existsSync(claudeProjectsDir());
97
109
  }
98
110
  *readEvents(input) {
99
111
  try {
100
- for (const jsonlPath of this.#walkJsonl(CLAUDE_PROJECTS_DIR)) {
112
+ for (const jsonlPath of this.#walkJsonl(claudeProjectsDir())) {
101
113
  const stat = fs.statSync(jsonlPath);
102
114
  if (stat.mtimeMs < input.sinceMs)
103
115
  continue;
@@ -128,7 +140,7 @@ export class ClaudeCodeProvider {
128
140
  }
129
141
  }
130
142
  listSessions(input = {}) {
131
- const root = input.location ?? CLAUDE_PROJECTS_DIR;
143
+ const root = input.location ?? claudeProjectsDir();
132
144
  const sinceMs = input.sinceMs ?? 0;
133
145
  const summaries = [];
134
146
  try {
@@ -286,7 +298,7 @@ export class ClaudeCodeProvider {
286
298
  const full = path.join(dir, entry.name);
287
299
  if (entry.isDirectory())
288
300
  yield* this.#walkJsonl(full);
289
- else if (entry.name.endsWith(".jsonl"))
301
+ else if (entry.name.endsWith(".jsonl") && entry.name !== "journal.jsonl")
290
302
  yield full;
291
303
  }
292
304
  }
@@ -14,6 +14,7 @@ import { fetchWithTimeout } from "../core/common.js";
14
14
  import { resolveSecret } from "../core/config/config.js";
15
15
  import { escapeJsonStringControls, parseJsonResponse, stripCodeFences, stripThinkBlocks } from "../core/parse.js";
16
16
  import { warnVerbose } from "../core/warn.js";
17
+ import { emitLlmUsage, extractUsageTokens } from "./usage-telemetry.js";
17
18
  // Re-export shared parse utilities so existing importers of `client.ts` continue
18
19
  // to resolve `parseJsonResponse` and `parseEmbeddedJsonResponse` from this module.
19
20
  export { escapeJsonStringControls, parseEmbeddedJsonResponse, parseJsonResponse, stripCodeFences, stripThinkBlocks, } from "../core/parse.js";
@@ -118,9 +119,23 @@ function looksLikeContextOverflow(message) {
118
119
  /**
119
120
  * Decide whether a first-attempt {@link LlmCallError} is eligible for a single
120
121
  * retry. Retryable: HTTP 5xx (`provider_error` with statusCode >= 500) and
121
- * `network_error` whose message looks like a transient connection reset
122
- * (ECONNRESET / EPIPE / "fetch failed"). NOT retryable: 4xx, `rate_limited`
123
- * (429), `timeout`, `parse_error`, and context-overflow-classified errors.
122
+ * `network_error` whose message looks like a transient connection drop.
123
+ * NOT retryable: 4xx, `rate_limited` (429), `timeout`, `parse_error`, and
124
+ * context-overflow-classified errors.
125
+ *
126
+ * The connection-drop heuristic covers the substrings emitted across runtimes
127
+ * for a mid-flight socket close:
128
+ * - `ECONNRESET` / `EPIPE` — Node/libuv socket reset codes
129
+ * - `fetch failed` — undici's generic wrapper message
130
+ * - `socket connection was closed` — Bun's message for a dropped connection
131
+ * (e.g. "The socket connection was closed unexpectedly.")
132
+ * - `terminated` / `other side closed` — undici's phrasings for the same
133
+ *
134
+ * These all describe a transient transport failure where a second attempt can
135
+ * legitimately succeed, which is exactly the case a single bounded retry is
136
+ * meant to absorb. Before this list was widened, Bun's "socket connection was
137
+ * closed unexpectedly" fell through unretried and surfaced as a recurring
138
+ * failure in the improve/reflect and capability-probe flows.
124
139
  */
125
140
  function isRetryable(err) {
126
141
  if (looksLikeContextOverflow(err.message))
@@ -130,7 +145,12 @@ function isRetryable(err) {
130
145
  }
131
146
  if (err.code === "network_error") {
132
147
  const lower = err.message.toLowerCase();
133
- return lower.includes("econnreset") || lower.includes("epipe") || lower.includes("fetch failed");
148
+ return (lower.includes("econnreset") ||
149
+ lower.includes("epipe") ||
150
+ lower.includes("fetch failed") ||
151
+ lower.includes("socket connection was closed") ||
152
+ lower.includes("terminated") ||
153
+ lower.includes("other side closed"));
134
154
  }
135
155
  return false;
136
156
  }
@@ -179,6 +199,10 @@ async function chatCompletionAttempt(config, messages, options, timeoutMs) {
179
199
  const responseFormat = options?.responseSchema && config.supportsJsonSchema
180
200
  ? { response_format: { type: "json_schema", json_schema: { schema: options.responseSchema, strict: true } } }
181
201
  : {};
202
+ // Wall-clock start for per-call usage telemetry (#576). Captured here so the
203
+ // emitted duration covers the full request/response/parse cycle of a single
204
+ // attempt, not the retry-wrapping `chatCompletion`.
205
+ const requestStartedAt = Date.now();
182
206
  let response;
183
207
  try {
184
208
  response = await fetchWithTimeout(config.endpoint, {
@@ -241,6 +265,16 @@ async function chatCompletionAttempt(config, messages, options, timeoutMs) {
241
265
  catch {
242
266
  throw new LlmCallError(`LLM response was not valid JSON ${config.endpoint}: ${redactErrorBody(rawOkBody)}`, "parse_error", response.status);
243
267
  }
268
+ // Per-call usage telemetry (#576). Best-effort and fully isolated: a missing
269
+ // or garbled usage block still records duration + model, and a throwing sink
270
+ // can never fail the call (emitLlmUsage swallows its own errors). The stage
271
+ // is supplied ambiently by emitLlmUsage; no `stage` param is threaded here.
272
+ emitLlmUsage({
273
+ model: typeof json.model === "string" && json.model ? json.model : config.model,
274
+ durationMs: Date.now() - requestStartedAt,
275
+ finishReason: typeof json.choices?.[0]?.finish_reason === "string" ? json.choices[0].finish_reason : undefined,
276
+ ...extractUsageTokens(json.usage),
277
+ });
244
278
  const content = (json.choices?.[0]?.message?.content ?? "").trim();
245
279
  const reasoning = (json.choices?.[0]?.message?.reasoning_content ?? "").trim();
246
280
  return content || reasoning;