akm-cli 0.9.0-beta.0 → 0.9.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +511 -0
- package/dist/assets/profiles/quick.json +2 -1
- package/dist/assets/templates/html/default.html +78 -0
- package/dist/assets/templates/html/health.html +732 -0
- package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
- package/dist/cli/shared.js +21 -5
- package/dist/cli.js +47 -5
- package/dist/commands/config-cli.js +0 -10
- package/dist/commands/feedback-cli.js +42 -37
- package/dist/commands/graph/graph.js +75 -71
- package/dist/commands/health/checks.js +48 -0
- package/dist/commands/health/html-report.js +666 -0
- package/dist/commands/health.js +186 -13
- package/dist/commands/improve/consolidate.js +39 -6
- package/dist/commands/improve/distill.js +26 -5
- package/dist/commands/improve/extract-prompt.js +1 -1
- package/dist/commands/improve/extract.js +52 -8
- package/dist/commands/improve/improve-auto-accept.js +33 -1
- package/dist/commands/improve/improve-cli.js +7 -0
- package/dist/commands/improve/improve-profiles.js +4 -0
- package/dist/commands/improve/improve.js +877 -433
- package/dist/commands/improve/proactive-maintenance.js +113 -0
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/improve/reflect.js +31 -0
- package/dist/commands/proposal/drain.js +73 -6
- package/dist/commands/proposal/proposal-cli.js +22 -10
- package/dist/commands/proposal/proposal.js +17 -1
- package/dist/commands/proposal/validators/proposals.js +365 -329
- package/dist/commands/read/curate.js +17 -0
- package/dist/commands/remember.js +6 -2
- package/dist/commands/sources/stash-cli.js +10 -2
- package/dist/commands/tasks/tasks.js +32 -8
- package/dist/core/config/config-schema.js +30 -0
- package/dist/core/file-lock.js +22 -0
- package/dist/core/logs-db.js +304 -0
- package/dist/core/paths.js +3 -0
- package/dist/core/state-db.js +152 -14
- package/dist/indexer/db/db.js +99 -13
- package/dist/indexer/ensure-index.js +152 -17
- package/dist/indexer/index-writer-lock.js +99 -0
- package/dist/indexer/indexer.js +114 -111
- package/dist/indexer/passes/memory-inference.js +61 -22
- package/dist/integrations/harnesses/claude/session-log.js +17 -5
- package/dist/llm/client.js +38 -4
- package/dist/llm/usage-persist.js +77 -0
- package/dist/llm/usage-telemetry.js +103 -0
- package/dist/output/context.js +3 -2
- package/dist/output/html-render.js +73 -0
- package/dist/output/shapes/helpers.js +17 -1
- package/dist/output/text/helpers.js +69 -1
- package/dist/scripts/migrate-storage.js +153 -25
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +21 -2
- package/dist/sources/providers/tar-utils.js +16 -8
- package/dist/tasks/backends/cron.js +46 -9
- package/dist/tasks/runner.js +99 -16
- package/dist/workflows/db.js +4 -0
- package/package.json +3 -2
- package/dist/commands/config-edit.js +0 -344
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import fs from "node:fs";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { probeLock, releaseLock, releaseLockIfOwned, tryAcquireLockSync } from "../core/file-lock.js";
|
|
7
|
+
import { getDbPath, getIndexWriterLockPath } from "../core/paths.js";
|
|
8
|
+
const INDEX_WRITER_LOCK_STALE_AFTER_MS = 12 * 60 * 60 * 1000;
|
|
9
|
+
const INDEX_WRITER_WAIT_MS = 100;
|
|
10
|
+
const heldLocks = new Map();
|
|
11
|
+
function buildPayload(purpose, pid = process.pid) {
|
|
12
|
+
return JSON.stringify({
|
|
13
|
+
pid,
|
|
14
|
+
purpose,
|
|
15
|
+
dbPath: getDbPath(),
|
|
16
|
+
startedAt: new Date().toISOString(),
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
function delay(ms) {
|
|
20
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
21
|
+
}
|
|
22
|
+
function throwIfAborted(signal) {
|
|
23
|
+
if (!signal?.aborted)
|
|
24
|
+
return;
|
|
25
|
+
throw signal.reason instanceof Error ? signal.reason : new Error("index writer wait aborted");
|
|
26
|
+
}
|
|
27
|
+
function releaseHeldLock(lockPath) {
|
|
28
|
+
const held = heldLocks.get(lockPath);
|
|
29
|
+
if (!held)
|
|
30
|
+
return;
|
|
31
|
+
held.depth -= 1;
|
|
32
|
+
if (held.depth > 0)
|
|
33
|
+
return;
|
|
34
|
+
heldLocks.delete(lockPath);
|
|
35
|
+
process.off("exit", held.exitHandler);
|
|
36
|
+
releaseLockIfOwned(lockPath, process.pid);
|
|
37
|
+
}
|
|
38
|
+
function retainHeldLock(lockPath) {
|
|
39
|
+
const existing = heldLocks.get(lockPath);
|
|
40
|
+
if (existing) {
|
|
41
|
+
existing.depth += 1;
|
|
42
|
+
return { lockPath, release: () => releaseHeldLock(lockPath) };
|
|
43
|
+
}
|
|
44
|
+
const exitHandler = () => releaseLockIfOwned(lockPath, process.pid);
|
|
45
|
+
process.on("exit", exitHandler);
|
|
46
|
+
heldLocks.set(lockPath, { depth: 1, exitHandler });
|
|
47
|
+
return { lockPath, release: () => releaseHeldLock(lockPath) };
|
|
48
|
+
}
|
|
49
|
+
function detachHeldLock(lockPath) {
|
|
50
|
+
const held = heldLocks.get(lockPath);
|
|
51
|
+
if (!held)
|
|
52
|
+
return;
|
|
53
|
+
heldLocks.delete(lockPath);
|
|
54
|
+
process.off("exit", held.exitHandler);
|
|
55
|
+
}
|
|
56
|
+
export async function acquireIndexWriterLease(options) {
|
|
57
|
+
const mode = options.mode ?? "wait";
|
|
58
|
+
const lockPath = getIndexWriterLockPath();
|
|
59
|
+
fs.mkdirSync(path.dirname(lockPath), { recursive: true });
|
|
60
|
+
if (heldLocks.has(lockPath)) {
|
|
61
|
+
return retainHeldLock(lockPath);
|
|
62
|
+
}
|
|
63
|
+
while (true) {
|
|
64
|
+
throwIfAborted(options.signal);
|
|
65
|
+
if (tryAcquireLockSync(lockPath, buildPayload(options.purpose))) {
|
|
66
|
+
return retainHeldLock(lockPath);
|
|
67
|
+
}
|
|
68
|
+
const probe = probeLock(lockPath, { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
|
|
69
|
+
if (probe.state === "held" && probe.holderPid === process.pid) {
|
|
70
|
+
return retainHeldLock(lockPath);
|
|
71
|
+
}
|
|
72
|
+
if (probe.state === "stale") {
|
|
73
|
+
releaseLock(lockPath);
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (mode === "try")
|
|
77
|
+
return undefined;
|
|
78
|
+
await delay(INDEX_WRITER_WAIT_MS);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
export async function withIndexWriterLease(options, run) {
|
|
82
|
+
const lease = await acquireIndexWriterLease(options);
|
|
83
|
+
if (!lease) {
|
|
84
|
+
throw new Error(`index writer lease unavailable for ${options.purpose}`);
|
|
85
|
+
}
|
|
86
|
+
try {
|
|
87
|
+
return await run();
|
|
88
|
+
}
|
|
89
|
+
finally {
|
|
90
|
+
lease.release();
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
export function handoffIndexWriterLeaseToPid(lease, pid, purpose) {
|
|
94
|
+
fs.writeFileSync(lease.lockPath, buildPayload(purpose, pid), "utf8");
|
|
95
|
+
detachHeldLock(lease.lockPath);
|
|
96
|
+
}
|
|
97
|
+
export function probeIndexWriterLease() {
|
|
98
|
+
return probeLock(getIndexWriterLockPath(), { staleAfterMs: INDEX_WRITER_LOCK_STALE_AFTER_MS });
|
|
99
|
+
}
|
package/dist/indexer/indexer.js
CHANGED
|
@@ -12,6 +12,7 @@ import { resolveIndexPassLLM } from "../llm/index-passes.js";
|
|
|
12
12
|
import { takeWorkflowDocument } from "../workflows/runtime/document-cache.js";
|
|
13
13
|
import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddableEntryCount, getEmbeddingCount, getEntriesByDir, getEntryCount, getIndexDirState, getMeta, isVecAvailable, openDatabase, openExistingDatabase, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db/db.js";
|
|
14
14
|
import { deleteStoredGraph } from "./db/graph-db.js";
|
|
15
|
+
import { withIndexWriterLease } from "./index-writer-lock.js";
|
|
15
16
|
import { applyCuratedFrontmatter, applyWikiFrontmatter, generateMetadataFlat, isEnrichmentComplete, isWorkflowSkipWarning, loadStashFile, shouldIndexStashFile, } from "./passes/metadata.js";
|
|
16
17
|
import { buildSearchText } from "./search/search-fields.js";
|
|
17
18
|
import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
|
|
@@ -225,119 +226,121 @@ function runCleanPass(db, dryRun) {
|
|
|
225
226
|
}
|
|
226
227
|
// ── Indexer ──────────────────────────────────────────────────────────────────
|
|
227
228
|
export async function akmIndex(options) {
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
229
|
+
return withIndexWriterLease({ purpose: "akm-index", signal: options?.signal }, async () => {
|
|
230
|
+
const stashDir = options?.stashDir || resolveStashDir();
|
|
231
|
+
const onProgress = options?.onProgress ?? (() => { });
|
|
232
|
+
const signal = options?.signal;
|
|
233
|
+
const reEnrich = options?.reEnrich === true;
|
|
234
|
+
const full = options?.full === true;
|
|
235
|
+
const clean = options?.clean === true;
|
|
236
|
+
const dryRun = options?.dryRun === true;
|
|
237
|
+
// Load config and resolve all stash sources
|
|
238
|
+
const { loadConfig } = await import("../core/config/config.js");
|
|
239
|
+
const config = loadConfig();
|
|
240
|
+
// One-time, read-only guard: warn if the writable stash still holds an
|
|
241
|
+
// un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
|
|
242
|
+
// entirely, so an unmigrated vault's `.env` data would silently never be
|
|
243
|
+
// indexed. Non-destructive — only stats, never reads/writes/deletes.
|
|
244
|
+
const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
|
|
245
|
+
warnOnUnmigratedVaults(stashDir);
|
|
246
|
+
// Ensure git stash caches are extracted before resolving stash dirs,
|
|
247
|
+
// so their content directories exist on disk for the walker to discover.
|
|
248
|
+
const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
|
|
249
|
+
await ensureSourceCaches(config, { force: full });
|
|
250
|
+
const allSourceEntries = resolveSourceEntries(stashDir, config);
|
|
251
|
+
const allSourceDirs = allSourceEntries.map((s) => s.path);
|
|
252
|
+
const t0 = Date.now();
|
|
253
|
+
// Open database — pass embedding dimension from config if available
|
|
254
|
+
const dbPath = getDbPath();
|
|
255
|
+
const embeddingDim = config.embedding?.dimension;
|
|
256
|
+
const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
|
|
257
|
+
try {
|
|
258
|
+
// Determine incremental vs full mode
|
|
259
|
+
const prevStashDir = getMeta(db, "stashDir");
|
|
260
|
+
const prevBuiltAt = getMeta(db, "builtAt");
|
|
261
|
+
const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
|
|
262
|
+
const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
|
|
263
|
+
// Assemble the run context
|
|
264
|
+
const ctx = {
|
|
265
|
+
db,
|
|
266
|
+
config,
|
|
267
|
+
sources: allSourceEntries,
|
|
268
|
+
sourceDirs: allSourceDirs,
|
|
269
|
+
full,
|
|
270
|
+
reEnrich,
|
|
271
|
+
stashDir,
|
|
272
|
+
onProgress,
|
|
273
|
+
signal,
|
|
274
|
+
timing: {
|
|
275
|
+
t0,
|
|
276
|
+
tWalkStart: t0,
|
|
277
|
+
tWalkEnd: t0,
|
|
278
|
+
tLlmEnd: t0,
|
|
279
|
+
tFtsEnd: t0,
|
|
280
|
+
tEmbedEnd: t0,
|
|
281
|
+
},
|
|
282
|
+
isIncremental,
|
|
283
|
+
builtAtMs,
|
|
284
|
+
hadRemovedSources: false,
|
|
285
|
+
scannedDirs: 0,
|
|
286
|
+
skippedDirs: 0,
|
|
287
|
+
generatedCount: 0,
|
|
288
|
+
walkWarnings: [],
|
|
289
|
+
dirsNeedingLlm: [],
|
|
290
|
+
embeddingResult: null,
|
|
291
|
+
graphExtractionResult: null,
|
|
292
|
+
};
|
|
293
|
+
onProgress({
|
|
294
|
+
phase: "summary",
|
|
295
|
+
message: buildIndexSummaryMessage({
|
|
296
|
+
mode: isIncremental ? "incremental" : "full",
|
|
297
|
+
sourcesCount: allSourceDirs.length,
|
|
298
|
+
semanticSearchMode: config.semanticSearchMode,
|
|
299
|
+
embeddingProvider: getEmbeddingProvider(config.embedding),
|
|
300
|
+
llmEnabled: !!resolveIndexPassLLM("enrichment", config),
|
|
301
|
+
vecAvailable: isVecAvailable(db),
|
|
302
|
+
}),
|
|
303
|
+
});
|
|
304
|
+
// ── Phase sequence ───────────────────────────────────────────────────────
|
|
305
|
+
await runSourceCachePhase(ctx);
|
|
306
|
+
await runWalkPhase(ctx);
|
|
307
|
+
await runEmbeddingPhase(ctx);
|
|
308
|
+
await runFinalizePhase(ctx);
|
|
309
|
+
// ────────────────────────────────────────────────────────────────────────
|
|
310
|
+
const { _verification: verification, _totalEntries: totalEntries } = ctx;
|
|
311
|
+
const { timing } = ctx;
|
|
312
|
+
// ── Clean pass ───────────────────────────────────────────────────────────
|
|
313
|
+
// After the normal index completes, remove entries whose source files no
|
|
314
|
+
// longer exist on disk. Remote entries (empty file_path) are skipped.
|
|
315
|
+
let cleanResult;
|
|
316
|
+
if (clean) {
|
|
317
|
+
cleanResult = runCleanPass(db, dryRun);
|
|
318
|
+
}
|
|
319
|
+
// ────────────────────────────────────────────────────────────────────────
|
|
320
|
+
return {
|
|
321
|
+
stashDir,
|
|
322
|
+
totalEntries,
|
|
323
|
+
generatedMetadata: ctx.generatedCount,
|
|
324
|
+
indexPath: dbPath,
|
|
294
325
|
mode: isIncremental ? "incremental" : "full",
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
const { _verification: verification, _totalEntries: totalEntries } = ctx;
|
|
309
|
-
const { timing } = ctx;
|
|
310
|
-
// ── Clean pass ───────────────────────────────────────────────────────────
|
|
311
|
-
// After the normal index completes, remove entries whose source files no
|
|
312
|
-
// longer exist on disk. Remote entries (empty file_path) are skipped.
|
|
313
|
-
let cleanResult;
|
|
314
|
-
if (clean) {
|
|
315
|
-
cleanResult = runCleanPass(db, dryRun);
|
|
326
|
+
directoriesScanned: ctx.scannedDirs,
|
|
327
|
+
directoriesSkipped: ctx.skippedDirs,
|
|
328
|
+
...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
|
|
329
|
+
verification,
|
|
330
|
+
timing: {
|
|
331
|
+
totalMs: Date.now() - timing.t0,
|
|
332
|
+
walkMs: timing.tWalkEnd - timing.tWalkStart,
|
|
333
|
+
llmMs: timing.tLlmEnd - timing.tWalkEnd,
|
|
334
|
+
embedMs: timing.tEmbedEnd - timing.tLlmEnd,
|
|
335
|
+
ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
|
|
336
|
+
},
|
|
337
|
+
...(cleanResult !== undefined ? { clean: cleanResult } : {}),
|
|
338
|
+
};
|
|
316
339
|
}
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
generatedMetadata: ctx.generatedCount,
|
|
322
|
-
indexPath: dbPath,
|
|
323
|
-
mode: isIncremental ? "incremental" : "full",
|
|
324
|
-
directoriesScanned: ctx.scannedDirs,
|
|
325
|
-
directoriesSkipped: ctx.skippedDirs,
|
|
326
|
-
...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
|
|
327
|
-
verification,
|
|
328
|
-
timing: {
|
|
329
|
-
totalMs: Date.now() - timing.t0,
|
|
330
|
-
walkMs: timing.tWalkEnd - timing.tWalkStart,
|
|
331
|
-
llmMs: timing.tLlmEnd - timing.tWalkEnd,
|
|
332
|
-
embedMs: timing.tEmbedEnd - timing.tLlmEnd,
|
|
333
|
-
ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
|
|
334
|
-
},
|
|
335
|
-
...(cleanResult !== undefined ? { clean: cleanResult } : {}),
|
|
336
|
-
};
|
|
337
|
-
}
|
|
338
|
-
finally {
|
|
339
|
-
closeDatabase(db);
|
|
340
|
-
}
|
|
340
|
+
finally {
|
|
341
|
+
closeDatabase(db);
|
|
342
|
+
}
|
|
343
|
+
});
|
|
341
344
|
}
|
|
342
345
|
// ── Extracted helpers for indexing ────────────────────────────────────────────
|
|
343
346
|
async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, doFullDelete = false, onProgress) {
|
|
@@ -119,6 +119,26 @@ export async function runMemoryInferencePass(ctx) {
|
|
|
119
119
|
// 2026-05-26).
|
|
120
120
|
if (signal?.aborted)
|
|
121
121
|
return { aborted: true };
|
|
122
|
+
// Pre-check (#588): when `<parent>.derived.md` is already on disk the
|
|
123
|
+
// inference is by definition complete — the parent only looks pending
|
|
124
|
+
// because `markParentProcessed` never ran (process killed between the
|
|
125
|
+
// child write and the mark) or the child was created externally (e.g.
|
|
126
|
+
// consolidation). Skip the LLM/cache call entirely and mark the parent
|
|
127
|
+
// so it never re-pends. Before this check, production measurements
|
|
128
|
+
// showed ~55% of the pass's LLM budget re-deriving such parents only to
|
|
129
|
+
// discover the existing child after the fact.
|
|
130
|
+
if (fs.existsSync(derivedChildPath(record))) {
|
|
131
|
+
markParentProcessed(record);
|
|
132
|
+
return {
|
|
133
|
+
skipped: false,
|
|
134
|
+
splitParent: false,
|
|
135
|
+
written: 0,
|
|
136
|
+
fromCache: false,
|
|
137
|
+
retryAttempts: 0,
|
|
138
|
+
childExists: true,
|
|
139
|
+
precheck: true,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
122
142
|
// Incremental cache: skip LLM call when body hash is unchanged and
|
|
123
143
|
// --re-enrich was not requested. The cache ref is the absolute file path.
|
|
124
144
|
const validate = (raw) => {
|
|
@@ -171,23 +191,30 @@ export async function runMemoryInferencePass(ctx) {
|
|
|
171
191
|
return { skipped: false, splitParent: true, written: writeOutcome.written, fromCache, retryAttempts };
|
|
172
192
|
}
|
|
173
193
|
// LLM produced a valid derived draft but no file was written — either
|
|
174
|
-
// because `<parent>.derived.md`
|
|
175
|
-
//
|
|
176
|
-
//
|
|
177
|
-
// into the freshAttempts
|
|
194
|
+
// because `<parent>.derived.md` appeared on disk after the pre-check
|
|
195
|
+
// above (a rare mid-flight race) or `writeAssetToSource` threw.
|
|
196
|
+
// Categorise as `childExists` so the consumed attempt is accounted for
|
|
197
|
+
// in health metrics rather than vanishing into the freshAttempts
|
|
198
|
+
// denominator.
|
|
178
199
|
//
|
|
179
|
-
// When the child
|
|
180
|
-
//
|
|
181
|
-
//
|
|
182
|
-
// (
|
|
183
|
-
//
|
|
184
|
-
//
|
|
185
|
-
// should be retried next run — so we key off the explicit `childExists`
|
|
186
|
-
// outcome rather than the conflated `written === 0`.
|
|
200
|
+
// When the child exists the inference is, by definition, complete — so
|
|
201
|
+
// mark the parent processed here too (#550), otherwise
|
|
202
|
+
// `isPendingMemory()` re-queues the same parent every run. A genuine
|
|
203
|
+
// write *failure* (`writeAssetToSource` threw) must NOT mark the parent
|
|
204
|
+
// — it should be retried next run — so we key off the explicit
|
|
205
|
+
// `childExists` outcome rather than the conflated `written === 0`.
|
|
187
206
|
if (writeOutcome.childExists) {
|
|
188
207
|
markParentProcessed(record);
|
|
189
208
|
}
|
|
190
|
-
return {
|
|
209
|
+
return {
|
|
210
|
+
skipped: false,
|
|
211
|
+
splitParent: false,
|
|
212
|
+
written: 0,
|
|
213
|
+
fromCache,
|
|
214
|
+
retryAttempts,
|
|
215
|
+
childExists: true,
|
|
216
|
+
precheck: false,
|
|
217
|
+
};
|
|
191
218
|
},
|
|
192
219
|
// Default concurrency of 4 for cloud APIs. Set `llm.concurrency: 1`
|
|
193
220
|
// in config.json for local model servers (LM Studio, Ollama).
|
|
@@ -224,11 +251,16 @@ export async function runMemoryInferencePass(ctx) {
|
|
|
224
251
|
result.writtenFacts += res.written;
|
|
225
252
|
}
|
|
226
253
|
else if ("childExists" in res && res.childExists) {
|
|
227
|
-
//
|
|
228
|
-
//
|
|
229
|
-
//
|
|
254
|
+
// Derived child already on disk. Track separately so this category is
|
|
255
|
+
// observable in health output and stops bleeding into the
|
|
256
|
+
// freshAttempts denominator. Pre-check skips (#588) are the routine
|
|
257
|
+
// self-healing path — no LLM attempt was consumed and the parent has
|
|
258
|
+
// been marked processed — so only the rare post-LLM case (mid-flight
|
|
259
|
+
// race or write failure) warrants a per-ref warning.
|
|
230
260
|
result.skippedChildExists += 1;
|
|
231
|
-
|
|
261
|
+
if (!res.precheck) {
|
|
262
|
+
warn(`memory inference: derived child for ${pending[i]?.ref ?? "<unknown>"} already existed or write failed; counted as skippedChildExists`);
|
|
263
|
+
}
|
|
232
264
|
}
|
|
233
265
|
else {
|
|
234
266
|
// The per-record state machine should cover every outcome. A hit here
|
|
@@ -324,6 +356,14 @@ function toMemoryName(memoriesDir, filePath) {
|
|
|
324
356
|
// user has organised under memories/.
|
|
325
357
|
return rel.replace(/\\/g, "/").replace(/\.md$/i, "");
|
|
326
358
|
}
|
|
359
|
+
/**
|
|
360
|
+
* Absolute path of the derived child for a parent memory. Single source of
|
|
361
|
+
* truth for the `<parent>.derived.md` naming convention — used both by the
|
|
362
|
+
* pre-LLM existence check (#588) and the write path.
|
|
363
|
+
*/
|
|
364
|
+
function derivedChildPath(parent) {
|
|
365
|
+
return path.join(parent.stashRoot, "memories", `${parent.name}.derived.md`);
|
|
366
|
+
}
|
|
327
367
|
async function writeDerivedMemory(parent, derived) {
|
|
328
368
|
const writeTarget = {
|
|
329
369
|
kind: "filesystem",
|
|
@@ -338,11 +378,10 @@ async function writeDerivedMemory(parent, derived) {
|
|
|
338
378
|
};
|
|
339
379
|
const childName = `${parent.name}.derived`;
|
|
340
380
|
const childRefStr = `memory:${childName}`;
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
//
|
|
344
|
-
//
|
|
345
|
-
// (#550) instead of re-queueing it forever.
|
|
381
|
+
if (fs.existsSync(derivedChildPath(parent))) {
|
|
382
|
+
// The derived child appeared on disk after the caller's pre-check (#588)
|
|
383
|
+
// — a rare mid-flight race. Report `childExists` so the caller marks the
|
|
384
|
+
// parent processed (#550) instead of re-queueing it forever.
|
|
346
385
|
return { written: 0, childExists: true };
|
|
347
386
|
}
|
|
348
387
|
try {
|
|
@@ -5,7 +5,19 @@ import fs from "node:fs";
|
|
|
5
5
|
import os from "node:os";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { extractInlineRefMentions } from "../../session-logs/inline-refs.js";
|
|
8
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Root directory holding Claude Code's per-project JSONL session logs.
|
|
10
|
+
*
|
|
11
|
+
* Resolved per call (not memoized at module load) so the `AKM_CLAUDE_PROJECTS_DIR`
|
|
12
|
+
* override can be set after import. The override exists so tests — and the
|
|
13
|
+
* isolated-storage sandbox — can point the scan at an empty fixture directory
|
|
14
|
+
* instead of the real `~/.claude/projects`, which on an actively-used machine
|
|
15
|
+
* holds many large session files and would make `akm health` (which scans it
|
|
16
|
+
* synchronously) slow and non-hermetic.
|
|
17
|
+
*/
|
|
18
|
+
function claudeProjectsDir() {
|
|
19
|
+
return process.env.AKM_CLAUDE_PROJECTS_DIR ?? path.join(os.homedir(), ".claude", "projects");
|
|
20
|
+
}
|
|
9
21
|
/**
|
|
10
22
|
* Parse a single Claude Code JSONL event into a normalized {@link SessionEvent}.
|
|
11
23
|
* Returns `undefined` for events that don't carry textual content (file
|
|
@@ -93,11 +105,11 @@ export class ClaudeCodeProvider {
|
|
|
93
105
|
// HARNESS_BY_ID.get("claude").runtimeId.
|
|
94
106
|
name = "claude-code";
|
|
95
107
|
isAvailable() {
|
|
96
|
-
return fs.existsSync(
|
|
108
|
+
return fs.existsSync(claudeProjectsDir());
|
|
97
109
|
}
|
|
98
110
|
*readEvents(input) {
|
|
99
111
|
try {
|
|
100
|
-
for (const jsonlPath of this.#walkJsonl(
|
|
112
|
+
for (const jsonlPath of this.#walkJsonl(claudeProjectsDir())) {
|
|
101
113
|
const stat = fs.statSync(jsonlPath);
|
|
102
114
|
if (stat.mtimeMs < input.sinceMs)
|
|
103
115
|
continue;
|
|
@@ -128,7 +140,7 @@ export class ClaudeCodeProvider {
|
|
|
128
140
|
}
|
|
129
141
|
}
|
|
130
142
|
listSessions(input = {}) {
|
|
131
|
-
const root = input.location ??
|
|
143
|
+
const root = input.location ?? claudeProjectsDir();
|
|
132
144
|
const sinceMs = input.sinceMs ?? 0;
|
|
133
145
|
const summaries = [];
|
|
134
146
|
try {
|
|
@@ -286,7 +298,7 @@ export class ClaudeCodeProvider {
|
|
|
286
298
|
const full = path.join(dir, entry.name);
|
|
287
299
|
if (entry.isDirectory())
|
|
288
300
|
yield* this.#walkJsonl(full);
|
|
289
|
-
else if (entry.name.endsWith(".jsonl"))
|
|
301
|
+
else if (entry.name.endsWith(".jsonl") && entry.name !== "journal.jsonl")
|
|
290
302
|
yield full;
|
|
291
303
|
}
|
|
292
304
|
}
|
package/dist/llm/client.js
CHANGED
|
@@ -14,6 +14,7 @@ import { fetchWithTimeout } from "../core/common.js";
|
|
|
14
14
|
import { resolveSecret } from "../core/config/config.js";
|
|
15
15
|
import { escapeJsonStringControls, parseJsonResponse, stripCodeFences, stripThinkBlocks } from "../core/parse.js";
|
|
16
16
|
import { warnVerbose } from "../core/warn.js";
|
|
17
|
+
import { emitLlmUsage, extractUsageTokens } from "./usage-telemetry.js";
|
|
17
18
|
// Re-export shared parse utilities so existing importers of `client.ts` continue
|
|
18
19
|
// to resolve `parseJsonResponse` and `parseEmbeddedJsonResponse` from this module.
|
|
19
20
|
export { escapeJsonStringControls, parseEmbeddedJsonResponse, parseJsonResponse, stripCodeFences, stripThinkBlocks, } from "../core/parse.js";
|
|
@@ -118,9 +119,23 @@ function looksLikeContextOverflow(message) {
|
|
|
118
119
|
/**
|
|
119
120
|
* Decide whether a first-attempt {@link LlmCallError} is eligible for a single
|
|
120
121
|
* retry. Retryable: HTTP 5xx (`provider_error` with statusCode >= 500) and
|
|
121
|
-
* `network_error` whose message looks like a transient connection
|
|
122
|
-
*
|
|
123
|
-
*
|
|
122
|
+
* `network_error` whose message looks like a transient connection drop.
|
|
123
|
+
* NOT retryable: 4xx, `rate_limited` (429), `timeout`, `parse_error`, and
|
|
124
|
+
* context-overflow-classified errors.
|
|
125
|
+
*
|
|
126
|
+
* The connection-drop heuristic covers the substrings emitted across runtimes
|
|
127
|
+
* for a mid-flight socket close:
|
|
128
|
+
* - `ECONNRESET` / `EPIPE` — Node/libuv socket reset codes
|
|
129
|
+
* - `fetch failed` — undici's generic wrapper message
|
|
130
|
+
* - `socket connection was closed` — Bun's message for a dropped connection
|
|
131
|
+
* (e.g. "The socket connection was closed unexpectedly.")
|
|
132
|
+
* - `terminated` / `other side closed` — undici's phrasings for the same
|
|
133
|
+
*
|
|
134
|
+
* These all describe a transient transport failure where a second attempt can
|
|
135
|
+
* legitimately succeed, which is exactly the case a single bounded retry is
|
|
136
|
+
* meant to absorb. Before this list was widened, Bun's "socket connection was
|
|
137
|
+
* closed unexpectedly" fell through unretried and surfaced as a recurring
|
|
138
|
+
* failure in the improve/reflect and capability-probe flows.
|
|
124
139
|
*/
|
|
125
140
|
function isRetryable(err) {
|
|
126
141
|
if (looksLikeContextOverflow(err.message))
|
|
@@ -130,7 +145,12 @@ function isRetryable(err) {
|
|
|
130
145
|
}
|
|
131
146
|
if (err.code === "network_error") {
|
|
132
147
|
const lower = err.message.toLowerCase();
|
|
133
|
-
return lower.includes("econnreset") ||
|
|
148
|
+
return (lower.includes("econnreset") ||
|
|
149
|
+
lower.includes("epipe") ||
|
|
150
|
+
lower.includes("fetch failed") ||
|
|
151
|
+
lower.includes("socket connection was closed") ||
|
|
152
|
+
lower.includes("terminated") ||
|
|
153
|
+
lower.includes("other side closed"));
|
|
134
154
|
}
|
|
135
155
|
return false;
|
|
136
156
|
}
|
|
@@ -179,6 +199,10 @@ async function chatCompletionAttempt(config, messages, options, timeoutMs) {
|
|
|
179
199
|
const responseFormat = options?.responseSchema && config.supportsJsonSchema
|
|
180
200
|
? { response_format: { type: "json_schema", json_schema: { schema: options.responseSchema, strict: true } } }
|
|
181
201
|
: {};
|
|
202
|
+
// Wall-clock start for per-call usage telemetry (#576). Captured here so the
|
|
203
|
+
// emitted duration covers the full request/response/parse cycle of a single
|
|
204
|
+
// attempt, not the retry-wrapping `chatCompletion`.
|
|
205
|
+
const requestStartedAt = Date.now();
|
|
182
206
|
let response;
|
|
183
207
|
try {
|
|
184
208
|
response = await fetchWithTimeout(config.endpoint, {
|
|
@@ -241,6 +265,16 @@ async function chatCompletionAttempt(config, messages, options, timeoutMs) {
|
|
|
241
265
|
catch {
|
|
242
266
|
throw new LlmCallError(`LLM response was not valid JSON ${config.endpoint}: ${redactErrorBody(rawOkBody)}`, "parse_error", response.status);
|
|
243
267
|
}
|
|
268
|
+
// Per-call usage telemetry (#576). Best-effort and fully isolated: a missing
|
|
269
|
+
// or garbled usage block still records duration + model, and a throwing sink
|
|
270
|
+
// can never fail the call (emitLlmUsage swallows its own errors). The stage
|
|
271
|
+
// is supplied ambiently by emitLlmUsage; no `stage` param is threaded here.
|
|
272
|
+
emitLlmUsage({
|
|
273
|
+
model: typeof json.model === "string" && json.model ? json.model : config.model,
|
|
274
|
+
durationMs: Date.now() - requestStartedAt,
|
|
275
|
+
finishReason: typeof json.choices?.[0]?.finish_reason === "string" ? json.choices[0].finish_reason : undefined,
|
|
276
|
+
...extractUsageTokens(json.usage),
|
|
277
|
+
});
|
|
244
278
|
const content = (json.choices?.[0]?.message?.content ?? "").trim();
|
|
245
279
|
const reasoning = (json.choices?.[0]?.message?.reasoning_content ?? "").trim();
|
|
246
280
|
return content || reasoning;
|