akm-cli 0.9.0-beta.2 → 0.9.0-beta.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +660 -0
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +5 -1
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/templates/html/default.html +78 -0
- package/dist/assets/templates/html/health.html +730 -0
- package/dist/assets/templates/html/vendor/echarts.min.js +45 -0
- package/dist/cli/shared.js +21 -5
- package/dist/cli.js +47 -5
- package/dist/commands/agent/contribute-cli.js +16 -3
- package/dist/commands/feedback-cli.js +15 -6
- package/dist/commands/graph/graph.js +75 -71
- package/dist/commands/health/checks.js +48 -0
- package/dist/commands/health/html-report.js +790 -0
- package/dist/commands/health.js +478 -15
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/consolidate.js +634 -111
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill.js +145 -69
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +115 -1
- package/dist/commands/improve/extract-prompt.js +33 -2
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +280 -35
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/homeostatic.js +467 -0
- package/dist/commands/improve/improve-auto-accept.js +139 -6
- package/dist/commands/improve/improve-profiles.js +12 -0
- package/dist/commands/improve/improve.js +2079 -608
- package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
- package/dist/commands/improve/outcome-loop.js +256 -0
- package/dist/commands/improve/proactive-maintenance.js +87 -0
- package/dist/commands/improve/procedural.js +409 -0
- package/dist/commands/improve/recombine.js +488 -0
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/improve/reflect.js +51 -1
- package/dist/commands/improve/related-sessions.js +120 -0
- package/dist/commands/improve/salience.js +386 -0
- package/dist/commands/improve/triage.js +95 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +34 -1
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +31 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +7 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/proposal/drain.js +73 -6
- package/dist/commands/proposal/proposal-cli.js +22 -10
- package/dist/commands/proposal/proposal.js +17 -1
- package/dist/commands/proposal/validators/proposals.js +369 -329
- package/dist/commands/read/curate.js +344 -80
- package/dist/commands/read/search-cli.js +7 -0
- package/dist/commands/read/search.js +1 -0
- package/dist/commands/read/show.js +67 -2
- package/dist/commands/remember.js +6 -2
- package/dist/commands/sources/installed-stashes.js +5 -1
- package/dist/commands/sources/stash-cli.js +10 -2
- package/dist/core/asset/asset-registry.js +2 -0
- package/dist/core/asset/asset-spec.js +14 -0
- package/dist/core/asset/frontmatter.js +166 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/config/config-schema.js +255 -2
- package/dist/core/config/config.js +2 -2
- package/dist/core/logs-db.js +305 -0
- package/dist/core/paths.js +3 -0
- package/dist/core/state-db.js +706 -42
- package/dist/indexer/db/db.js +364 -38
- package/dist/indexer/db/graph-db.js +129 -86
- package/dist/indexer/ensure-index.js +152 -17
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/graph/graph-extraction.js +203 -3
- package/dist/indexer/index-writer-lock.js +99 -0
- package/dist/indexer/indexer.js +114 -111
- package/dist/indexer/passes/memory-inference.js +71 -25
- package/dist/indexer/passes/staleness-detect.js +2 -5
- package/dist/indexer/search/db-search.js +15 -4
- package/dist/indexer/search/ranking-contributors.js +22 -0
- package/dist/indexer/search/ranking.js +4 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/integrations/agent/prompts.js +1 -0
- package/dist/integrations/harnesses/claude/session-log.js +27 -5
- package/dist/integrations/harnesses/opencode/session-log.js +9 -0
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/client.js +38 -4
- package/dist/llm/embedder.js +27 -3
- package/dist/llm/embedders/local.js +66 -2
- package/dist/llm/graph-extract.js +2 -1
- package/dist/llm/memory-infer.js +4 -8
- package/dist/llm/metadata-enhance.js +9 -1
- package/dist/llm/usage-persist.js +77 -0
- package/dist/llm/usage-telemetry.js +103 -0
- package/dist/output/context.js +3 -2
- package/dist/output/html-render.js +73 -0
- package/dist/output/renderers.js +73 -1
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/shapes/helpers.js +17 -1
- package/dist/output/text/helpers.js +78 -1
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +1262 -591
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +485 -270
- package/dist/sources/providers/tar-utils.js +16 -8
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/tasks/runner.js +99 -16
- package/dist/workflows/db.js +5 -2
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/data-and-telemetry.md +1 -0
- package/package.json +9 -6
package/dist/indexer/indexer.js
CHANGED
|
@@ -12,6 +12,7 @@ import { resolveIndexPassLLM } from "../llm/index-passes.js";
|
|
|
12
12
|
import { takeWorkflowDocument } from "../workflows/runtime/document-cache.js";
|
|
13
13
|
import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddableEntryCount, getEmbeddingCount, getEntriesByDir, getEntryCount, getIndexDirState, getMeta, isVecAvailable, openDatabase, openExistingDatabase, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db/db.js";
|
|
14
14
|
import { deleteStoredGraph } from "./db/graph-db.js";
|
|
15
|
+
import { withIndexWriterLease } from "./index-writer-lock.js";
|
|
15
16
|
import { applyCuratedFrontmatter, applyWikiFrontmatter, generateMetadataFlat, isEnrichmentComplete, isWorkflowSkipWarning, loadStashFile, shouldIndexStashFile, } from "./passes/metadata.js";
|
|
16
17
|
import { buildSearchText } from "./search/search-fields.js";
|
|
17
18
|
import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
|
|
@@ -225,119 +226,121 @@ function runCleanPass(db, dryRun) {
|
|
|
225
226
|
}
|
|
226
227
|
// ── Indexer ──────────────────────────────────────────────────────────────────
|
|
227
228
|
export async function akmIndex(options) {
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
229
|
+
return withIndexWriterLease({ purpose: "akm-index", signal: options?.signal }, async () => {
|
|
230
|
+
const stashDir = options?.stashDir || resolveStashDir();
|
|
231
|
+
const onProgress = options?.onProgress ?? (() => { });
|
|
232
|
+
const signal = options?.signal;
|
|
233
|
+
const reEnrich = options?.reEnrich === true;
|
|
234
|
+
const full = options?.full === true;
|
|
235
|
+
const clean = options?.clean === true;
|
|
236
|
+
const dryRun = options?.dryRun === true;
|
|
237
|
+
// Load config and resolve all stash sources
|
|
238
|
+
const { loadConfig } = await import("../core/config/config.js");
|
|
239
|
+
const config = loadConfig();
|
|
240
|
+
// One-time, read-only guard: warn if the writable stash still holds an
|
|
241
|
+
// un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
|
|
242
|
+
// entirely, so an unmigrated vault's `.env` data would silently never be
|
|
243
|
+
// indexed. Non-destructive — only stats, never reads/writes/deletes.
|
|
244
|
+
const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
|
|
245
|
+
warnOnUnmigratedVaults(stashDir);
|
|
246
|
+
// Ensure git stash caches are extracted before resolving stash dirs,
|
|
247
|
+
// so their content directories exist on disk for the walker to discover.
|
|
248
|
+
const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
|
|
249
|
+
await ensureSourceCaches(config, { force: full });
|
|
250
|
+
const allSourceEntries = resolveSourceEntries(stashDir, config);
|
|
251
|
+
const allSourceDirs = allSourceEntries.map((s) => s.path);
|
|
252
|
+
const t0 = Date.now();
|
|
253
|
+
// Open database — pass embedding dimension from config if available
|
|
254
|
+
const dbPath = getDbPath();
|
|
255
|
+
const embeddingDim = config.embedding?.dimension;
|
|
256
|
+
const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
|
|
257
|
+
try {
|
|
258
|
+
// Determine incremental vs full mode
|
|
259
|
+
const prevStashDir = getMeta(db, "stashDir");
|
|
260
|
+
const prevBuiltAt = getMeta(db, "builtAt");
|
|
261
|
+
const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
|
|
262
|
+
const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
|
|
263
|
+
// Assemble the run context
|
|
264
|
+
const ctx = {
|
|
265
|
+
db,
|
|
266
|
+
config,
|
|
267
|
+
sources: allSourceEntries,
|
|
268
|
+
sourceDirs: allSourceDirs,
|
|
269
|
+
full,
|
|
270
|
+
reEnrich,
|
|
271
|
+
stashDir,
|
|
272
|
+
onProgress,
|
|
273
|
+
signal,
|
|
274
|
+
timing: {
|
|
275
|
+
t0,
|
|
276
|
+
tWalkStart: t0,
|
|
277
|
+
tWalkEnd: t0,
|
|
278
|
+
tLlmEnd: t0,
|
|
279
|
+
tFtsEnd: t0,
|
|
280
|
+
tEmbedEnd: t0,
|
|
281
|
+
},
|
|
282
|
+
isIncremental,
|
|
283
|
+
builtAtMs,
|
|
284
|
+
hadRemovedSources: false,
|
|
285
|
+
scannedDirs: 0,
|
|
286
|
+
skippedDirs: 0,
|
|
287
|
+
generatedCount: 0,
|
|
288
|
+
walkWarnings: [],
|
|
289
|
+
dirsNeedingLlm: [],
|
|
290
|
+
embeddingResult: null,
|
|
291
|
+
graphExtractionResult: null,
|
|
292
|
+
};
|
|
293
|
+
onProgress({
|
|
294
|
+
phase: "summary",
|
|
295
|
+
message: buildIndexSummaryMessage({
|
|
296
|
+
mode: isIncremental ? "incremental" : "full",
|
|
297
|
+
sourcesCount: allSourceDirs.length,
|
|
298
|
+
semanticSearchMode: config.semanticSearchMode,
|
|
299
|
+
embeddingProvider: getEmbeddingProvider(config.embedding),
|
|
300
|
+
llmEnabled: !!resolveIndexPassLLM("enrichment", config),
|
|
301
|
+
vecAvailable: isVecAvailable(db),
|
|
302
|
+
}),
|
|
303
|
+
});
|
|
304
|
+
// ── Phase sequence ───────────────────────────────────────────────────────
|
|
305
|
+
await runSourceCachePhase(ctx);
|
|
306
|
+
await runWalkPhase(ctx);
|
|
307
|
+
await runEmbeddingPhase(ctx);
|
|
308
|
+
await runFinalizePhase(ctx);
|
|
309
|
+
// ────────────────────────────────────────────────────────────────────────
|
|
310
|
+
const { _verification: verification, _totalEntries: totalEntries } = ctx;
|
|
311
|
+
const { timing } = ctx;
|
|
312
|
+
// ── Clean pass ───────────────────────────────────────────────────────────
|
|
313
|
+
// After the normal index completes, remove entries whose source files no
|
|
314
|
+
// longer exist on disk. Remote entries (empty file_path) are skipped.
|
|
315
|
+
let cleanResult;
|
|
316
|
+
if (clean) {
|
|
317
|
+
cleanResult = runCleanPass(db, dryRun);
|
|
318
|
+
}
|
|
319
|
+
// ────────────────────────────────────────────────────────────────────────
|
|
320
|
+
return {
|
|
321
|
+
stashDir,
|
|
322
|
+
totalEntries,
|
|
323
|
+
generatedMetadata: ctx.generatedCount,
|
|
324
|
+
indexPath: dbPath,
|
|
294
325
|
mode: isIncremental ? "incremental" : "full",
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
const { _verification: verification, _totalEntries: totalEntries } = ctx;
|
|
309
|
-
const { timing } = ctx;
|
|
310
|
-
// ── Clean pass ───────────────────────────────────────────────────────────
|
|
311
|
-
// After the normal index completes, remove entries whose source files no
|
|
312
|
-
// longer exist on disk. Remote entries (empty file_path) are skipped.
|
|
313
|
-
let cleanResult;
|
|
314
|
-
if (clean) {
|
|
315
|
-
cleanResult = runCleanPass(db, dryRun);
|
|
326
|
+
directoriesScanned: ctx.scannedDirs,
|
|
327
|
+
directoriesSkipped: ctx.skippedDirs,
|
|
328
|
+
...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
|
|
329
|
+
verification,
|
|
330
|
+
timing: {
|
|
331
|
+
totalMs: Date.now() - timing.t0,
|
|
332
|
+
walkMs: timing.tWalkEnd - timing.tWalkStart,
|
|
333
|
+
llmMs: timing.tLlmEnd - timing.tWalkEnd,
|
|
334
|
+
embedMs: timing.tEmbedEnd - timing.tLlmEnd,
|
|
335
|
+
ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
|
|
336
|
+
},
|
|
337
|
+
...(cleanResult !== undefined ? { clean: cleanResult } : {}),
|
|
338
|
+
};
|
|
316
339
|
}
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
generatedMetadata: ctx.generatedCount,
|
|
322
|
-
indexPath: dbPath,
|
|
323
|
-
mode: isIncremental ? "incremental" : "full",
|
|
324
|
-
directoriesScanned: ctx.scannedDirs,
|
|
325
|
-
directoriesSkipped: ctx.skippedDirs,
|
|
326
|
-
...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
|
|
327
|
-
verification,
|
|
328
|
-
timing: {
|
|
329
|
-
totalMs: Date.now() - timing.t0,
|
|
330
|
-
walkMs: timing.tWalkEnd - timing.tWalkStart,
|
|
331
|
-
llmMs: timing.tLlmEnd - timing.tWalkEnd,
|
|
332
|
-
embedMs: timing.tEmbedEnd - timing.tLlmEnd,
|
|
333
|
-
ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
|
|
334
|
-
},
|
|
335
|
-
...(cleanResult !== undefined ? { clean: cleanResult } : {}),
|
|
336
|
-
};
|
|
337
|
-
}
|
|
338
|
-
finally {
|
|
339
|
-
closeDatabase(db);
|
|
340
|
-
}
|
|
340
|
+
finally {
|
|
341
|
+
closeDatabase(db);
|
|
342
|
+
}
|
|
343
|
+
});
|
|
341
344
|
}
|
|
342
345
|
// ── Extracted helpers for indexing ────────────────────────────────────────────
|
|
343
346
|
async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, doFullDelete = false, onProgress) {
|
|
@@ -119,6 +119,26 @@ export async function runMemoryInferencePass(ctx) {
|
|
|
119
119
|
// 2026-05-26).
|
|
120
120
|
if (signal?.aborted)
|
|
121
121
|
return { aborted: true };
|
|
122
|
+
// Pre-check (#588): when `<parent>.derived.md` is already on disk the
|
|
123
|
+
// inference is by definition complete — the parent only looks pending
|
|
124
|
+
// because `markParentProcessed` never ran (process killed between the
|
|
125
|
+
// child write and the mark) or the child was created externally (e.g.
|
|
126
|
+
// consolidation). Skip the LLM/cache call entirely and mark the parent
|
|
127
|
+
// so it never re-pends. Before this check, production measurements
|
|
128
|
+
// showed ~55% of the pass's LLM budget re-deriving such parents only to
|
|
129
|
+
// discover the existing child after the fact.
|
|
130
|
+
if (fs.existsSync(derivedChildPath(record))) {
|
|
131
|
+
markParentProcessed(record);
|
|
132
|
+
return {
|
|
133
|
+
skipped: false,
|
|
134
|
+
splitParent: false,
|
|
135
|
+
written: 0,
|
|
136
|
+
fromCache: false,
|
|
137
|
+
retryAttempts: 0,
|
|
138
|
+
childExists: true,
|
|
139
|
+
precheck: true,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
122
142
|
// Incremental cache: skip LLM call when body hash is unchanged and
|
|
123
143
|
// --re-enrich was not requested. The cache ref is the absolute file path.
|
|
124
144
|
const validate = (raw) => {
|
|
@@ -171,23 +191,30 @@ export async function runMemoryInferencePass(ctx) {
|
|
|
171
191
|
return { skipped: false, splitParent: true, written: writeOutcome.written, fromCache, retryAttempts };
|
|
172
192
|
}
|
|
173
193
|
// LLM produced a valid derived draft but no file was written — either
|
|
174
|
-
// because `<parent>.derived.md`
|
|
175
|
-
//
|
|
176
|
-
//
|
|
177
|
-
// into the freshAttempts
|
|
194
|
+
// because `<parent>.derived.md` appeared on disk after the pre-check
|
|
195
|
+
// above (a rare mid-flight race) or `writeAssetToSource` threw.
|
|
196
|
+
// Categorise as `childExists` so the consumed attempt is accounted for
|
|
197
|
+
// in health metrics rather than vanishing into the freshAttempts
|
|
198
|
+
// denominator.
|
|
178
199
|
//
|
|
179
|
-
// When the child
|
|
180
|
-
//
|
|
181
|
-
//
|
|
182
|
-
// (
|
|
183
|
-
//
|
|
184
|
-
//
|
|
185
|
-
// should be retried next run — so we key off the explicit `childExists`
|
|
186
|
-
// outcome rather than the conflated `written === 0`.
|
|
200
|
+
// When the child exists the inference is, by definition, complete — so
|
|
201
|
+
// mark the parent processed here too (#550), otherwise
|
|
202
|
+
// `isPendingMemory()` re-queues the same parent every run. A genuine
|
|
203
|
+
// write *failure* (`writeAssetToSource` threw) must NOT mark the parent
|
|
204
|
+
// — it should be retried next run — so we key off the explicit
|
|
205
|
+
// `childExists` outcome rather than the conflated `written === 0`.
|
|
187
206
|
if (writeOutcome.childExists) {
|
|
188
207
|
markParentProcessed(record);
|
|
189
208
|
}
|
|
190
|
-
return {
|
|
209
|
+
return {
|
|
210
|
+
skipped: false,
|
|
211
|
+
splitParent: false,
|
|
212
|
+
written: 0,
|
|
213
|
+
fromCache,
|
|
214
|
+
retryAttempts,
|
|
215
|
+
childExists: true,
|
|
216
|
+
precheck: false,
|
|
217
|
+
};
|
|
191
218
|
},
|
|
192
219
|
// Default concurrency of 4 for cloud APIs. Set `llm.concurrency: 1`
|
|
193
220
|
// in config.json for local model servers (LM Studio, Ollama).
|
|
@@ -224,11 +251,16 @@ export async function runMemoryInferencePass(ctx) {
|
|
|
224
251
|
result.writtenFacts += res.written;
|
|
225
252
|
}
|
|
226
253
|
else if ("childExists" in res && res.childExists) {
|
|
227
|
-
//
|
|
228
|
-
//
|
|
229
|
-
//
|
|
254
|
+
// Derived child already on disk. Track separately so this category is
|
|
255
|
+
// observable in health output and stops bleeding into the
|
|
256
|
+
// freshAttempts denominator. Pre-check skips (#588) are the routine
|
|
257
|
+
// self-healing path — no LLM attempt was consumed and the parent has
|
|
258
|
+
// been marked processed — so only the rare post-LLM case (mid-flight
|
|
259
|
+
// race or write failure) warrants a per-ref warning.
|
|
230
260
|
result.skippedChildExists += 1;
|
|
231
|
-
|
|
261
|
+
if (!res.precheck) {
|
|
262
|
+
warn(`memory inference: derived child for ${pending[i]?.ref ?? "<unknown>"} already existed or write failed; counted as skippedChildExists`);
|
|
263
|
+
}
|
|
232
264
|
}
|
|
233
265
|
else {
|
|
234
266
|
// The per-record state machine should cover every outcome. A hit here
|
|
@@ -324,6 +356,14 @@ function toMemoryName(memoriesDir, filePath) {
|
|
|
324
356
|
// user has organised under memories/.
|
|
325
357
|
return rel.replace(/\\/g, "/").replace(/\.md$/i, "");
|
|
326
358
|
}
|
|
359
|
+
/**
|
|
360
|
+
* Absolute path of the derived child for a parent memory. Single source of
|
|
361
|
+
* truth for the `<parent>.derived.md` naming convention — used both by the
|
|
362
|
+
* pre-LLM existence check (#588) and the write path.
|
|
363
|
+
*/
|
|
364
|
+
function derivedChildPath(parent) {
|
|
365
|
+
return path.join(parent.stashRoot, "memories", `${parent.name}.derived.md`);
|
|
366
|
+
}
|
|
327
367
|
async function writeDerivedMemory(parent, derived) {
|
|
328
368
|
const writeTarget = {
|
|
329
369
|
kind: "filesystem",
|
|
@@ -338,11 +378,10 @@ async function writeDerivedMemory(parent, derived) {
|
|
|
338
378
|
};
|
|
339
379
|
const childName = `${parent.name}.derived`;
|
|
340
380
|
const childRefStr = `memory:${childName}`;
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
//
|
|
344
|
-
//
|
|
345
|
-
// (#550) instead of re-queueing it forever.
|
|
381
|
+
if (fs.existsSync(derivedChildPath(parent))) {
|
|
382
|
+
// The derived child appeared on disk after the caller's pre-check (#588)
|
|
383
|
+
// — a rare mid-flight race. Report `childExists` so the caller marks the
|
|
384
|
+
// parent processed (#550) instead of re-queueing it forever.
|
|
346
385
|
return { written: 0, childExists: true };
|
|
347
386
|
}
|
|
348
387
|
try {
|
|
@@ -386,10 +425,17 @@ function markParentProcessed(parent) {
|
|
|
386
425
|
warn(`memory inference: failed to re-read parent ${parent.filePath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
387
426
|
return;
|
|
388
427
|
}
|
|
389
|
-
const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
|
|
390
428
|
const block = parseFrontmatterBlock(raw);
|
|
391
|
-
|
|
392
|
-
|
|
429
|
+
if (!block) {
|
|
430
|
+
// Cannot safely rewrite malformed frontmatter — skip marking so the memory
|
|
431
|
+
// is retried on the next run once the frontmatter is repaired. Writing with
|
|
432
|
+
// `body = raw` would wrap the entire file (including the bad frontmatter)
|
|
433
|
+
// in a new block, producing a duplicate-frontmatter corruption.
|
|
434
|
+
warn(`memory inference: skipping markParentProcessed for ${parent.filePath} — could not parse frontmatter block`);
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
|
|
438
|
+
const next = assembleAsset(updatedFm, block.content);
|
|
393
439
|
try {
|
|
394
440
|
fs.writeFileSync(parent.filePath, next, "utf8");
|
|
395
441
|
}
|
|
@@ -44,6 +44,7 @@
|
|
|
44
44
|
import { createHash } from "node:crypto";
|
|
45
45
|
import fs from "node:fs";
|
|
46
46
|
import path from "node:path";
|
|
47
|
+
import stalenessDetectSystemPrompt from "../../assets/prompts/staleness-detect-system.md" with { type: "text" };
|
|
47
48
|
import { assembleAsset } from "../../core/asset/asset-serialize.js";
|
|
48
49
|
import { parseFrontmatter, parseFrontmatterBlock } from "../../core/asset/frontmatter.js";
|
|
49
50
|
import { concurrentMap } from "../../core/concurrent.js";
|
|
@@ -319,11 +320,7 @@ function pickSimilar(candidate, all) {
|
|
|
319
320
|
return scored.slice(0, TOP_K_SIMILAR).map((s) => s.snap);
|
|
320
321
|
}
|
|
321
322
|
// ── LLM dispatch ────────────────────────────────────────────────────────────
|
|
322
|
-
const SYSTEM_PROMPT =
|
|
323
|
-
"Respond on the first line with exactly YES or NO.\n" +
|
|
324
|
-
"If YES, the second line MUST be of the form `SUPERSEDED_BY: <ref>` where <ref> is the exact ref of the superseding memory from the list provided. Do NOT invent refs.\n" +
|
|
325
|
-
"If NO, do not include any additional lines.\n" +
|
|
326
|
-
"No prose, no preamble, no markdown.";
|
|
323
|
+
const SYSTEM_PROMPT = stalenessDetectSystemPrompt;
|
|
327
324
|
async function askValidator(connection, candidate, allMemories, signal, timeoutMs) {
|
|
328
325
|
const similar = pickSimilar(candidate, allMemories);
|
|
329
326
|
if (similar.length === 0) {
|
|
@@ -65,6 +65,7 @@ export async function searchLocal(input) {
|
|
|
65
65
|
const includeProposed = input.includeProposed === true;
|
|
66
66
|
const beliefFilter = input.beliefFilter ?? "all";
|
|
67
67
|
const restrictToSources = input.restrictToSources === true;
|
|
68
|
+
const includeExcludedTypes = input.includeExcludedTypes === true;
|
|
68
69
|
const rendererRegistry = input.rendererRegistry ?? defaultRendererRegistry;
|
|
69
70
|
const allSourceDirs = sources.map((s) => s.path);
|
|
70
71
|
const rawStatus = readSemanticStatus();
|
|
@@ -114,7 +115,7 @@ export async function searchLocal(input) {
|
|
|
114
115
|
mode: "keyword",
|
|
115
116
|
};
|
|
116
117
|
}
|
|
117
|
-
const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources);
|
|
118
|
+
const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources, includeExcludedTypes);
|
|
118
119
|
return {
|
|
119
120
|
hits,
|
|
120
121
|
tip: hits.length === 0
|
|
@@ -131,14 +132,19 @@ export async function searchLocal(input) {
|
|
|
131
132
|
}
|
|
132
133
|
}
|
|
133
134
|
// ── Database search ─────────────────────────────────────────────────────────
|
|
134
|
-
async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false) {
|
|
135
|
+
async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false, includeExcludedTypes = false) {
|
|
135
136
|
const hasSearchableTokens = query.length > 0 && sanitizeFtsQuery(query).length > 0;
|
|
137
|
+
// #627 — resolve the default type-exclusion policy. It applies ONLY on the
|
|
138
|
+
// untyped ('any') path and only when the caller did not opt back in via
|
|
139
|
+
// `includeExcludedTypes`. When the config key is ABSENT a built-in default of
|
|
140
|
+
// ['session'] is applied; an explicit empty list disables exclusion.
|
|
141
|
+
const defaultExcludes = searchType === "any" && !includeExcludedTypes ? (config.search?.defaultExcludeTypes ?? ["session"]) : [];
|
|
136
142
|
// Empty queries — including ones that sanitize down to no searchable FTS
|
|
137
143
|
// tokens such as "." — should enumerate matching entries instead of
|
|
138
144
|
// returning an empty result set from FTS.
|
|
139
145
|
if (!hasSearchableTokens) {
|
|
140
146
|
const typeFilter = searchType === "any" ? undefined : searchType;
|
|
141
|
-
const allEntries = getAllEntries(db, typeFilter);
|
|
147
|
+
const allEntries = getAllEntries(db, typeFilter, defaultExcludes);
|
|
142
148
|
// Deduplicate by file path — multiple entries can share the same file
|
|
143
149
|
const seenFilePaths = new Set();
|
|
144
150
|
const uniqueEntries = allEntries.filter((ie) => {
|
|
@@ -187,7 +193,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
|
|
|
187
193
|
const typeFilter = searchType === "any" ? undefined : searchType;
|
|
188
194
|
const tEmbed0 = Date.now();
|
|
189
195
|
const embeddingPromise = tryVecScores(db, query, limit * 3, config);
|
|
190
|
-
const ftsResults = searchFts(db, query, limit * 3, typeFilter);
|
|
196
|
+
const ftsResults = searchFts(db, query, limit * 3, typeFilter, defaultExcludes);
|
|
191
197
|
const embeddingScores = await embeddingPromise;
|
|
192
198
|
const embedMs = Date.now() - tEmbed0;
|
|
193
199
|
const tRank0 = Date.now();
|
|
@@ -208,6 +214,11 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
|
|
|
208
214
|
embedScoreMap,
|
|
209
215
|
getEntryById: (id) => getEntryById(db, id) ?? undefined,
|
|
210
216
|
typeFilter,
|
|
217
|
+
// #627 — also exclude default-hidden types from the vector-only branch so a
|
|
218
|
+
// session asset that is a top-k vector neighbor (but not an FTS match) does
|
|
219
|
+
// not leak into default ('any') results. defaultExcludes is already []
|
|
220
|
+
// unless this is the untyped path without includeExcludedTypes.
|
|
221
|
+
excludeTypes: defaultExcludes,
|
|
211
222
|
});
|
|
212
223
|
// ── Scoring Phase ──────────────────────────────────────────────────────
|
|
213
224
|
// Apply boosts as multiplicative factors (all boosts in a single phase
|
|
@@ -9,6 +9,9 @@ const TYPE_BOOST = {
|
|
|
9
9
|
agent: 0.3,
|
|
10
10
|
script: 0.2,
|
|
11
11
|
knowledge: 0.22,
|
|
12
|
+
// Facts are authoritative, durable declarations about the stash — rank them
|
|
13
|
+
// alongside knowledge so they surface reliably when relevant.
|
|
14
|
+
fact: 0.22,
|
|
12
15
|
memory: -0.02,
|
|
13
16
|
};
|
|
14
17
|
const MAX_BOOST_SUM = 3.0;
|
|
@@ -206,6 +209,24 @@ const lessonStrengthContributor = {
|
|
|
206
209
|
return Math.min(0.3, 0.06 * strength);
|
|
207
210
|
},
|
|
208
211
|
};
|
|
212
|
+
/**
|
|
213
|
+
* Pinned-fact boost.
|
|
214
|
+
*
|
|
215
|
+
* Facts marked `pinned: true` form the small always-injected "core context"
|
|
216
|
+
* (see docs/design/fact-asset-type.md). The fact metadata contributor records
|
|
217
|
+
* a `pinned` search hint; here we give those facts a modest additive boost so
|
|
218
|
+
* the core outranks ordinary facts on otherwise-equal queries. Capped small so
|
|
219
|
+
* it cannot overpower an exact-name match.
|
|
220
|
+
*/
|
|
221
|
+
const pinnedFactRankingContributor = {
|
|
222
|
+
name: "pinned-fact-ranking",
|
|
223
|
+
appliesTo(item) {
|
|
224
|
+
return item.entry.type === "fact" && (item.entry.searchHints?.includes("pinned") ?? false);
|
|
225
|
+
},
|
|
226
|
+
adjust() {
|
|
227
|
+
return 0.15;
|
|
228
|
+
},
|
|
229
|
+
};
|
|
209
230
|
/**
|
|
210
231
|
* Blend ratio for scoped vs. global utility signals.
|
|
211
232
|
*
|
|
@@ -310,6 +331,7 @@ export const defaultRankingContributors = [
|
|
|
310
331
|
graphRankingContributor,
|
|
311
332
|
captureModeRankingContributor,
|
|
312
333
|
lessonStrengthContributor,
|
|
334
|
+
pinnedFactRankingContributor,
|
|
313
335
|
projectContextRankingContributor,
|
|
314
336
|
];
|
|
315
337
|
export const defaultUtilityRankingContributors = [utilityRankingContributor];
|
|
@@ -20,6 +20,7 @@ export function normalizeFtsScores(results) {
|
|
|
20
20
|
export function combineSearchScores(options) {
|
|
21
21
|
const FTS_WEIGHT = 0.7;
|
|
22
22
|
const VEC_WEIGHT = 0.3;
|
|
23
|
+
const excludeTypeSet = options.excludeTypes && options.excludeTypes.length > 0 ? new Set(options.excludeTypes) : null;
|
|
23
24
|
const scored = [];
|
|
24
25
|
const seenIds = new Set();
|
|
25
26
|
for (const [id, { score: ftsScore, result }] of options.ftsScoreMap) {
|
|
@@ -42,6 +43,9 @@ export function combineSearchScores(options) {
|
|
|
42
43
|
continue;
|
|
43
44
|
if (options.typeFilter && found.entry.type !== options.typeFilter)
|
|
44
45
|
continue;
|
|
46
|
+
// #627 — drop vector-only neighbors whose type is excluded on the default path.
|
|
47
|
+
if (excludeTypeSet?.has(found.entry.type))
|
|
48
|
+
continue;
|
|
45
49
|
scored.push({
|
|
46
50
|
id,
|
|
47
51
|
entry: found.entry,
|
|
@@ -78,6 +78,15 @@ const DIR_TYPE_MAP = [
|
|
|
78
78
|
type: "session",
|
|
79
79
|
test: (ext) => ext === ".md",
|
|
80
80
|
},
|
|
81
|
+
{
|
|
82
|
+
// Durable stash-level facts live under `facts/<category>/<name>.md`.
|
|
83
|
+
// classifyByDirectory walks every ancestor dir, so nested category
|
|
84
|
+
// subdirs still match. Without this entry a fact file would fall through
|
|
85
|
+
// to classifyBySmartMd and be mistyped as `knowledge`.
|
|
86
|
+
dir: "facts",
|
|
87
|
+
type: "fact",
|
|
88
|
+
test: (ext) => ext === ".md",
|
|
89
|
+
},
|
|
81
90
|
];
|
|
82
91
|
const COMMAND_PLACEHOLDER_RE = /\$ARGUMENTS|\$[123]\b/;
|
|
83
92
|
// Files that should never be treated as the typed asset for the surrounding
|
|
@@ -43,6 +43,7 @@ const TYPE_HINTS = {
|
|
|
43
43
|
script: "script assets are executable text files. Include a shebang and minimal usage comment.",
|
|
44
44
|
env: "env assets are `.env` files holding a group of related CONFIGURATION for an app/service (KEY=VALUE pairs, `#` comments) — URLs, flags, and any credentials it needs. Values may or may not be sensitive; all are protected (key names discoverable, values stay on disk). Inject with `akm env run env:<name> -- <cmd>` (the safe path — values never reach stdout/your context); do NOT run `akm env export` and read its output, as that prints values. For a single sensitive value used on its own for authentication (token, key, cert) use a `secret` instead. Never echo values back to the user.",
|
|
45
45
|
wiki: "wiki assets are markdown reference pages with `# Title` and structured headings.",
|
|
46
|
+
fact: "fact assets are durable stash-level facts (personal/team/project details, coding conventions, stash-meta). Frontmatter SHOULD include `description` and a `category` (personal|team|project|convention|meta); set `pinned: true` only for the small always-injected core. Keep each fact short, high-signal, and self-contained — it is durable context, not an episodic note.",
|
|
46
47
|
};
|
|
47
48
|
function hintForType(type) {
|
|
48
49
|
return TYPE_HINTS[type] ?? `assets of type "${type}" — produce sensible markdown with optional frontmatter.`;
|
|
@@ -5,7 +5,19 @@ import fs from "node:fs";
|
|
|
5
5
|
import os from "node:os";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import { extractInlineRefMentions } from "../../session-logs/inline-refs.js";
|
|
8
|
-
|
|
8
|
+
/**
|
|
9
|
+
* Root directory holding Claude Code's per-project JSONL session logs.
|
|
10
|
+
*
|
|
11
|
+
* Resolved per call (not memoized at module load) so the `AKM_CLAUDE_PROJECTS_DIR`
|
|
12
|
+
* override can be set after import. The override exists so tests — and the
|
|
13
|
+
* isolated-storage sandbox — can point the scan at an empty fixture directory
|
|
14
|
+
* instead of the real `~/.claude/projects`, which on an actively-used machine
|
|
15
|
+
* holds many large session files and would make `akm health` (which scans it
|
|
16
|
+
* synchronously) slow and non-hermetic.
|
|
17
|
+
*/
|
|
18
|
+
function claudeProjectsDir() {
|
|
19
|
+
return process.env.AKM_CLAUDE_PROJECTS_DIR ?? path.join(os.homedir(), ".claude", "projects");
|
|
20
|
+
}
|
|
9
21
|
/**
|
|
10
22
|
* Parse a single Claude Code JSONL event into a normalized {@link SessionEvent}.
|
|
11
23
|
* Returns `undefined` for events that don't carry textual content (file
|
|
@@ -93,11 +105,21 @@ export class ClaudeCodeProvider {
|
|
|
93
105
|
// HARNESS_BY_ID.get("claude").runtimeId.
|
|
94
106
|
name = "claude-code";
|
|
95
107
|
isAvailable() {
|
|
96
|
-
return fs.existsSync(
|
|
108
|
+
return fs.existsSync(claudeProjectsDir());
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Directory holding Claude Code's per-project session JSONL files
|
|
112
|
+
* (`~/.claude/projects`, honoring `AKM_CLAUDE_PROJECTS_DIR`). Returns `[]`
|
|
113
|
+
* when the directory does not exist on this machine. See {@link
|
|
114
|
+
* SessionLogHarness.watchRoots}.
|
|
115
|
+
*/
|
|
116
|
+
watchRoots() {
|
|
117
|
+
const dir = claudeProjectsDir();
|
|
118
|
+
return fs.existsSync(dir) ? [dir] : [];
|
|
97
119
|
}
|
|
98
120
|
*readEvents(input) {
|
|
99
121
|
try {
|
|
100
|
-
for (const jsonlPath of this.#walkJsonl(
|
|
122
|
+
for (const jsonlPath of this.#walkJsonl(claudeProjectsDir())) {
|
|
101
123
|
const stat = fs.statSync(jsonlPath);
|
|
102
124
|
if (stat.mtimeMs < input.sinceMs)
|
|
103
125
|
continue;
|
|
@@ -128,7 +150,7 @@ export class ClaudeCodeProvider {
|
|
|
128
150
|
}
|
|
129
151
|
}
|
|
130
152
|
listSessions(input = {}) {
|
|
131
|
-
const root = input.location ??
|
|
153
|
+
const root = input.location ?? claudeProjectsDir();
|
|
132
154
|
const sinceMs = input.sinceMs ?? 0;
|
|
133
155
|
const summaries = [];
|
|
134
156
|
try {
|
|
@@ -286,7 +308,7 @@ export class ClaudeCodeProvider {
|
|
|
286
308
|
const full = path.join(dir, entry.name);
|
|
287
309
|
if (entry.isDirectory())
|
|
288
310
|
yield* this.#walkJsonl(full);
|
|
289
|
-
else if (entry.name.endsWith(".jsonl"))
|
|
311
|
+
else if (entry.name.endsWith(".jsonl") && entry.name !== "journal.jsonl")
|
|
290
312
|
yield full;
|
|
291
313
|
}
|
|
292
314
|
}
|