akm-cli 0.9.0-beta.3 → 0.9.0-beta.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +600 -0
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +5 -1
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/templates/html/health.html +281 -111
- package/dist/cli.js +14 -3
- package/dist/commands/agent/contribute-cli.js +16 -3
- package/dist/commands/feedback-cli.js +15 -6
- package/dist/commands/graph/graph.js +75 -71
- package/dist/commands/health/checks.js +48 -0
- package/dist/commands/health/html-report.js +422 -80
- package/dist/commands/health.js +381 -9
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/consolidate.js +634 -111
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill.js +145 -69
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +115 -1
- package/dist/commands/improve/extract-prompt.js +33 -2
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +244 -35
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/homeostatic.js +467 -0
- package/dist/commands/improve/improve-auto-accept.js +113 -6
- package/dist/commands/improve/improve-profiles.js +12 -0
- package/dist/commands/improve/improve.js +1974 -614
- package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
- package/dist/commands/improve/outcome-loop.js +256 -0
- package/dist/commands/improve/proactive-maintenance.js +87 -0
- package/dist/commands/improve/procedural.js +409 -0
- package/dist/commands/improve/recombine.js +528 -0
- package/dist/commands/improve/reflect.js +26 -1
- package/dist/commands/improve/related-sessions.js +120 -0
- package/dist/commands/improve/salience.js +386 -0
- package/dist/commands/improve/triage.js +95 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +34 -1
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +31 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +7 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/proposal/proposal.js +5 -0
- package/dist/commands/proposal/validators/proposals.js +71 -54
- package/dist/commands/read/curate.js +344 -80
- package/dist/commands/read/search-cli.js +7 -0
- package/dist/commands/read/search.js +1 -0
- package/dist/commands/read/show.js +67 -2
- package/dist/commands/sources/installed-stashes.js +5 -1
- package/dist/commands/sources/stash-cli.js +10 -2
- package/dist/core/asset/asset-registry.js +2 -0
- package/dist/core/asset/asset-spec.js +14 -0
- package/dist/core/asset/frontmatter.js +166 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/config/config-schema.js +259 -2
- package/dist/core/config/config.js +2 -2
- package/dist/core/logs-db.js +4 -3
- package/dist/core/paths.js +3 -0
- package/dist/core/state-db.js +649 -30
- package/dist/indexer/db/db.js +364 -38
- package/dist/indexer/db/graph-db.js +129 -86
- package/dist/indexer/ensure-index.js +152 -17
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/graph/graph-extraction.js +203 -3
- package/dist/indexer/index-writer-lock.js +99 -0
- package/dist/indexer/indexer.js +114 -111
- package/dist/indexer/passes/memory-inference.js +10 -3
- package/dist/indexer/passes/staleness-detect.js +2 -5
- package/dist/indexer/search/db-search.js +15 -4
- package/dist/indexer/search/ranking-contributors.js +22 -0
- package/dist/indexer/search/ranking.js +4 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/integrations/agent/prompts.js +1 -0
- package/dist/integrations/harnesses/claude/session-log.js +11 -1
- package/dist/integrations/harnesses/opencode/session-log.js +9 -0
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/client.js +23 -4
- package/dist/llm/embedder.js +27 -3
- package/dist/llm/embedders/local.js +66 -2
- package/dist/llm/graph-extract.js +2 -1
- package/dist/llm/memory-infer.js +4 -8
- package/dist/llm/metadata-enhance.js +9 -1
- package/dist/output/renderers.js +73 -1
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/text/helpers.js +9 -0
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +1242 -594
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +473 -270
- package/dist/sources/providers/tar-utils.js +16 -8
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/workflows/db.js +3 -4
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/data-and-telemetry.md +1 -0
- package/package.json +9 -6
package/dist/indexer/indexer.js
CHANGED
|
@@ -12,6 +12,7 @@ import { resolveIndexPassLLM } from "../llm/index-passes.js";
|
|
|
12
12
|
import { takeWorkflowDocument } from "../workflows/runtime/document-cache.js";
|
|
13
13
|
import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddableEntryCount, getEmbeddingCount, getEntriesByDir, getEntryCount, getIndexDirState, getMeta, isVecAvailable, openDatabase, openExistingDatabase, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db/db.js";
|
|
14
14
|
import { deleteStoredGraph } from "./db/graph-db.js";
|
|
15
|
+
import { withIndexWriterLease } from "./index-writer-lock.js";
|
|
15
16
|
import { applyCuratedFrontmatter, applyWikiFrontmatter, generateMetadataFlat, isEnrichmentComplete, isWorkflowSkipWarning, loadStashFile, shouldIndexStashFile, } from "./passes/metadata.js";
|
|
16
17
|
import { buildSearchText } from "./search/search-fields.js";
|
|
17
18
|
import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./search/semantic-status.js";
|
|
@@ -225,119 +226,121 @@ function runCleanPass(db, dryRun) {
|
|
|
225
226
|
}
|
|
226
227
|
// ── Indexer ──────────────────────────────────────────────────────────────────
|
|
227
228
|
export async function akmIndex(options) {
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
229
|
+
return withIndexWriterLease({ purpose: "akm-index", signal: options?.signal }, async () => {
|
|
230
|
+
const stashDir = options?.stashDir || resolveStashDir();
|
|
231
|
+
const onProgress = options?.onProgress ?? (() => { });
|
|
232
|
+
const signal = options?.signal;
|
|
233
|
+
const reEnrich = options?.reEnrich === true;
|
|
234
|
+
const full = options?.full === true;
|
|
235
|
+
const clean = options?.clean === true;
|
|
236
|
+
const dryRun = options?.dryRun === true;
|
|
237
|
+
// Load config and resolve all stash sources
|
|
238
|
+
const { loadConfig } = await import("../core/config/config.js");
|
|
239
|
+
const config = loadConfig();
|
|
240
|
+
// One-time, read-only guard: warn if the writable stash still holds an
|
|
241
|
+
// un-migrated `vaults/` directory. In 0.9.0 the indexer skips `vaults/`
|
|
242
|
+
// entirely, so an unmigrated vault's `.env` data would silently never be
|
|
243
|
+
// indexed. Non-destructive — only stats, never reads/writes/deletes.
|
|
244
|
+
const { warnOnUnmigratedVaults } = await import("./usage/unmigrated-vaults-guard.js");
|
|
245
|
+
warnOnUnmigratedVaults(stashDir);
|
|
246
|
+
// Ensure git stash caches are extracted before resolving stash dirs,
|
|
247
|
+
// so their content directories exist on disk for the walker to discover.
|
|
248
|
+
const { ensureSourceCaches, resolveSourceEntries } = await import("./search/search-source.js");
|
|
249
|
+
await ensureSourceCaches(config, { force: full });
|
|
250
|
+
const allSourceEntries = resolveSourceEntries(stashDir, config);
|
|
251
|
+
const allSourceDirs = allSourceEntries.map((s) => s.path);
|
|
252
|
+
const t0 = Date.now();
|
|
253
|
+
// Open database — pass embedding dimension from config if available
|
|
254
|
+
const dbPath = getDbPath();
|
|
255
|
+
const embeddingDim = config.embedding?.dimension;
|
|
256
|
+
const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
|
|
257
|
+
try {
|
|
258
|
+
// Determine incremental vs full mode
|
|
259
|
+
const prevStashDir = getMeta(db, "stashDir");
|
|
260
|
+
const prevBuiltAt = getMeta(db, "builtAt");
|
|
261
|
+
const isIncremental = !full && prevStashDir === stashDir && !!prevBuiltAt;
|
|
262
|
+
const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
|
|
263
|
+
// Assemble the run context
|
|
264
|
+
const ctx = {
|
|
265
|
+
db,
|
|
266
|
+
config,
|
|
267
|
+
sources: allSourceEntries,
|
|
268
|
+
sourceDirs: allSourceDirs,
|
|
269
|
+
full,
|
|
270
|
+
reEnrich,
|
|
271
|
+
stashDir,
|
|
272
|
+
onProgress,
|
|
273
|
+
signal,
|
|
274
|
+
timing: {
|
|
275
|
+
t0,
|
|
276
|
+
tWalkStart: t0,
|
|
277
|
+
tWalkEnd: t0,
|
|
278
|
+
tLlmEnd: t0,
|
|
279
|
+
tFtsEnd: t0,
|
|
280
|
+
tEmbedEnd: t0,
|
|
281
|
+
},
|
|
282
|
+
isIncremental,
|
|
283
|
+
builtAtMs,
|
|
284
|
+
hadRemovedSources: false,
|
|
285
|
+
scannedDirs: 0,
|
|
286
|
+
skippedDirs: 0,
|
|
287
|
+
generatedCount: 0,
|
|
288
|
+
walkWarnings: [],
|
|
289
|
+
dirsNeedingLlm: [],
|
|
290
|
+
embeddingResult: null,
|
|
291
|
+
graphExtractionResult: null,
|
|
292
|
+
};
|
|
293
|
+
onProgress({
|
|
294
|
+
phase: "summary",
|
|
295
|
+
message: buildIndexSummaryMessage({
|
|
296
|
+
mode: isIncremental ? "incremental" : "full",
|
|
297
|
+
sourcesCount: allSourceDirs.length,
|
|
298
|
+
semanticSearchMode: config.semanticSearchMode,
|
|
299
|
+
embeddingProvider: getEmbeddingProvider(config.embedding),
|
|
300
|
+
llmEnabled: !!resolveIndexPassLLM("enrichment", config),
|
|
301
|
+
vecAvailable: isVecAvailable(db),
|
|
302
|
+
}),
|
|
303
|
+
});
|
|
304
|
+
// ── Phase sequence ───────────────────────────────────────────────────────
|
|
305
|
+
await runSourceCachePhase(ctx);
|
|
306
|
+
await runWalkPhase(ctx);
|
|
307
|
+
await runEmbeddingPhase(ctx);
|
|
308
|
+
await runFinalizePhase(ctx);
|
|
309
|
+
// ────────────────────────────────────────────────────────────────────────
|
|
310
|
+
const { _verification: verification, _totalEntries: totalEntries } = ctx;
|
|
311
|
+
const { timing } = ctx;
|
|
312
|
+
// ── Clean pass ───────────────────────────────────────────────────────────
|
|
313
|
+
// After the normal index completes, remove entries whose source files no
|
|
314
|
+
// longer exist on disk. Remote entries (empty file_path) are skipped.
|
|
315
|
+
let cleanResult;
|
|
316
|
+
if (clean) {
|
|
317
|
+
cleanResult = runCleanPass(db, dryRun);
|
|
318
|
+
}
|
|
319
|
+
// ────────────────────────────────────────────────────────────────────────
|
|
320
|
+
return {
|
|
321
|
+
stashDir,
|
|
322
|
+
totalEntries,
|
|
323
|
+
generatedMetadata: ctx.generatedCount,
|
|
324
|
+
indexPath: dbPath,
|
|
294
325
|
mode: isIncremental ? "incremental" : "full",
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
const { _verification: verification, _totalEntries: totalEntries } = ctx;
|
|
309
|
-
const { timing } = ctx;
|
|
310
|
-
// ── Clean pass ───────────────────────────────────────────────────────────
|
|
311
|
-
// After the normal index completes, remove entries whose source files no
|
|
312
|
-
// longer exist on disk. Remote entries (empty file_path) are skipped.
|
|
313
|
-
let cleanResult;
|
|
314
|
-
if (clean) {
|
|
315
|
-
cleanResult = runCleanPass(db, dryRun);
|
|
326
|
+
directoriesScanned: ctx.scannedDirs,
|
|
327
|
+
directoriesSkipped: ctx.skippedDirs,
|
|
328
|
+
...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
|
|
329
|
+
verification,
|
|
330
|
+
timing: {
|
|
331
|
+
totalMs: Date.now() - timing.t0,
|
|
332
|
+
walkMs: timing.tWalkEnd - timing.tWalkStart,
|
|
333
|
+
llmMs: timing.tLlmEnd - timing.tWalkEnd,
|
|
334
|
+
embedMs: timing.tEmbedEnd - timing.tLlmEnd,
|
|
335
|
+
ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
|
|
336
|
+
},
|
|
337
|
+
...(cleanResult !== undefined ? { clean: cleanResult } : {}),
|
|
338
|
+
};
|
|
316
339
|
}
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
generatedMetadata: ctx.generatedCount,
|
|
322
|
-
indexPath: dbPath,
|
|
323
|
-
mode: isIncremental ? "incremental" : "full",
|
|
324
|
-
directoriesScanned: ctx.scannedDirs,
|
|
325
|
-
directoriesSkipped: ctx.skippedDirs,
|
|
326
|
-
...(ctx.walkWarnings.length > 0 ? { warnings: ctx.walkWarnings } : {}),
|
|
327
|
-
verification,
|
|
328
|
-
timing: {
|
|
329
|
-
totalMs: Date.now() - timing.t0,
|
|
330
|
-
walkMs: timing.tWalkEnd - timing.tWalkStart,
|
|
331
|
-
llmMs: timing.tLlmEnd - timing.tWalkEnd,
|
|
332
|
-
embedMs: timing.tEmbedEnd - timing.tLlmEnd,
|
|
333
|
-
ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
|
|
334
|
-
},
|
|
335
|
-
...(cleanResult !== undefined ? { clean: cleanResult } : {}),
|
|
336
|
-
};
|
|
337
|
-
}
|
|
338
|
-
finally {
|
|
339
|
-
closeDatabase(db);
|
|
340
|
-
}
|
|
340
|
+
finally {
|
|
341
|
+
closeDatabase(db);
|
|
342
|
+
}
|
|
343
|
+
});
|
|
341
344
|
}
|
|
342
345
|
// ── Extracted helpers for indexing ────────────────────────────────────────────
|
|
343
346
|
async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadRemovedSources, doFullDelete = false, onProgress) {
|
|
@@ -425,10 +425,17 @@ function markParentProcessed(parent) {
|
|
|
425
425
|
warn(`memory inference: failed to re-read parent ${parent.filePath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
426
426
|
return;
|
|
427
427
|
}
|
|
428
|
-
const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
|
|
429
428
|
const block = parseFrontmatterBlock(raw);
|
|
430
|
-
|
|
431
|
-
|
|
429
|
+
if (!block) {
|
|
430
|
+
// Cannot safely rewrite malformed frontmatter — skip marking so the memory
|
|
431
|
+
// is retried on the next run once the frontmatter is repaired. Writing with
|
|
432
|
+
// `body = raw` would wrap the entire file (including the bad frontmatter)
|
|
433
|
+
// in a new block, producing a duplicate-frontmatter corruption.
|
|
434
|
+
warn(`memory inference: skipping markParentProcessed for ${parent.filePath} — could not parse frontmatter block`);
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
|
|
438
|
+
const next = assembleAsset(updatedFm, block.content);
|
|
432
439
|
try {
|
|
433
440
|
fs.writeFileSync(parent.filePath, next, "utf8");
|
|
434
441
|
}
|
|
@@ -44,6 +44,7 @@
|
|
|
44
44
|
import { createHash } from "node:crypto";
|
|
45
45
|
import fs from "node:fs";
|
|
46
46
|
import path from "node:path";
|
|
47
|
+
import stalenessDetectSystemPrompt from "../../assets/prompts/staleness-detect-system.md" with { type: "text" };
|
|
47
48
|
import { assembleAsset } from "../../core/asset/asset-serialize.js";
|
|
48
49
|
import { parseFrontmatter, parseFrontmatterBlock } from "../../core/asset/frontmatter.js";
|
|
49
50
|
import { concurrentMap } from "../../core/concurrent.js";
|
|
@@ -319,11 +320,7 @@ function pickSimilar(candidate, all) {
|
|
|
319
320
|
return scored.slice(0, TOP_K_SIMILAR).map((s) => s.snap);
|
|
320
321
|
}
|
|
321
322
|
// ── LLM dispatch ────────────────────────────────────────────────────────────
|
|
322
|
-
const SYSTEM_PROMPT =
|
|
323
|
-
"Respond on the first line with exactly YES or NO.\n" +
|
|
324
|
-
"If YES, the second line MUST be of the form `SUPERSEDED_BY: <ref>` where <ref> is the exact ref of the superseding memory from the list provided. Do NOT invent refs.\n" +
|
|
325
|
-
"If NO, do not include any additional lines.\n" +
|
|
326
|
-
"No prose, no preamble, no markdown.";
|
|
323
|
+
const SYSTEM_PROMPT = stalenessDetectSystemPrompt;
|
|
327
324
|
async function askValidator(connection, candidate, allMemories, signal, timeoutMs) {
|
|
328
325
|
const similar = pickSimilar(candidate, allMemories);
|
|
329
326
|
if (similar.length === 0) {
|
|
@@ -65,6 +65,7 @@ export async function searchLocal(input) {
|
|
|
65
65
|
const includeProposed = input.includeProposed === true;
|
|
66
66
|
const beliefFilter = input.beliefFilter ?? "all";
|
|
67
67
|
const restrictToSources = input.restrictToSources === true;
|
|
68
|
+
const includeExcludedTypes = input.includeExcludedTypes === true;
|
|
68
69
|
const rendererRegistry = input.rendererRegistry ?? defaultRendererRegistry;
|
|
69
70
|
const allSourceDirs = sources.map((s) => s.path);
|
|
70
71
|
const rawStatus = readSemanticStatus();
|
|
@@ -114,7 +115,7 @@ export async function searchLocal(input) {
|
|
|
114
115
|
mode: "keyword",
|
|
115
116
|
};
|
|
116
117
|
}
|
|
117
|
-
const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources);
|
|
118
|
+
const { hits, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry, filters, includeProposed, beliefFilter, restrictToSources, includeExcludedTypes);
|
|
118
119
|
return {
|
|
119
120
|
hits,
|
|
120
121
|
tip: hits.length === 0
|
|
@@ -131,14 +132,19 @@ export async function searchLocal(input) {
|
|
|
131
132
|
}
|
|
132
133
|
}
|
|
133
134
|
// ── Database search ─────────────────────────────────────────────────────────
|
|
134
|
-
async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false) {
|
|
135
|
+
async function searchDatabase(db, query, searchType, limit, stashDir, allSourceDirs, config, sources, rendererRegistry = defaultRendererRegistry, filters, includeProposed = false, beliefFilter = "all", restrictToSources = false, includeExcludedTypes = false) {
|
|
135
136
|
const hasSearchableTokens = query.length > 0 && sanitizeFtsQuery(query).length > 0;
|
|
137
|
+
// #627 — resolve the default type-exclusion policy. It applies ONLY on the
|
|
138
|
+
// untyped ('any') path and only when the caller did not opt back in via
|
|
139
|
+
// `includeExcludedTypes`. When the config key is ABSENT a built-in default of
|
|
140
|
+
// ['session'] is applied; an explicit empty list disables exclusion.
|
|
141
|
+
const defaultExcludes = searchType === "any" && !includeExcludedTypes ? (config.search?.defaultExcludeTypes ?? ["session"]) : [];
|
|
136
142
|
// Empty queries — including ones that sanitize down to no searchable FTS
|
|
137
143
|
// tokens such as "." — should enumerate matching entries instead of
|
|
138
144
|
// returning an empty result set from FTS.
|
|
139
145
|
if (!hasSearchableTokens) {
|
|
140
146
|
const typeFilter = searchType === "any" ? undefined : searchType;
|
|
141
|
-
const allEntries = getAllEntries(db, typeFilter);
|
|
147
|
+
const allEntries = getAllEntries(db, typeFilter, defaultExcludes);
|
|
142
148
|
// Deduplicate by file path — multiple entries can share the same file
|
|
143
149
|
const seenFilePaths = new Set();
|
|
144
150
|
const uniqueEntries = allEntries.filter((ie) => {
|
|
@@ -187,7 +193,7 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
|
|
|
187
193
|
const typeFilter = searchType === "any" ? undefined : searchType;
|
|
188
194
|
const tEmbed0 = Date.now();
|
|
189
195
|
const embeddingPromise = tryVecScores(db, query, limit * 3, config);
|
|
190
|
-
const ftsResults = searchFts(db, query, limit * 3, typeFilter);
|
|
196
|
+
const ftsResults = searchFts(db, query, limit * 3, typeFilter, defaultExcludes);
|
|
191
197
|
const embeddingScores = await embeddingPromise;
|
|
192
198
|
const embedMs = Date.now() - tEmbed0;
|
|
193
199
|
const tRank0 = Date.now();
|
|
@@ -208,6 +214,11 @@ async function searchDatabase(db, query, searchType, limit, stashDir, allSourceD
|
|
|
208
214
|
embedScoreMap,
|
|
209
215
|
getEntryById: (id) => getEntryById(db, id) ?? undefined,
|
|
210
216
|
typeFilter,
|
|
217
|
+
// #627 — also exclude default-hidden types from the vector-only branch so a
|
|
218
|
+
// session asset that is a top-k vector neighbor (but not an FTS match) does
|
|
219
|
+
// not leak into default ('any') results. defaultExcludes is already []
|
|
220
|
+
// unless this is the untyped path without includeExcludedTypes.
|
|
221
|
+
excludeTypes: defaultExcludes,
|
|
211
222
|
});
|
|
212
223
|
// ── Scoring Phase ──────────────────────────────────────────────────────
|
|
213
224
|
// Apply boosts as multiplicative factors (all boosts in a single phase
|
|
@@ -9,6 +9,9 @@ const TYPE_BOOST = {
|
|
|
9
9
|
agent: 0.3,
|
|
10
10
|
script: 0.2,
|
|
11
11
|
knowledge: 0.22,
|
|
12
|
+
// Facts are authoritative, durable declarations about the stash — rank them
|
|
13
|
+
// alongside knowledge so they surface reliably when relevant.
|
|
14
|
+
fact: 0.22,
|
|
12
15
|
memory: -0.02,
|
|
13
16
|
};
|
|
14
17
|
const MAX_BOOST_SUM = 3.0;
|
|
@@ -206,6 +209,24 @@ const lessonStrengthContributor = {
|
|
|
206
209
|
return Math.min(0.3, 0.06 * strength);
|
|
207
210
|
},
|
|
208
211
|
};
|
|
212
|
+
/**
|
|
213
|
+
* Pinned-fact boost.
|
|
214
|
+
*
|
|
215
|
+
* Facts marked `pinned: true` form the small always-injected "core context"
|
|
216
|
+
* (see docs/design/fact-asset-type.md). The fact metadata contributor records
|
|
217
|
+
* a `pinned` search hint; here we give those facts a modest additive boost so
|
|
218
|
+
* the core outranks ordinary facts on otherwise-equal queries. Capped small so
|
|
219
|
+
* it cannot overpower an exact-name match.
|
|
220
|
+
*/
|
|
221
|
+
const pinnedFactRankingContributor = {
|
|
222
|
+
name: "pinned-fact-ranking",
|
|
223
|
+
appliesTo(item) {
|
|
224
|
+
return item.entry.type === "fact" && (item.entry.searchHints?.includes("pinned") ?? false);
|
|
225
|
+
},
|
|
226
|
+
adjust() {
|
|
227
|
+
return 0.15;
|
|
228
|
+
},
|
|
229
|
+
};
|
|
209
230
|
/**
|
|
210
231
|
* Blend ratio for scoped vs. global utility signals.
|
|
211
232
|
*
|
|
@@ -310,6 +331,7 @@ export const defaultRankingContributors = [
|
|
|
310
331
|
graphRankingContributor,
|
|
311
332
|
captureModeRankingContributor,
|
|
312
333
|
lessonStrengthContributor,
|
|
334
|
+
pinnedFactRankingContributor,
|
|
313
335
|
projectContextRankingContributor,
|
|
314
336
|
];
|
|
315
337
|
export const defaultUtilityRankingContributors = [utilityRankingContributor];
|
|
@@ -20,6 +20,7 @@ export function normalizeFtsScores(results) {
|
|
|
20
20
|
export function combineSearchScores(options) {
|
|
21
21
|
const FTS_WEIGHT = 0.7;
|
|
22
22
|
const VEC_WEIGHT = 0.3;
|
|
23
|
+
const excludeTypeSet = options.excludeTypes && options.excludeTypes.length > 0 ? new Set(options.excludeTypes) : null;
|
|
23
24
|
const scored = [];
|
|
24
25
|
const seenIds = new Set();
|
|
25
26
|
for (const [id, { score: ftsScore, result }] of options.ftsScoreMap) {
|
|
@@ -42,6 +43,9 @@ export function combineSearchScores(options) {
|
|
|
42
43
|
continue;
|
|
43
44
|
if (options.typeFilter && found.entry.type !== options.typeFilter)
|
|
44
45
|
continue;
|
|
46
|
+
// #627 — drop vector-only neighbors whose type is excluded on the default path.
|
|
47
|
+
if (excludeTypeSet?.has(found.entry.type))
|
|
48
|
+
continue;
|
|
45
49
|
scored.push({
|
|
46
50
|
id,
|
|
47
51
|
entry: found.entry,
|
|
@@ -78,6 +78,15 @@ const DIR_TYPE_MAP = [
|
|
|
78
78
|
type: "session",
|
|
79
79
|
test: (ext) => ext === ".md",
|
|
80
80
|
},
|
|
81
|
+
{
|
|
82
|
+
// Durable stash-level facts live under `facts/<category>/<name>.md`.
|
|
83
|
+
// classifyByDirectory walks every ancestor dir, so nested category
|
|
84
|
+
// subdirs still match. Without this entry a fact file would fall through
|
|
85
|
+
// to classifyBySmartMd and be mistyped as `knowledge`.
|
|
86
|
+
dir: "facts",
|
|
87
|
+
type: "fact",
|
|
88
|
+
test: (ext) => ext === ".md",
|
|
89
|
+
},
|
|
81
90
|
];
|
|
82
91
|
const COMMAND_PLACEHOLDER_RE = /\$ARGUMENTS|\$[123]\b/;
|
|
83
92
|
// Files that should never be treated as the typed asset for the surrounding
|
|
@@ -43,6 +43,7 @@ const TYPE_HINTS = {
|
|
|
43
43
|
script: "script assets are executable text files. Include a shebang and minimal usage comment.",
|
|
44
44
|
env: "env assets are `.env` files holding a group of related CONFIGURATION for an app/service (KEY=VALUE pairs, `#` comments) — URLs, flags, and any credentials it needs. Values may or may not be sensitive; all are protected (key names discoverable, values stay on disk). Inject with `akm env run env:<name> -- <cmd>` (the safe path — values never reach stdout/your context); do NOT run `akm env export` and read its output, as that prints values. For a single sensitive value used on its own for authentication (token, key, cert) use a `secret` instead. Never echo values back to the user.",
|
|
45
45
|
wiki: "wiki assets are markdown reference pages with `# Title` and structured headings.",
|
|
46
|
+
fact: "fact assets are durable stash-level facts (personal/team/project details, coding conventions, stash-meta). Frontmatter SHOULD include `description` and a `category` (personal|team|project|convention|meta); set `pinned: true` only for the small always-injected core. Keep each fact short, high-signal, and self-contained — it is durable context, not an episodic note.",
|
|
46
47
|
};
|
|
47
48
|
function hintForType(type) {
|
|
48
49
|
return TYPE_HINTS[type] ?? `assets of type "${type}" — produce sensible markdown with optional frontmatter.`;
|
|
@@ -107,6 +107,16 @@ export class ClaudeCodeProvider {
|
|
|
107
107
|
isAvailable() {
|
|
108
108
|
return fs.existsSync(claudeProjectsDir());
|
|
109
109
|
}
|
|
110
|
+
/**
|
|
111
|
+
* Directory holding Claude Code's per-project session JSONL files
|
|
112
|
+
* (`~/.claude/projects`, honoring `AKM_CLAUDE_PROJECTS_DIR`). Returns `[]`
|
|
113
|
+
* when the directory does not exist on this machine. See {@link
|
|
114
|
+
* SessionLogHarness.watchRoots}.
|
|
115
|
+
*/
|
|
116
|
+
watchRoots() {
|
|
117
|
+
const dir = claudeProjectsDir();
|
|
118
|
+
return fs.existsSync(dir) ? [dir] : [];
|
|
119
|
+
}
|
|
110
120
|
*readEvents(input) {
|
|
111
121
|
try {
|
|
112
122
|
for (const jsonlPath of this.#walkJsonl(claudeProjectsDir())) {
|
|
@@ -298,7 +308,7 @@ export class ClaudeCodeProvider {
|
|
|
298
308
|
const full = path.join(dir, entry.name);
|
|
299
309
|
if (entry.isDirectory())
|
|
300
310
|
yield* this.#walkJsonl(full);
|
|
301
|
-
else if (entry.name.endsWith(".jsonl"))
|
|
311
|
+
else if (entry.name.endsWith(".jsonl") && entry.name !== "journal.jsonl")
|
|
302
312
|
yield full;
|
|
303
313
|
}
|
|
304
314
|
}
|
|
@@ -26,6 +26,15 @@ export class OpenCodeProvider {
|
|
|
26
26
|
isAvailable() {
|
|
27
27
|
return fs.existsSync(this.#baseDir);
|
|
28
28
|
}
|
|
29
|
+
/**
|
|
30
|
+
* Directory holding opencode's per-project session metadata files
|
|
31
|
+
* (`<base>/storage/session`). Returns `[]` when it does not exist on this
|
|
32
|
+
* machine. See {@link SessionLogHarness.watchRoots}.
|
|
33
|
+
*/
|
|
34
|
+
watchRoots() {
|
|
35
|
+
const sessionRoot = path.join(this.#baseDir, "storage", "session");
|
|
36
|
+
return fs.existsSync(sessionRoot) ? [sessionRoot] : [];
|
|
37
|
+
}
|
|
29
38
|
*readEvents(input) {
|
|
30
39
|
// Legacy behavior: stream raw log lines from the top-level dir and `log/`
|
|
31
40
|
// subdirectory. Kept to keep `getExecutionLogCandidates` working without
|
|
@@ -28,6 +28,22 @@ const ERROR_PATTERNS = /error|failed|exception|cannot|undefined|null pointer|ENO
|
|
|
28
28
|
export function getAvailableHarnesses() {
|
|
29
29
|
return HARNESSES.filter((harness) => harness.isAvailable());
|
|
30
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* Map each available harness to its `{ harnessName, roots }` watch target,
|
|
33
|
+
* skipping harnesses that expose no roots (absent `watchRoots()` or an empty
|
|
34
|
+
* result). This is the one stable entry point the watcher uses so it never
|
|
35
|
+
* reaches into providers directly.
|
|
36
|
+
*/
|
|
37
|
+
export function getWatchTargets() {
|
|
38
|
+
const targets = [];
|
|
39
|
+
for (const harness of getAvailableHarnesses()) {
|
|
40
|
+
const roots = harness.watchRoots?.() ?? [];
|
|
41
|
+
if (roots.length === 0)
|
|
42
|
+
continue;
|
|
43
|
+
targets.push({ harnessName: harness.name, roots });
|
|
44
|
+
}
|
|
45
|
+
return targets;
|
|
46
|
+
}
|
|
31
47
|
export function normalizeSessionTopic(text) {
|
|
32
48
|
const normalized = text.replace(/\s+/g, " ").trim().toLowerCase();
|
|
33
49
|
if (normalized.length < 10)
|
package/dist/llm/client.js
CHANGED
|
@@ -119,9 +119,23 @@ function looksLikeContextOverflow(message) {
|
|
|
119
119
|
/**
|
|
120
120
|
* Decide whether a first-attempt {@link LlmCallError} is eligible for a single
|
|
121
121
|
* retry. Retryable: HTTP 5xx (`provider_error` with statusCode >= 500) and
|
|
122
|
-
* `network_error` whose message looks like a transient connection
|
|
123
|
-
*
|
|
124
|
-
*
|
|
122
|
+
* `network_error` whose message looks like a transient connection drop.
|
|
123
|
+
* NOT retryable: 4xx, `rate_limited` (429), `timeout`, `parse_error`, and
|
|
124
|
+
* context-overflow-classified errors.
|
|
125
|
+
*
|
|
126
|
+
* The connection-drop heuristic covers the substrings emitted across runtimes
|
|
127
|
+
* for a mid-flight socket close:
|
|
128
|
+
* - `ECONNRESET` / `EPIPE` — Node/libuv socket reset codes
|
|
129
|
+
* - `fetch failed` — undici's generic wrapper message
|
|
130
|
+
* - `socket connection was closed` — Bun's message for a dropped connection
|
|
131
|
+
* (e.g. "The socket connection was closed unexpectedly.")
|
|
132
|
+
* - `terminated` / `other side closed` — undici's phrasings for the same
|
|
133
|
+
*
|
|
134
|
+
* These all describe a transient transport failure where a second attempt can
|
|
135
|
+
* legitimately succeed, which is exactly the case a single bounded retry is
|
|
136
|
+
* meant to absorb. Before this list was widened, Bun's "socket connection was
|
|
137
|
+
* closed unexpectedly" fell through unretried and surfaced as a recurring
|
|
138
|
+
* failure in the improve/reflect and capability-probe flows.
|
|
125
139
|
*/
|
|
126
140
|
function isRetryable(err) {
|
|
127
141
|
if (looksLikeContextOverflow(err.message))
|
|
@@ -131,7 +145,12 @@ function isRetryable(err) {
|
|
|
131
145
|
}
|
|
132
146
|
if (err.code === "network_error") {
|
|
133
147
|
const lower = err.message.toLowerCase();
|
|
134
|
-
return lower.includes("econnreset") ||
|
|
148
|
+
return (lower.includes("econnreset") ||
|
|
149
|
+
lower.includes("epipe") ||
|
|
150
|
+
lower.includes("fetch failed") ||
|
|
151
|
+
lower.includes("socket connection was closed") ||
|
|
152
|
+
lower.includes("terminated") ||
|
|
153
|
+
lower.includes("other side closed"));
|
|
135
154
|
}
|
|
136
155
|
return false;
|
|
137
156
|
}
|
package/dist/llm/embedder.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
3
|
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
4
|
import { embedCacheKey, getCachedEmbedding, setCachedEmbedding } from "./embedders/cache.js";
|
|
5
|
-
import { isTransformersAvailable, LocalEmbedder } from "./embedders/local.js";
|
|
5
|
+
import { DEFAULT_LOCAL_MODEL, isTransformersAvailable, LocalEmbedder } from "./embedders/local.js";
|
|
6
6
|
import { hasRemoteEndpoint, RemoteEmbedder } from "./embedders/remote.js";
|
|
7
7
|
// ── Re-exports (public API) ─────────────────────────────────────────────────
|
|
8
8
|
export { clearEmbeddingCache } from "./embedders/cache.js";
|
|
@@ -52,7 +52,8 @@ export async function embed(text, embeddingConfig, signal) {
|
|
|
52
52
|
/**
|
|
53
53
|
* Generate embeddings for multiple texts in batch.
|
|
54
54
|
* Uses the OpenAI-compatible batch API for remote endpoints (batches of 100).
|
|
55
|
-
*
|
|
55
|
+
* Uses the LocalEmbedder.embedBatch path for the local transformer pipeline,
|
|
56
|
+
* which processes texts in chunks of 32 for genuine batched inference.
|
|
56
57
|
*/
|
|
57
58
|
export async function embedBatch(texts, embeddingConfig, signal) {
|
|
58
59
|
if (texts.length === 0)
|
|
@@ -60,8 +61,13 @@ export async function embedBatch(texts, embeddingConfig, signal) {
|
|
|
60
61
|
if (embeddingConfig && hasRemoteEndpoint(embeddingConfig)) {
|
|
61
62
|
return new RemoteEmbedder(embeddingConfig).embedBatch(texts, signal);
|
|
62
63
|
}
|
|
63
|
-
// Local transformer:
|
|
64
|
+
// Local transformer: use the batched path (chunks of 32 via LocalEmbedder).
|
|
65
|
+
// When a localModel override is set we cannot share the singleton (which uses
|
|
66
|
+
// the default model), so fall back to per-text embedWithModel in that case.
|
|
64
67
|
const localModel = embeddingConfig?.localModel;
|
|
68
|
+
if (!localModel) {
|
|
69
|
+
return getLocalEmbedder().embedBatch(texts, signal);
|
|
70
|
+
}
|
|
65
71
|
const results = [];
|
|
66
72
|
for (const text of texts) {
|
|
67
73
|
if (signal?.aborted) {
|
|
@@ -77,6 +83,24 @@ export async function embedBatch(texts, embeddingConfig, signal) {
|
|
|
77
83
|
// facade and its `@huggingface/transformers` import chain. Re-export
|
|
78
84
|
// preserves the existing public API.
|
|
79
85
|
export { cosineSimilarity } from "./embedders/types.js";
|
|
86
|
+
// ── Model ID resolution ─────────────────────────────────────────────────────
|
|
87
|
+
/**
|
|
88
|
+
* Derive a stable string identifier for the embedding model in use.
|
|
89
|
+
* This is the `model_id` stored in `body_embeddings` (and used for the
|
|
90
|
+
* drop-all-on-mismatch purge when the model changes).
|
|
91
|
+
*
|
|
92
|
+
* Rules:
|
|
93
|
+
* - Remote endpoint: use `config.model` (the API-level model name).
|
|
94
|
+
* - Local transformers: use `config.localModel ?? DEFAULT_LOCAL_MODEL`.
|
|
95
|
+
* - No config: use `DEFAULT_LOCAL_MODEL` (the shared singleton model).
|
|
96
|
+
*/
|
|
97
|
+
export function resolveEmbeddingModelId(embeddingConfig) {
|
|
98
|
+
if (!embeddingConfig)
|
|
99
|
+
return DEFAULT_LOCAL_MODEL;
|
|
100
|
+
if (hasRemoteEndpoint(embeddingConfig))
|
|
101
|
+
return embeddingConfig.model ?? "remote";
|
|
102
|
+
return embeddingConfig.localModel ?? DEFAULT_LOCAL_MODEL;
|
|
103
|
+
}
|
|
80
104
|
// ── Availability check ──────────────────────────────────────────────────────
|
|
81
105
|
/**
|
|
82
106
|
* Check whether embedding is available with a detailed reason on failure.
|