akm-cli 0.9.0-beta.5 → 0.9.0-beta.51
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +711 -0
- package/README.md +12 -4
- package/dist/akm +38 -0
- package/dist/akm-migrate-storage +38 -0
- package/dist/assets/profiles/default.json +9 -4
- package/dist/assets/profiles/frequent.json +1 -1
- package/dist/assets/profiles/memory-focus.json +1 -1
- package/dist/assets/profiles/quick.json +1 -1
- package/dist/assets/profiles/synthesize.json +15 -0
- package/dist/assets/profiles/thorough.json +1 -1
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +6 -2
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
- package/dist/assets/templates/html/health.html +281 -111
- package/dist/assets/wiki/ingest-workflow-template.md +38 -10
- package/dist/cli/parse-args.js +46 -1
- package/dist/cli/shared.js +28 -0
- package/dist/cli.js +27 -11
- package/dist/commands/agent/agent-dispatch.js +2 -2
- package/dist/commands/agent/agent-support.js +0 -7
- package/dist/commands/agent/contribute-cli.js +17 -4
- package/dist/commands/config-cli.js +18 -2
- package/dist/commands/env/child-env.js +47 -0
- package/dist/commands/env/env-cli.js +33 -26
- package/dist/commands/env/secret-cli.js +36 -22
- package/dist/commands/feedback-cli.js +15 -6
- package/dist/commands/graph/graph-cli.js +5 -13
- package/dist/commands/graph/graph.js +76 -72
- package/dist/commands/health/checks.js +49 -1
- package/dist/commands/health/html-report.js +422 -80
- package/dist/commands/health.js +386 -9
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/consolidate/chunking.js +141 -0
- package/dist/commands/improve/consolidate/eligibility.js +81 -0
- package/dist/commands/improve/consolidate/merge.js +145 -0
- package/dist/commands/improve/consolidate/sanitize.js +231 -0
- package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
- package/dist/commands/improve/consolidate.js +635 -660
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill.js +159 -69
- package/dist/commands/improve/eligibility.js +434 -0
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +124 -2
- package/dist/commands/improve/extract-prompt.js +39 -2
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +389 -40
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/homeostatic.js +467 -0
- package/dist/commands/improve/improve-auto-accept.js +138 -7
- package/dist/commands/improve/improve-cli.js +36 -61
- package/dist/commands/improve/improve-profiles.js +14 -0
- package/dist/commands/improve/improve-result-file.js +14 -25
- package/dist/commands/improve/improve-session.js +58 -0
- package/dist/commands/improve/improve.js +485 -2498
- package/dist/commands/improve/locks.js +154 -0
- package/dist/commands/improve/loop-stages.js +1083 -0
- package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
- package/dist/commands/improve/outcome-loop.js +256 -0
- package/dist/commands/improve/preparation.js +1966 -0
- package/dist/commands/improve/proactive-maintenance.js +115 -0
- package/dist/commands/improve/procedural.js +418 -0
- package/dist/commands/improve/recombine.js +850 -0
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/improve/reflect.js +183 -40
- package/dist/commands/improve/salience.js +438 -0
- package/dist/commands/improve/triage.js +93 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +38 -1
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +31 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +7 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/proposal/drain-policies.js +5 -0
- package/dist/commands/proposal/drain.js +43 -50
- package/dist/commands/proposal/proposal-cli.js +21 -31
- package/dist/commands/proposal/proposal.js +5 -0
- package/dist/commands/proposal/propose.js +7 -2
- package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
- package/dist/commands/proposal/validators/proposals.js +189 -63
- package/dist/commands/read/curate.js +414 -94
- package/dist/commands/read/knowledge.js +6 -3
- package/dist/commands/read/search-cli.js +9 -4
- package/dist/commands/read/search.js +10 -6
- package/dist/commands/read/show.js +86 -7
- package/dist/commands/sources/init.js +49 -17
- package/dist/commands/sources/installed-stashes.js +11 -3
- package/dist/commands/sources/schema-repair.js +43 -45
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/source-add.js +7 -3
- package/dist/commands/sources/stash-cli.js +28 -40
- package/dist/commands/sources/stash-skeleton.js +23 -8
- package/dist/commands/tasks/tasks-cli.js +19 -27
- package/dist/commands/tasks/tasks.js +39 -11
- package/dist/commands/wiki-cli.js +21 -35
- package/dist/core/asset/asset-registry.js +3 -1
- package/dist/core/asset/asset-spec.js +18 -2
- package/dist/core/asset/frontmatter.js +166 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/authoring-rules.js +92 -0
- package/dist/core/common.js +0 -5
- package/dist/core/config/config-migration.js +12 -11
- package/dist/core/config/config-schema.js +340 -56
- package/dist/core/config/config-types.js +3 -3
- package/dist/core/config/config.js +28 -7
- package/dist/core/events.js +3 -7
- package/dist/core/improve-types.js +11 -8
- package/dist/core/logs-db.js +10 -66
- package/dist/core/parse.js +36 -16
- package/dist/core/paths.js +3 -0
- package/dist/core/standards/resolve-standards-context.js +87 -0
- package/dist/core/standards/resolve-stash-standards.js +99 -0
- package/dist/core/standards/resolve-type-conventions.js +66 -0
- package/dist/core/state/migrations.js +714 -0
- package/dist/core/state-db.js +525 -474
- package/dist/indexer/db/db.js +439 -247
- package/dist/indexer/db/graph-db.js +129 -86
- package/dist/indexer/ensure-index.js +152 -17
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/graph/graph-extraction.js +218 -4
- package/dist/indexer/index-writer-lock.js +99 -0
- package/dist/indexer/indexer.js +123 -221
- package/dist/indexer/passes/dir-staleness.js +114 -0
- package/dist/indexer/passes/memory-inference.js +13 -5
- package/dist/indexer/passes/staleness-detect.js +2 -5
- package/dist/indexer/search/db-search.js +19 -6
- package/dist/indexer/search/ranking-contributors.js +22 -0
- package/dist/indexer/search/ranking.js +4 -0
- package/dist/indexer/search/search-source.js +17 -18
- package/dist/indexer/search/semantic-status.js +4 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/integrations/agent/config.js +6 -53
- package/dist/integrations/agent/index.js +2 -18
- package/dist/integrations/agent/prompts.js +75 -9
- package/dist/integrations/agent/runner-dispatch.js +59 -0
- package/dist/integrations/harnesses/claude/session-log.js +11 -1
- package/dist/integrations/harnesses/index.js +2 -3
- package/dist/integrations/harnesses/opencode/session-log.js +173 -3
- package/dist/integrations/harnesses/opencode-sdk/index.js +2 -2
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +0 -2
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/client.js +45 -15
- package/dist/llm/embedder.js +42 -3
- package/dist/llm/embedders/deterministic.js +66 -0
- package/dist/llm/embedders/local.js +66 -2
- package/dist/llm/feature-gate.js +8 -4
- package/dist/llm/graph-extract.js +67 -44
- package/dist/llm/memory-infer-impl.js +138 -0
- package/dist/llm/memory-infer.js +1 -127
- package/dist/llm/metadata-enhance.js +44 -31
- package/dist/llm/structured-call.js +49 -0
- package/dist/migrate-storage-node.mjs +8 -0
- package/dist/output/context.js +5 -5
- package/dist/output/renderers.js +74 -2
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/shapes/passthrough.js +0 -1
- package/dist/output/text/helpers.js +16 -1
- package/dist/registry/providers/skills-sh.js +21 -147
- package/dist/registry/providers/static-index.js +15 -157
- package/dist/registry/resolve.js +22 -9
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +2617 -1961
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +759 -510
- package/dist/setup/setup.js +29 -8
- package/dist/sources/include.js +6 -2
- package/dist/sources/providers/filesystem.js +0 -1
- package/dist/sources/providers/git-install.js +210 -0
- package/dist/sources/providers/git-provider.js +234 -0
- package/dist/sources/providers/git-stash.js +248 -0
- package/dist/sources/providers/git.js +10 -661
- package/dist/sources/providers/npm.js +2 -6
- package/dist/sources/providers/provider-utils.js +13 -7
- package/dist/sources/providers/sync-from-ref.js +9 -1
- package/dist/sources/providers/tar-utils.js +16 -8
- package/dist/sources/providers/website.js +9 -5
- package/dist/sources/website-ingest.js +187 -29
- package/dist/sources/wiki-fetchers/registry.js +53 -0
- package/dist/sources/wiki-fetchers/youtube.js +239 -0
- package/dist/storage/database.js +45 -10
- package/dist/storage/managed-db.js +82 -0
- package/dist/storage/repositories/registry-cache.js +92 -0
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/tasks/backends/cron.js +1 -1
- package/dist/tasks/backends/launchd.js +1 -1
- package/dist/tasks/backends/schtasks.js +1 -1
- package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
- package/dist/tasks/runner.js +5 -13
- package/dist/text-import-hook.mjs +0 -0
- package/dist/wiki/wiki.js +37 -0
- package/dist/workflows/db.js +3 -4
- package/dist/workflows/runtime/runs.js +1 -117
- package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/data-and-telemetry.md +3 -2
- package/docs/migration/release-notes/0.9.0.md +39 -0
- package/package.json +13 -11
- package/dist/commands/db-cli.js +0 -23
- package/dist/indexer/db/db-backup.js +0 -376
|
@@ -5,52 +5,50 @@ import { createHash } from "node:crypto";
|
|
|
5
5
|
import fs from "node:fs";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import readline from "node:readline";
|
|
8
|
-
import
|
|
8
|
+
import consolidateSystemPrompt from "../../assets/prompts/consolidate-system.md" with { type: "text" };
|
|
9
9
|
import { parseAssetRef } from "../../core/asset/asset-ref.js";
|
|
10
10
|
import { assembleAssetFromString, serializeFrontmatter } from "../../core/asset/asset-serialize.js";
|
|
11
11
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
12
12
|
import { resolveStashDir, timestampForFilename } from "../../core/common.js";
|
|
13
13
|
import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
|
|
14
14
|
import { ConfigError } from "../../core/errors.js";
|
|
15
|
-
|
|
15
|
+
// Note: appendEvent import removed (WS-3a: archive TTL machinery retired)
|
|
16
16
|
import { parseEmbeddedJsonResponse } from "../../core/parse.js";
|
|
17
|
+
import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
|
|
17
18
|
import { detectTruncatedDescription } from "../../core/text-truncation.js";
|
|
18
|
-
import {
|
|
19
|
+
import { hasSupersededStatus, MERGE_ABSOLUTE_FLOOR_CHARS, MERGE_SHRINK_RATIO_MIN, validateProposalFrontmatter, } from "../proposal/validators/proposal-quality-validators.js";
|
|
19
20
|
import { createProposal, isProposalSkipped, listProposals } from "../proposal/validators/proposals.js";
|
|
21
|
+
import { cacheHash, runDeterministicDedup, stripFrontmatterBody } from "./dedup.js";
|
|
22
|
+
import { checkGenerationGuard, checkLexicalDiversity, computeMergedGeneration, readAssetGeneration, runHomeostaticDemotion, shouldSkipHotProbationInLlm, } from "./homeostatic.js";
|
|
20
23
|
import { writeContradictEdge } from "./memory/memory-belief.js";
|
|
21
24
|
// Re-export the moved helpers so existing test imports continue to resolve.
|
|
22
25
|
export { hasSupersededStatus, validateProposalFrontmatter };
|
|
26
|
+
import { getBodyEmbeddings, getConsolidationJudgedMap, openStateDatabase, upsertBodyEmbeddings, upsertConsolidationJudged, withStateDb, } from "../../core/state-db.js";
|
|
23
27
|
import { warn } from "../../core/warn.js";
|
|
24
28
|
import { commitWriteTargetBoundary, deleteAssetFromSource, resolveWriteTarget, writeAssetToSource, } from "../../core/write-source.js";
|
|
25
29
|
import { closeDatabase, findEntryIdByRef, getAllEntries, getEntryById, getNeighborsByEntryId, openExistingDatabase, } from "../../indexer/db/db.js";
|
|
26
30
|
import { resolveImproveProcessRunnerFromProfile, runnerIsLlm } from "../../integrations/agent/runner.js";
|
|
27
31
|
import { chatCompletion } from "../../llm/client.js";
|
|
28
|
-
import { cosineSimilarity, embedBatch } from "../../llm/embedder.js";
|
|
32
|
+
import { cosineSimilarity, embedBatch, resolveEmbeddingModelId } from "../../llm/embedder.js";
|
|
29
33
|
import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
|
|
34
|
+
// Chunk sizing + per-chunk prompt assembly live in ./consolidate/chunking.
|
|
35
|
+
// Imported for internal use by the orchestrator and re-exported for importers.
|
|
36
|
+
import { buildChunkPrompt, computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
|
|
37
|
+
export { buildChunkPrompt, computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
|
|
38
|
+
// LLM-output sanitization (pure string/frontmatter transforms) lives in
|
|
39
|
+
// ./consolidate/sanitize. Imported for internal use + re-exported for importers.
|
|
40
|
+
import { normalizeUpdatedField, sanitizeMergedContent } from "./consolidate/sanitize.js";
|
|
41
|
+
export { normalizeUpdatedField, sanitizeMergedContent, stripOuterCodeFence } from "./consolidate/sanitize.js";
|
|
42
|
+
// Eligibility / safety predicates live in ./consolidate/eligibility. Imported
|
|
43
|
+
// for internal guard use; the two public predicates are re-exported.
|
|
44
|
+
import { consolidateGuardStatus, isConsolidationEligibleMemoryName, isHotCapturedMemory, isSessionCaptureMemoryName, } from "./consolidate/eligibility.js";
|
|
45
|
+
export { isConsolidationEligibleMemoryName, isHotCapturedMemory, isSessionCaptureMemoryName, } from "./consolidate/eligibility.js";
|
|
46
|
+
// Plan parsing / merging (pure op-reconciliation algebra) lives in
|
|
47
|
+
// ./consolidate/merge. Imported for internal use; mergePlans re-exported.
|
|
48
|
+
import { isValidOp, mergePlans } from "./consolidate/merge.js";
|
|
49
|
+
export { mergePlans } from "./consolidate/merge.js";
|
|
30
50
|
// ── Prompts ─────────────────────────────────────────────────────────────────
|
|
31
|
-
const CONSOLIDATE_SYSTEM_PROMPT =
|
|
32
|
-
|
|
33
|
-
Rules:
|
|
34
|
-
1. MERGE: Two or more memories are substantially duplicated or closely related → propose merging. Return the primary ref to keep and secondary refs to delete. Do NOT include mergedContent — the merge will be executed in a separate step.
|
|
35
|
-
2. DELETE: Memory is clearly outdated, contradicted, or redundant → propose deletion. NEVER propose delete for memories annotated \`(captureMode: hot)\` — they are user-explicit and only the user can retire them. The downstream guard will refuse these regardless, so proposing them just wastes tokens.
|
|
36
|
-
3. PROMOTE: Memory expresses a stable, reusable fact suitable as a \`knowledge:\` asset → propose promotion. Do NOT delete the source memory. NEVER propose promote / merge / contradict for memories annotated \`(already queued)\` — they have a pending proposal whose body matches; a duplicate will be deterministically dropped, so proposing them just wastes tokens.
|
|
37
|
-
4. CONTRADICT: Two memories make mutually exclusive factual claims about the same subject (e.g. "always use VPN" vs "VPN is optional") → mark the older or less authoritative one as contradicted. This writes a contradictedBy edge so the belief-resolution SCC algorithm can resolve the conflict. Do NOT delete contradicted memories — let the belief resolver decide.
|
|
38
|
-
5. KEEP: Memory is unique and current → omit from output.
|
|
39
|
-
|
|
40
|
-
Return ONLY JSON (no prose, no code fences):
|
|
41
|
-
{
|
|
42
|
-
"operations": [
|
|
43
|
-
{ "op": "merge", "primary": "memory:<name>", "secondaries": ["memory:<name>", ...], "mergeStrategy": "synthesize", "confidence": 0.95 },
|
|
44
|
-
{ "op": "delete", "ref": "memory:<name>", "reason": "<brief reason>", "confidence": 0.90 },
|
|
45
|
-
{ "op": "promote", "ref": "memory:<name>", "knowledgeRef": "knowledge:<suggested-slug>", "reason": "<brief reason>", "description": "<one sentence describing the new knowledge asset>", "confidence": 0.92 },
|
|
46
|
-
{ "op": "contradict", "ref": "memory:<name>", "contradictedByRef": "memory:<name>", "reason": "<brief reason>", "confidence": 0.88 }
|
|
47
|
-
],
|
|
48
|
-
"warnings": ["<optional concerns>"]
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
For every operation, emit a \`confidence\` field in [0, 1] expressing your certainty that the operation is correct and safe. Use 0.95+ only when evidence is unambiguous. Omit the field rather than guessing if you are uncertain.
|
|
52
|
-
|
|
53
|
-
When the merged content includes an \`updated\` frontmatter field, the value MUST be a real ISO date string (e.g. \`updated: 2026-05-20\`). NEVER emit \`updated: today\`, \`updated: {today}\`, \`updated: {today: null}\`, \`updated: now\`, or any other literal placeholder/template-variable. If you do not have a real source-of-truth date, OMIT the \`updated\` field entirely — the post-processor will not invent one for you.`;
|
|
51
|
+
const CONSOLIDATE_SYSTEM_PROMPT = consolidateSystemPrompt;
|
|
54
52
|
/**
|
|
55
53
|
* JSON Schema for structured consolidate plans (PR 1 of the asset-writers
|
|
56
54
|
* decision — see knowledge:projects/akm/asset-writers-investigation/00-synthesis).
|
|
@@ -136,140 +134,17 @@ export const CONSOLIDATE_PLAN_JSON_SCHEMA = {
|
|
|
136
134
|
},
|
|
137
135
|
},
|
|
138
136
|
};
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* Returns true when the memory file has `captureMode: hot` in its frontmatter.
|
|
144
|
-
*
|
|
145
|
-
* Hot memories are USER-EXPLICIT (written via `akm remember` on the hot path).
|
|
146
|
-
* The consolidate LLM is forbidden from deleting or auto-merging them — the
|
|
147
|
-
* user wrote them on purpose and only the user can decide to retire them.
|
|
148
|
-
*
|
|
149
|
-
* Reads the file once per check; consolidate runs against ~10 memories per
|
|
150
|
-
* chunk so the IO cost is trivial. Returns false on any read/parse error
|
|
151
|
-
* (fail-safe: an unparseable file is treated as not-hot, but the broader
|
|
152
|
-
* consolidate flow already guards against unparseable memories elsewhere).
|
|
153
|
-
*
|
|
154
|
-
* Defends against four observed defect classes (see
|
|
155
|
-
* `memory:akm-improve-critical-review-2026-05-20`):
|
|
156
|
-
* - LLM marks a memory contradicted then deletes (dangling contradictedBy)
|
|
157
|
-
* - LLM merges two unrelated memories sharing a topic keyword
|
|
158
|
-
* - LLM judges a recent durable design memo as "redundant"
|
|
159
|
-
* - Cascade deletes (LLM uses ref:X as `contradictedBy` for ref:Y then deletes both)
|
|
160
|
-
*/
|
|
161
|
-
export function isHotCapturedMemory(filePath) {
|
|
162
|
-
try {
|
|
163
|
-
if (!fs.existsSync(filePath))
|
|
164
|
-
return false;
|
|
165
|
-
const content = fs.readFileSync(filePath, "utf8");
|
|
166
|
-
const parsed = parseFrontmatter(content);
|
|
167
|
-
return hasHotCaptureMode(parsed.data);
|
|
168
|
-
}
|
|
169
|
-
catch {
|
|
170
|
-
return false;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
function consolidateGuardStatus(filePath) {
|
|
174
|
-
if (!fs.existsSync(filePath))
|
|
175
|
-
return "missing";
|
|
176
|
-
let content;
|
|
177
|
-
try {
|
|
178
|
-
content = fs.readFileSync(filePath, "utf8");
|
|
179
|
-
}
|
|
180
|
-
catch {
|
|
181
|
-
return "unparseable";
|
|
182
|
-
}
|
|
183
|
-
let parsed;
|
|
184
|
-
try {
|
|
185
|
-
parsed = parseFrontmatter(content);
|
|
186
|
-
}
|
|
187
|
-
catch {
|
|
188
|
-
return "unparseable";
|
|
189
|
-
}
|
|
190
|
-
const data = parsed.data;
|
|
191
|
-
if (!data || Object.keys(data).length === 0)
|
|
192
|
-
return "unparseable";
|
|
193
|
-
return hasHotCaptureMode(data) ? "hot" : "safe";
|
|
194
|
-
}
|
|
195
|
-
// ── Chunk sizing ─────────────────────────────────────────────────────────────
|
|
196
|
-
/**
|
|
197
|
-
* Conservative chars-per-token estimate used when computing prompt budgets.
|
|
198
|
-
* English text averages roughly 4 chars/token for most LLM tokenizers. We use
|
|
199
|
-
* 3 to stay conservative (shorter tokens = more tokens per char).
|
|
200
|
-
*/
|
|
201
|
-
const CHARS_PER_TOKEN = 3;
|
|
202
|
-
/**
|
|
203
|
-
* Overhead budget reserved for the system prompt, chunk header lines, and per-
|
|
204
|
-
* memory metadata lines (name, description, tags, separator). Measured at
|
|
205
|
-
* roughly 600 chars for the system prompt + ~100 chars of header + ~50 chars
|
|
206
|
-
* per memory × chunk size. We round up to 2 000 tokens to leave room for the
|
|
207
|
-
* model's own output.
|
|
208
|
-
*/
|
|
209
|
-
const PROMPT_OVERHEAD_TOKENS = 2_000;
|
|
210
|
-
/**
|
|
211
|
-
* Default effective token budget used when the default LLM profile's
|
|
212
|
-
* `contextLength` is not set. This is intentionally conservative (4 096)
|
|
213
|
-
* rather than being set to the model's actual context window, because:
|
|
214
|
-
*
|
|
215
|
-
* - When the agent path is used, the agent CLI (e.g. opencode)
|
|
216
|
-
* prepends its own large system prompt + conversation history before
|
|
217
|
-
* forwarding to the model. That overhead easily consumes 30K+ tokens on
|
|
218
|
-
* a model with a 16K context window, leaving very little room for
|
|
219
|
-
* chunk content.
|
|
220
|
-
* - When the HTTP path is used (an LLM profile is selected), only the akm
|
|
221
|
-
* system prompt and user prompt are sent, so the budget can be set to the
|
|
222
|
-
* model's actual context length via profiles.llm[defaults.llm].contextLength.
|
|
223
|
-
*
|
|
224
|
-
* Set profiles.llm[defaults.llm].contextLength in your config file to the
|
|
225
|
-
* model's actual context window to allow larger chunks on the HTTP path.
|
|
226
|
-
*/
|
|
227
|
-
export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
|
|
228
|
-
/**
|
|
229
|
-
* Given the model's context window and the per-memory body truncation limit,
|
|
230
|
-
* return the maximum number of memories that can safely fit in one chunk
|
|
231
|
-
* without the prompt overflowing the context window.
|
|
232
|
-
*
|
|
233
|
-
* The formula is:
|
|
234
|
-
* usableTokens = contextLength - PROMPT_OVERHEAD_TOKENS
|
|
235
|
-
* tokensPerMemory = ceil(bodyTruncation / CHARS_PER_TOKEN)
|
|
236
|
-
* chunkSize = floor(usableTokens / tokensPerMemory)
|
|
237
|
-
*
|
|
238
|
-
* Result is clamped between 1 and 50 to avoid degenerate values.
|
|
239
|
-
*
|
|
240
|
-
* @param contextLength - Model context window in tokens.
|
|
241
|
-
* @param bodyTruncation - Max chars per memory body included in the prompt.
|
|
242
|
-
* @param maxChunkSize - Optional override for the hardcoded cap of 50 (1–50).
|
|
243
|
-
*/
|
|
244
|
-
export function computeSafeChunkSize(contextLength, bodyTruncation, maxChunkSize) {
|
|
245
|
-
const usableTokens = Math.max(contextLength - PROMPT_OVERHEAD_TOKENS, 0);
|
|
246
|
-
const tokensPerMemory = Math.max(Math.ceil(bodyTruncation / CHARS_PER_TOKEN), 1);
|
|
247
|
-
const raw = Math.floor(usableTokens / tokensPerMemory);
|
|
248
|
-
return Math.max(1, Math.min(maxChunkSize ?? 50, raw));
|
|
249
|
-
}
|
|
250
|
-
// ── Similarity clustering (C-1 / #380) ──────────────────────────────────────
|
|
251
|
-
/**
|
|
252
|
-
* Re-order memories so that similar ones are placed adjacent to each other
|
|
253
|
-
* before the memories are sliced into chunks. This ensures high-similarity
|
|
254
|
-
* memories land in the same LLM context window, allowing the consolidate
|
|
255
|
-
* model to detect and merge duplicates that would otherwise be split across
|
|
256
|
-
* chunks and survive indefinitely.
|
|
257
|
-
*
|
|
258
|
-
* Algorithm: greedy nearest-neighbour chain starting from the first memory.
|
|
259
|
-
* Each step selects the unused memory with the highest cosine similarity to
|
|
260
|
-
* the last-placed memory. O(n²) — acceptable for the expected N < 200.
|
|
261
|
-
*
|
|
262
|
-
* mem0 arXiv:2504.19413 — every candidate compared against whole store.
|
|
263
|
-
* A-MEM arXiv:2502.12110 — atomic notes linked by similarity.
|
|
264
|
-
*
|
|
265
|
-
* Returns the original order unchanged when:
|
|
266
|
-
* - The embedding config is not present.
|
|
267
|
-
* - Embedding requests fail (fail-open).
|
|
268
|
-
* - There are fewer than 3 memories (no benefit to reordering).
|
|
269
|
-
*/
|
|
270
|
-
async function clusterMemoriesBySimilarity(memories, config) {
|
|
137
|
+
async function clusterMemoriesBySimilarity(memories, config, stateDb) {
|
|
138
|
+
const noTelemetry = { embedMs: 0, cacheHits: 0, cacheMisses: 0 };
|
|
271
139
|
if (memories.length < 3 || !config.embedding)
|
|
272
|
-
return memories;
|
|
140
|
+
return { ordered: memories, embedTelemetry: noTelemetry };
|
|
141
|
+
// WS-3a: cluster uses description+tags as the embedding input (NOT the raw
|
|
142
|
+
// body) — this is intentionally different from the dedup/body cache because
|
|
143
|
+
// the clustering goal is semantic grouping, not dedup twin detection.
|
|
144
|
+
// The body_embeddings cache is keyed by cacheHash(body); clustering inputs
|
|
145
|
+
// are keyed by cacheHash(description+tags text). Re-use the same table with
|
|
146
|
+
// a distinct hash so the two lookup sets never collide.
|
|
147
|
+
const modelId = resolveEmbeddingModelId(config.embedding);
|
|
273
148
|
const texts = memories.map((m) => {
|
|
274
149
|
const parts = [];
|
|
275
150
|
if (m.description)
|
|
@@ -278,16 +153,91 @@ async function clusterMemoriesBySimilarity(memories, config) {
|
|
|
278
153
|
parts.push(m.tags.join(" "));
|
|
279
154
|
return parts.join(". ") || m.name;
|
|
280
155
|
});
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
156
|
+
// Compute content hashes for the cluster texts (not bodies — different input).
|
|
157
|
+
const contentHashes = texts.map((t) => createHash("sha256").update(t, "utf8").digest("hex"));
|
|
158
|
+
// WS-5: track embed cache hits/misses for perf telemetry.
|
|
159
|
+
let embedMs = 0;
|
|
160
|
+
let cacheHits = 0;
|
|
161
|
+
let cacheMisses = 0;
|
|
162
|
+
let cachedVecs = new Map();
|
|
163
|
+
if (stateDb) {
|
|
164
|
+
try {
|
|
165
|
+
cachedVecs = getBodyEmbeddings(stateDb, contentHashes, modelId);
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
// Fail open.
|
|
169
|
+
cachedVecs = new Map();
|
|
170
|
+
}
|
|
284
171
|
}
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
172
|
+
const missIndices = [];
|
|
173
|
+
const missTexts = [];
|
|
174
|
+
for (let i = 0; i < texts.length; i++) {
|
|
175
|
+
if (!cachedVecs.has(contentHashes[i])) {
|
|
176
|
+
missIndices.push(i);
|
|
177
|
+
missTexts.push(texts[i]);
|
|
178
|
+
cacheMisses++;
|
|
179
|
+
}
|
|
180
|
+
else {
|
|
181
|
+
cacheHits++;
|
|
182
|
+
}
|
|
288
183
|
}
|
|
184
|
+
let missVecs = [];
|
|
185
|
+
if (missTexts.length > 0) {
|
|
186
|
+
const embedStart = Date.now();
|
|
187
|
+
try {
|
|
188
|
+
missVecs = await embedBatch(missTexts, config.embedding);
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
// Fail open: embedding failures degrade gracefully to original order.
|
|
192
|
+
return { ordered: memories, embedTelemetry: { embedMs, cacheHits, cacheMisses } };
|
|
193
|
+
}
|
|
194
|
+
finally {
|
|
195
|
+
embedMs += Date.now() - embedStart;
|
|
196
|
+
}
|
|
197
|
+
// Upsert newly computed vectors into the cache.
|
|
198
|
+
if (stateDb && missVecs.length === missTexts.length) {
|
|
199
|
+
try {
|
|
200
|
+
const toUpsert = missIndices.map((idx, pos) => ({
|
|
201
|
+
contentHash: contentHashes[idx],
|
|
202
|
+
embedding: missVecs[pos],
|
|
203
|
+
modelId,
|
|
204
|
+
}));
|
|
205
|
+
upsertBodyEmbeddings(stateDb, toUpsert);
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
// Fail open: cache write errors are non-fatal.
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
// Assemble the full embedding array in memories order.
|
|
213
|
+
let embeddings = null;
|
|
214
|
+
{
|
|
215
|
+
const assembled = [];
|
|
216
|
+
let ok = true;
|
|
217
|
+
for (let i = 0; i < memories.length; i++) {
|
|
218
|
+
const hash = contentHashes[i];
|
|
219
|
+
const cached = cachedVecs.get(hash);
|
|
220
|
+
if (cached) {
|
|
221
|
+
assembled.push(cached);
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
const missPos = missIndices.indexOf(i);
|
|
225
|
+
const vec = missPos >= 0 ? missVecs[missPos] : undefined;
|
|
226
|
+
if (vec) {
|
|
227
|
+
assembled.push(vec);
|
|
228
|
+
}
|
|
229
|
+
else {
|
|
230
|
+
ok = false;
|
|
231
|
+
break;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
if (ok && assembled.length === memories.length) {
|
|
235
|
+
embeddings = assembled;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
const embedTelemetry = { embedMs, cacheHits, cacheMisses };
|
|
289
239
|
if (!embeddings || embeddings.length !== memories.length)
|
|
290
|
-
return memories;
|
|
240
|
+
return { ordered: memories, embedTelemetry };
|
|
291
241
|
// Greedy nearest-neighbour chain.
|
|
292
242
|
const used = new Array(memories.length).fill(false);
|
|
293
243
|
const ordered = [];
|
|
@@ -313,87 +263,16 @@ async function clusterMemoriesBySimilarity(memories, config) {
|
|
|
313
263
|
used[bestIdx] = true;
|
|
314
264
|
current = bestIdx;
|
|
315
265
|
}
|
|
316
|
-
return ordered;
|
|
266
|
+
return { ordered, embedTelemetry };
|
|
317
267
|
}
|
|
318
268
|
// ── Chunk helpers ────────────────────────────────────────────────────────────
|
|
319
|
-
/**
|
|
320
|
-
* Build the per-chunk user prompt fed to the consolidate LLM.
|
|
321
|
-
*
|
|
322
|
-
* Each memory is annotated with two flags that drive the system-prompt
|
|
323
|
-
* rules at lines 181-186:
|
|
324
|
-
* - `(captureMode: hot)` — user-explicit memory; system prompt rule 2
|
|
325
|
-
* forbids proposing delete. ~60 wasted LLM verdicts/4h on this user's
|
|
326
|
-
* stack before this annotation.
|
|
327
|
-
* - `(already queued)` — the memory's body hash matches a pending
|
|
328
|
-
* consolidate proposal; system prompt rule 3 forbids proposing
|
|
329
|
-
* promote/merge/contradict. ~107/4h before this annotation.
|
|
330
|
-
*
|
|
331
|
-
* Both annotations are visible to the LLM. `pendingProposalBodyHashes`
|
|
332
|
-
* is precomputed once per run by `loadPendingConsolidateProposalHashes`
|
|
333
|
-
* so the cost stays O(memories) inside the chunk loop.
|
|
334
|
-
*/
|
|
335
|
-
export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks, bodyTruncation, pendingProposalBodyHashes = new Set()) {
|
|
336
|
-
const start = memories[0] ? `memory:${memories[0].name}` : "";
|
|
337
|
-
const end = memories[memories.length - 1] ? `memory:${memories[memories.length - 1].name}` : "";
|
|
338
|
-
const annotationsByIndex = [];
|
|
339
|
-
const hotRefs = [];
|
|
340
|
-
for (const m of memories) {
|
|
341
|
-
let body = "";
|
|
342
|
-
try {
|
|
343
|
-
body = fs.readFileSync(m.filePath, "utf8");
|
|
344
|
-
}
|
|
345
|
-
catch {
|
|
346
|
-
body = "(unreadable)";
|
|
347
|
-
}
|
|
348
|
-
const parsed = parseFrontmatter(body);
|
|
349
|
-
const isHot = parsed.data.captureMode === "hot";
|
|
350
|
-
const bodyHash = createHash("sha256").update(parsed.content.trim(), "utf8").digest("hex");
|
|
351
|
-
const isAlreadyQueued = pendingProposalBodyHashes.has(bodyHash);
|
|
352
|
-
annotationsByIndex.push({ isHot, isAlreadyQueued, body });
|
|
353
|
-
if (isHot)
|
|
354
|
-
hotRefs.push(`memory:${m.name}`);
|
|
355
|
-
}
|
|
356
|
-
const lines = [
|
|
357
|
-
`Source: ${sourceName}`,
|
|
358
|
-
`Chunk ${chunkIndex + 1} of ${totalChunks}, memories ${start}–${end}:`,
|
|
359
|
-
"",
|
|
360
|
-
];
|
|
361
|
-
// Top-of-prompt protection block for hot refs. Neutral phrasing — avoid
|
|
362
|
-
// op-words like "promote", "merge", "contradict" so the model doesn't
|
|
363
|
-
// accidentally treat the warning as a hint to use that op elsewhere
|
|
364
|
-
// (variant B leaked the word "contradict" into the control sample
|
|
365
|
-
// during the diagnostic).
|
|
366
|
-
if (hotRefs.length > 0) {
|
|
367
|
-
lines.push("⛔ DO NOT propose any `delete` operation for these refs — they are user-explicit (captureMode: hot) and the downstream guard refuses them regardless. Proposing delete for any of these only wastes tokens.");
|
|
368
|
-
for (const ref of hotRefs)
|
|
369
|
-
lines.push(` - ${ref}`);
|
|
370
|
-
lines.push("");
|
|
371
|
-
}
|
|
372
|
-
for (let i = 0; i < memories.length; i++) {
|
|
373
|
-
const m = memories[i];
|
|
374
|
-
const { isHot, isAlreadyQueued, body } = annotationsByIndex[i];
|
|
375
|
-
const annotations = [];
|
|
376
|
-
if (isHot)
|
|
377
|
-
annotations.push("captureMode: hot");
|
|
378
|
-
if (isAlreadyQueued)
|
|
379
|
-
annotations.push("already queued");
|
|
380
|
-
const annotationSuffix = annotations.length > 0 ? ` (${annotations.join("; ")})` : "";
|
|
381
|
-
lines.push(`[${i + 1}] memory:${m.name}${annotationSuffix}`);
|
|
382
|
-
lines.push(`Description: ${m.description || "(none)"}`);
|
|
383
|
-
lines.push(`Tags: ${m.tags.length > 0 ? m.tags.join(", ") : "(none)"}`);
|
|
384
|
-
lines.push("---");
|
|
385
|
-
lines.push(body.slice(0, bodyTruncation));
|
|
386
|
-
lines.push("");
|
|
387
|
-
}
|
|
388
|
-
return lines.join("\n");
|
|
389
|
-
}
|
|
390
269
|
/**
|
|
391
270
|
* Precompute body-hashes of all currently-pending consolidate proposals so
|
|
392
271
|
* the per-chunk prompt can annotate memories whose body would just produce
|
|
393
|
-
* a deterministic `dedup_pending_proposal` skip.
|
|
394
|
-
*
|
|
395
|
-
*
|
|
396
|
-
* nothing" so the LLM still proposes
|
|
272
|
+
* a deterministic `dedup_pending_proposal` skip. Uses `cacheHash` (case-
|
|
273
|
+
* preserving stripped body) — the same domain used by the body-embedding
|
|
274
|
+
* cache and `computeMemoryContentHash`. Empty set on any read/parse error
|
|
275
|
+
* — fail-safe to "annotate nothing" so the LLM still proposes.
|
|
397
276
|
*/
|
|
398
277
|
function loadPendingConsolidateProposalHashes(stashDir) {
|
|
399
278
|
const hashes = new Set();
|
|
@@ -401,8 +280,7 @@ function loadPendingConsolidateProposalHashes(stashDir) {
|
|
|
401
280
|
const pending = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
|
|
402
281
|
for (const p of pending) {
|
|
403
282
|
try {
|
|
404
|
-
|
|
405
|
-
hashes.add(createHash("sha256").update(body, "utf8").digest("hex"));
|
|
283
|
+
hashes.add(cacheHash(p.payload.content));
|
|
406
284
|
}
|
|
407
285
|
catch {
|
|
408
286
|
// skip malformed payloads — they can't dedup anyway
|
|
@@ -414,148 +292,6 @@ function loadPendingConsolidateProposalHashes(stashDir) {
|
|
|
414
292
|
}
|
|
415
293
|
return hashes;
|
|
416
294
|
}
|
|
417
|
-
function isValidOp(op) {
|
|
418
|
-
if (typeof op !== "object" || op === null)
|
|
419
|
-
return false;
|
|
420
|
-
const o = op;
|
|
421
|
-
if (o.op === "merge") {
|
|
422
|
-
return typeof o.primary === "string" && Array.isArray(o.secondaries);
|
|
423
|
-
}
|
|
424
|
-
if (o.op === "delete") {
|
|
425
|
-
return typeof o.ref === "string";
|
|
426
|
-
}
|
|
427
|
-
if (o.op === "promote") {
|
|
428
|
-
return typeof o.ref === "string" && typeof o.knowledgeRef === "string";
|
|
429
|
-
}
|
|
430
|
-
if (o.op === "contradict") {
|
|
431
|
-
return typeof o.ref === "string" && typeof o.contradictedByRef === "string";
|
|
432
|
-
}
|
|
433
|
-
return false;
|
|
434
|
-
}
|
|
435
|
-
export function mergePlans(chunks, knownRefs) {
|
|
436
|
-
const mergeOps = new Map();
|
|
437
|
-
const deleteOps = new Map();
|
|
438
|
-
const promoteOps = new Map();
|
|
439
|
-
// C-3 / #382: contradict ops keyed by `ref|contradictedByRef` to deduplicate.
|
|
440
|
-
const contradictOps = new Map();
|
|
441
|
-
const warnings = [];
|
|
442
|
-
for (const chunk of chunks) {
|
|
443
|
-
for (const op of chunk) {
|
|
444
|
-
if (op.op === "merge") {
|
|
445
|
-
// Drop ops whose primary the LLM hallucinated (not in the loaded memory
|
|
446
|
-
// pool). Without this guard, a hallucinated primary flows all the way to
|
|
447
|
-
// Phase B where !memoryByRef.has(primary) fires and charges every real
|
|
448
|
-
// secondary with merge_primary_missing — masking LLM hallucinations as
|
|
449
|
-
// filter regressions in health metrics.
|
|
450
|
-
if (knownRefs && !knownRefs.has(op.primary)) {
|
|
451
|
-
warnings.push(`mergePlans: primary ${op.primary} not in loaded memory pool (LLM hallucination) — dropping op before execution.`);
|
|
452
|
-
// Use a dedicated skip reason so dashboards can distinguish
|
|
453
|
-
// hallucinated primaries from stale-DB regressions.
|
|
454
|
-
// Secondaries are real refs; they are NOT charged here — they remain
|
|
455
|
-
// available for other ops to claim.
|
|
456
|
-
continue;
|
|
457
|
-
}
|
|
458
|
-
// Filter hallucinated secondaries while preserving real ones.
|
|
459
|
-
let mergeOp = op;
|
|
460
|
-
if (knownRefs) {
|
|
461
|
-
const filteredSecondaries = op.secondaries.filter((sec) => {
|
|
462
|
-
if (!knownRefs.has(sec)) {
|
|
463
|
-
warnings.push(`mergePlans: secondary ${sec} not in loaded memory pool (LLM hallucination) — dropping from op.`);
|
|
464
|
-
return false;
|
|
465
|
-
}
|
|
466
|
-
return true;
|
|
467
|
-
});
|
|
468
|
-
if (filteredSecondaries.length !== op.secondaries.length) {
|
|
469
|
-
mergeOp = { ...op, secondaries: filteredSecondaries };
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
|
-
// merge wins over delete
|
|
473
|
-
if (deleteOps.has(mergeOp.primary)) {
|
|
474
|
-
deleteOps.delete(mergeOp.primary);
|
|
475
|
-
}
|
|
476
|
-
for (const sec of mergeOp.secondaries) {
|
|
477
|
-
if (deleteOps.has(sec))
|
|
478
|
-
deleteOps.delete(sec);
|
|
479
|
-
}
|
|
480
|
-
mergeOps.set(mergeOp.primary, mergeOp);
|
|
481
|
-
}
|
|
482
|
-
else if (op.op === "delete") {
|
|
483
|
-
// merge and promote both win over delete. A promote is non-destructive
|
|
484
|
-
// (creates a proposal) but the source memory is counted in `promoted`;
|
|
485
|
-
// if a delete also fires, the ref lands in both `promoted` and
|
|
486
|
-
// `skipReasons`, breaking the invariant by +1.
|
|
487
|
-
if (!mergeOps.has(op.ref) && !promoteOps.has(op.ref)) {
|
|
488
|
-
deleteOps.set(op.ref, op);
|
|
489
|
-
}
|
|
490
|
-
}
|
|
491
|
-
else if (op.op === "promote") {
|
|
492
|
-
// C-2 / #381: when both a promote and a merge target the same ref,
|
|
493
|
-
// queue the promote FIRST rather than discarding it. The promote op
|
|
494
|
-
// routes through createProposal (the human-gated proposal queue), so
|
|
495
|
-
// it is non-destructive. The merge follows after the proposal is
|
|
496
|
-
// created. This preserves the human reviewer's ability to inspect the
|
|
497
|
-
// promotion before the source memory is merged/deleted.
|
|
498
|
-
// AGM K*8 — retain the maximally informative consistent subset.
|
|
499
|
-
promoteOps.set(op.ref, op);
|
|
500
|
-
}
|
|
501
|
-
else if (op.op === "contradict") {
|
|
502
|
-
// Deduplicate by ref+contradictedByRef pair.
|
|
503
|
-
const key = `${op.ref}|${op.contradictedByRef}`;
|
|
504
|
-
if (!contradictOps.has(key)) {
|
|
505
|
-
contradictOps.set(key, op);
|
|
506
|
-
}
|
|
507
|
-
}
|
|
508
|
-
}
|
|
509
|
-
}
|
|
510
|
-
// Second pass: enforce merge-wins-over-delete and deduplicate secondaries.
|
|
511
|
-
//
|
|
512
|
-
// 1. Delete/secondary ordering bug: the per-chunk loop removes delete ops
|
|
513
|
-
// for secondaries that were already in deleteOps, but misses the case
|
|
514
|
-
// where the delete chunk came first. A full sweep here fixes both orders.
|
|
515
|
-
//
|
|
516
|
-
// 2. Cross-merge secondary dedup: if ref A is a secondary in two merge ops,
|
|
517
|
-
// only the first (insertion-order) retains it. Without this, a successful
|
|
518
|
-
// merge credits A to mergedSecondaries and a later merge's emitMerge-
|
|
519
|
-
// FailureSkips also charges A to skipReasons — double-counting A while
|
|
520
|
-
// processed has it only once.
|
|
521
|
-
//
|
|
522
|
-
// 3. Primary-as-secondary dedup: if ref A is a primary in one merge op and
|
|
523
|
-
// a secondary in another, remove A from the secondary list. Both merges
|
|
524
|
-
// would otherwise claim A (merged++ for A, then mergedSecondaries++ for A)
|
|
525
|
-
// breaking the invariant the same way.
|
|
526
|
-
// Also remove delete ops for any ref claimed by a promote op (handles the
|
|
527
|
-
// case where the delete chunk appeared before the promote chunk).
|
|
528
|
-
for (const ref of promoteOps.keys()) {
|
|
529
|
-
deleteOps.delete(ref);
|
|
530
|
-
}
|
|
531
|
-
const claimedSecondaries = new Set();
|
|
532
|
-
for (const mergeOp of mergeOps.values()) {
|
|
533
|
-
deleteOps.delete(mergeOp.primary);
|
|
534
|
-
mergeOp.secondaries = mergeOp.secondaries.filter((sec) => {
|
|
535
|
-
if (mergeOps.has(sec)) {
|
|
536
|
-
warnings.push(`Merge: secondary ${sec} is also a merge primary — removing from secondary list to avoid double-count.`);
|
|
537
|
-
return false;
|
|
538
|
-
}
|
|
539
|
-
if (claimedSecondaries.has(sec)) {
|
|
540
|
-
warnings.push(`Merge: secondary ${sec} appears in multiple merge ops — retaining in first op only.`);
|
|
541
|
-
return false;
|
|
542
|
-
}
|
|
543
|
-
claimedSecondaries.add(sec);
|
|
544
|
-
deleteOps.delete(sec);
|
|
545
|
-
return true;
|
|
546
|
-
});
|
|
547
|
-
}
|
|
548
|
-
// C-2 / #381: promote ops are ordered BEFORE merge ops so that the
|
|
549
|
-
// human-gated proposal queue entry is created before any destructive merge.
|
|
550
|
-
// Phase B processes ops in array order, so promote executes first.
|
|
551
|
-
const ops = [
|
|
552
|
-
...promoteOps.values(),
|
|
553
|
-
...mergeOps.values(),
|
|
554
|
-
...deleteOps.values(),
|
|
555
|
-
...contradictOps.values(),
|
|
556
|
-
];
|
|
557
|
-
return { ops, warnings };
|
|
558
|
-
}
|
|
559
295
|
function getJournalPath(stashDir) {
|
|
560
296
|
return path.join(stashDir, ".akm", "consolidate-journal.json");
|
|
561
297
|
}
|
|
@@ -672,6 +408,28 @@ function backupFile(filePath, backupDir, name) {
|
|
|
672
408
|
// best-effort
|
|
673
409
|
}
|
|
674
410
|
}
|
|
411
|
+
// ── WS-3b: Generation frontmatter injection ───────────────────────────────────
|
|
412
|
+
/**
|
|
413
|
+
* Inject `generation` (and optionally `source_refs`) into merged content.
|
|
414
|
+
* generation = max(sourceGenerations) + 1.
|
|
415
|
+
* Fails open — returns original content if frontmatter can't be parsed.
|
|
416
|
+
*/
|
|
417
|
+
function injectGenerationFrontmatter(mergedContent, sourceGenerations, allParticipants) {
|
|
418
|
+
try {
|
|
419
|
+
const parsed = parseFrontmatter(mergedContent);
|
|
420
|
+
const updatedFm = {
|
|
421
|
+
...parsed.data,
|
|
422
|
+
generation: computeMergedGeneration(sourceGenerations),
|
|
423
|
+
};
|
|
424
|
+
if (!updatedFm.source_refs) {
|
|
425
|
+
updatedFm.source_refs = allParticipants;
|
|
426
|
+
}
|
|
427
|
+
return assembleAssetFromString(serializeFrontmatter(updatedFm), parsed.content);
|
|
428
|
+
}
|
|
429
|
+
catch {
|
|
430
|
+
return mergedContent; // fail open
|
|
431
|
+
}
|
|
432
|
+
}
|
|
675
433
|
// ── Archive helper (P1-B: soft-invalidation) ─────────────────────────────────
|
|
676
434
|
/**
|
|
677
435
|
* Move a memory asset to `.akm/archive/` with `status: superseded` frontmatter
|
|
@@ -752,6 +510,25 @@ function resolveConsolidateLlmConfig(config) {
|
|
|
752
510
|
// fall back to the default LLM profile rather than disabling the pass.
|
|
753
511
|
return getDefaultLlmConfig(config);
|
|
754
512
|
}
|
|
513
|
+
// ── Judged-state cache (#581) ────────────────────────────────────────────────
|
|
514
|
+
/**
|
|
515
|
+
* Stable content hash for a memory file used by the judged-state cache (#581)
|
|
516
|
+
* and the body-embedding cache (WS-3a). Uses `cacheHash` from dedup.ts:
|
|
517
|
+
* sha256 of the case-preserving stripped body. Two memories that differ only
|
|
518
|
+
* in frontmatter (`updated:`, `inferenceProcessed:`) hash identically, so a
|
|
519
|
+
* cosmetic frontmatter touch never forces a needless re-judge — only a body
|
|
520
|
+
* change does. Returns `undefined` on any read/parse error so callers fail
|
|
521
|
+
* open (treat the memory as un-cached → it stays in the LLM pool).
|
|
522
|
+
*/
|
|
523
|
+
function computeMemoryContentHash(filePath) {
|
|
524
|
+
try {
|
|
525
|
+
const raw = fs.readFileSync(filePath, "utf8");
|
|
526
|
+
return cacheHash(raw);
|
|
527
|
+
}
|
|
528
|
+
catch {
|
|
529
|
+
return undefined;
|
|
530
|
+
}
|
|
531
|
+
}
|
|
755
532
|
// ── Main entry point ─────────────────────────────────────────────────────────
|
|
756
533
|
export async function akmConsolidate(opts = {}) {
|
|
757
534
|
const startMs = Date.now();
|
|
@@ -780,6 +557,27 @@ export async function akmConsolidate(opts = {}) {
|
|
|
780
557
|
}
|
|
781
558
|
const warnings = [];
|
|
782
559
|
checkForIncompleteJournal(stashDir, opts.recoveryMode ?? "abort", warnings);
|
|
560
|
+
// WS-3a: open one state.db handle shared by the body-embedding cache (dedup
|
|
561
|
+
// + cluster) and the judged-state cache. All callers in the function body
|
|
562
|
+
// receive this handle; it is closed in the `finally` block below.
|
|
563
|
+
// Fail-open: any open error leaves it `undefined` and all cache paths skip.
|
|
564
|
+
let sharedStateDb;
|
|
565
|
+
try {
|
|
566
|
+
sharedStateDb = openStateDatabase();
|
|
567
|
+
}
|
|
568
|
+
catch {
|
|
569
|
+
// State DB unavailable → skip the embedding cache for this run.
|
|
570
|
+
}
|
|
571
|
+
try {
|
|
572
|
+
return await akmConsolidateInner(opts, config, stashDir, startMs, sourceRun, warnings, sharedStateDb);
|
|
573
|
+
}
|
|
574
|
+
finally {
|
|
575
|
+
sharedStateDb?.close();
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
// Inner implementation — all early-return paths are here; sharedStateDb is
|
|
579
|
+
// closed by the outer finally in `akmConsolidate`.
|
|
580
|
+
async function akmConsolidateInner(opts, config, stashDir, startMs, sourceRun, warnings, sharedStateDb) {
|
|
783
581
|
let memories = loadMemoriesForSource(opts.target, stashDir, warnings);
|
|
784
582
|
// Pre-flight: filter out stale DB entries whose files no longer exist on
|
|
785
583
|
// disk. Without this, memories deleted by a prior run (but not yet
|
|
@@ -791,6 +589,80 @@ export async function akmConsolidate(opts = {}) {
|
|
|
791
589
|
warnings.push(`Pre-flight: filtered ${staleCount} stale DB entr${staleCount === 1 ? "y" : "ies"} (file absent on disk) from memory pool before chunking.`);
|
|
792
590
|
}
|
|
793
591
|
memories = memories.filter((m) => fs.existsSync(m.filePath));
|
|
592
|
+
// ── WS-3b Step 0a: Homeostatic demotion ────────────────────────────────────
|
|
593
|
+
// DEFAULT OFF. Before any LLM merge, demote retrievalSalience in state.db
|
|
594
|
+
// for stale/low-value assets so the merge pool is bounded and high-SNR.
|
|
595
|
+
// Demotion is state.db-only (file content untouched); re-promotable on
|
|
596
|
+
// re-retrieval. Only fires when `homeostaticDemotion.enabled === true`.
|
|
597
|
+
const homeostaticConfig = config.profiles?.improve?.default?.processes?.consolidate?.homeostaticDemotion ?? {};
|
|
598
|
+
if (homeostaticConfig.enabled && sharedStateDb) {
|
|
599
|
+
const demotionResult = runHomeostaticDemotion(sharedStateDb, homeostaticConfig);
|
|
600
|
+
if (demotionResult.demoted > 0) {
|
|
601
|
+
warnings.push(`Homeostatic demotion: demoted retrievalSalience for ${demotionResult.demoted} stale asset(s) before merge pool assembly.`);
|
|
602
|
+
}
|
|
603
|
+
warnings.push(...demotionResult.warnings);
|
|
604
|
+
}
|
|
605
|
+
// ── WS-3b Step 0c: Filter hot-probation assets from LLM merge pool ─────────
|
|
606
|
+
// Hot-probation assets (system-generated, not yet graduated from intake pass)
|
|
607
|
+
// are processed by the dedup pre-pass but excluded from the LLM clustering.
|
|
608
|
+
// This prevents noisy extractions from polluting LLM context. The dedup pass
|
|
609
|
+
// below still runs against them so they're cleaned up deterministically.
|
|
610
|
+
// DEFAULT OFF — only active when `processes.extract.hotProbation.enabled === true`
|
|
611
|
+
// (the flag that causes extract to tag new extractions as hot-probation).
|
|
612
|
+
// Without that flag no assets will ever carry the hot-probation marker, so
|
|
613
|
+
// running the filter loop would be pure unnecessary I/O over the full corpus.
|
|
614
|
+
const hotProbationEnabled = config.profiles?.improve?.default?.processes?.extract?.hotProbation
|
|
615
|
+
?.enabled === true;
|
|
616
|
+
let hotProbationCount = 0;
|
|
617
|
+
if (hotProbationEnabled) {
|
|
618
|
+
const hotProbationMemories = [];
|
|
619
|
+
const nonProbationMemories = [];
|
|
620
|
+
for (const m of memories) {
|
|
621
|
+
try {
|
|
622
|
+
const raw = fs.readFileSync(m.filePath, "utf8");
|
|
623
|
+
const parsed = parseFrontmatter(raw);
|
|
624
|
+
if (shouldSkipHotProbationInLlm(parsed.data)) {
|
|
625
|
+
hotProbationMemories.push(m);
|
|
626
|
+
hotProbationCount++;
|
|
627
|
+
}
|
|
628
|
+
else {
|
|
629
|
+
nonProbationMemories.push(m);
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
catch {
|
|
633
|
+
nonProbationMemories.push(m); // fail open
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
if (hotProbationCount > 0) {
|
|
637
|
+
warnings.push(`Hot-probation: ${hotProbationCount} hot-probation asset(s) routed to dedup-only pass (excluded from LLM merge pool).`);
|
|
638
|
+
memories = nonProbationMemories;
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
// ── Deterministic dedup pre-pass (#617) ─────────────────────────────────────
|
|
642
|
+
// Cheap, no-LLM fast path that collapses the obvious near-duplicates
|
|
643
|
+
// (`.derived` ↔ origin pairs + content twins) BEFORE the embedding-clustered
|
|
644
|
+
// LLM consolidation. DEFAULT OFF — when `dedup.enabled !== true` this is a
|
|
645
|
+
// no-op and the pass behaves byte-identically to today. Collapsed variants
|
|
646
|
+
// are pruned from the LLM pool so the model only ever sees genuinely
|
|
647
|
+
// distinct-but-related memories. Each dropped variant is archived (soft
|
|
648
|
+
// invalidation) before deletion, matching the LLM merge path.
|
|
649
|
+
// Dry-run never mutates the filesystem, so the dedup pre-pass is skipped
|
|
650
|
+
// entirely under `--dry-run` (the LLM plan preview below is unaffected).
|
|
651
|
+
let dedupCollapsed = 0;
|
|
652
|
+
if (opts.dedup?.enabled && !opts.dryRun) {
|
|
653
|
+
const dedupTimestamp = timestampForFilename();
|
|
654
|
+
const dedupResult = await runDeterministicDedup(stashDir, opts.dedup, config, (variantFilePath, variantName) => {
|
|
655
|
+
archiveMemory(variantFilePath, stashDir, `memory:${variantName}`, "collapsed by deterministic dedup pre-pass", -1, undefined, warnings);
|
|
656
|
+
backupFile(variantFilePath, getBackupDir(stashDir, dedupTimestamp), variantName);
|
|
657
|
+
}, opts.signal, sharedStateDb);
|
|
658
|
+
dedupCollapsed = dedupResult.collapsed;
|
|
659
|
+
warnings.push(...dedupResult.warnings);
|
|
660
|
+
if (dedupResult.consumedRefs.length > 0) {
|
|
661
|
+
const consumed = new Set(dedupResult.consumedRefs);
|
|
662
|
+
memories = memories.filter((m) => !consumed.has(`memory:${m.name}`));
|
|
663
|
+
warnings.push(`Deterministic dedup: collapsed ${dedupResult.collapsed} near-duplicate memor${dedupResult.collapsed === 1 ? "y" : "ies"} (no LLM) before chunking.`);
|
|
664
|
+
}
|
|
665
|
+
}
|
|
794
666
|
if (memories.length === 0) {
|
|
795
667
|
return {
|
|
796
668
|
schemaVersion: 1,
|
|
@@ -801,7 +673,10 @@ export async function akmConsolidate(opts = {}) {
|
|
|
801
673
|
target: opts.target ?? stashDir,
|
|
802
674
|
processed: 0,
|
|
803
675
|
merged: 0,
|
|
804
|
-
|
|
676
|
+
// #617: the deterministic dedup pre-pass may have emptied the pool by
|
|
677
|
+
// collapsing every remaining memory into a canonical. Surface those
|
|
678
|
+
// collapses in `deleted` so the run reports the work it actually did.
|
|
679
|
+
deleted: dedupCollapsed,
|
|
805
680
|
promoted: [],
|
|
806
681
|
contradicted: 0,
|
|
807
682
|
warnings,
|
|
@@ -809,7 +684,7 @@ export async function akmConsolidate(opts = {}) {
|
|
|
809
684
|
};
|
|
810
685
|
}
|
|
811
686
|
if (opts.incrementalSince) {
|
|
812
|
-
memories = narrowToIncrementalCandidates(memories, opts.incrementalSince, warnings);
|
|
687
|
+
memories = narrowToIncrementalCandidates(memories, opts.incrementalSince, warnings, opts.neighborsPerChanged);
|
|
813
688
|
if (memories.length === 0) {
|
|
814
689
|
return {
|
|
815
690
|
schemaVersion: 1,
|
|
@@ -828,6 +703,109 @@ export async function akmConsolidate(opts = {}) {
|
|
|
828
703
|
};
|
|
829
704
|
}
|
|
830
705
|
}
|
|
706
|
+
// WS-5 perf telemetry accumulators. These are collected throughout the run and
|
|
707
|
+
// merged into `perfTelemetry` on the final ConsolidateResult.
|
|
708
|
+
// `dedupPoolSize` = memories entering judgedCache narrowing (after dedup+incremental+limit).
|
|
709
|
+
// `judgedCacheSkipped` = memories skipped by the cache.
|
|
710
|
+
// `llmPoolSize` = memories actually sent to the LLM.
|
|
711
|
+
// `embedMs/cacheHits/cacheMisses` = accumulated from clusterMemoriesBySimilarity.
|
|
712
|
+
const perfMs = { dedupPoolSize: memories.length, judgedCacheSkipped: 0 };
|
|
713
|
+
// ── Judged-state cache narrowing (#581) ─────────────────────────────────────
|
|
714
|
+
// DEFAULT OFF. When enabled, skip every memory whose current content hash
|
|
715
|
+
// equals the hash recorded the last time the consolidate LLM judged it
|
|
716
|
+
// (judged-unchanged → no re-judge). This converts coverage from O(window) to
|
|
717
|
+
// O(changed/new) so one run can sweep the whole corpus while the LLM only
|
|
718
|
+
// sees genuinely new/changed memories. `currentHashByName` is populated for
|
|
719
|
+
// EVERY surviving memory (whether or not the cache is on) so the post-LLM
|
|
720
|
+
// recording step can upsert judged state without re-reading the files; when
|
|
721
|
+
// the cache is off it stays empty and the recording step is a no-op.
|
|
722
|
+
const judgedCacheEnabled = opts.judgedCache?.enabled !== false;
|
|
723
|
+
const currentHashByName = new Map();
|
|
724
|
+
if (judgedCacheEnabled) {
|
|
725
|
+
for (const m of memories) {
|
|
726
|
+
const h = computeMemoryContentHash(m.filePath);
|
|
727
|
+
if (h !== undefined)
|
|
728
|
+
currentHashByName.set(m.name, h);
|
|
729
|
+
}
|
|
730
|
+
let cachedMap = new Map();
|
|
731
|
+
{
|
|
732
|
+
// Use the shared state.db handle if available; open a local one otherwise.
|
|
733
|
+
const dbForJudged = sharedStateDb;
|
|
734
|
+
if (dbForJudged) {
|
|
735
|
+
try {
|
|
736
|
+
cachedMap = getConsolidationJudgedMap(dbForJudged, memories.map((m) => `memory:${m.name}`));
|
|
737
|
+
}
|
|
738
|
+
catch {
|
|
739
|
+
cachedMap = new Map();
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
else {
|
|
743
|
+
try {
|
|
744
|
+
cachedMap = withStateDb((localDb) => getConsolidationJudgedMap(localDb, memories.map((m) => `memory:${m.name}`)));
|
|
745
|
+
}
|
|
746
|
+
catch {
|
|
747
|
+
// State DB unavailable → fail open: judge the full pool this run.
|
|
748
|
+
cachedMap = new Map();
|
|
749
|
+
}
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
const beforeCount = memories.length;
|
|
753
|
+
memories = memories.filter((m) => {
|
|
754
|
+
const cur = currentHashByName.get(m.name);
|
|
755
|
+
// No readable hash → keep (fail open; let the LLM judge it).
|
|
756
|
+
if (cur === undefined)
|
|
757
|
+
return true;
|
|
758
|
+
const cached = cachedMap.get(`memory:${m.name}`);
|
|
759
|
+
// Skip only when previously judged AND content is byte-identical since.
|
|
760
|
+
return !(cached !== undefined && cached.content_hash === cur);
|
|
761
|
+
});
|
|
762
|
+
const skipped = beforeCount - memories.length;
|
|
763
|
+
perfMs.judgedCacheSkipped = skipped; // WS-5 perf telemetry
|
|
764
|
+
if (skipped > 0) {
|
|
765
|
+
warnings.push(`Judged-state cache: skipped ${skipped} memor${skipped === 1 ? "y" : "ies"} judged-unchanged (no LLM); ${memories.length} remain for judging.`);
|
|
766
|
+
}
|
|
767
|
+
if (memories.length === 0) {
|
|
768
|
+
return {
|
|
769
|
+
schemaVersion: 1,
|
|
770
|
+
ok: true,
|
|
771
|
+
shape: "consolidate-result",
|
|
772
|
+
dryRun: opts.dryRun ?? false,
|
|
773
|
+
previewOnly: false,
|
|
774
|
+
target: opts.target ?? stashDir,
|
|
775
|
+
processed: 0,
|
|
776
|
+
merged: 0,
|
|
777
|
+
deleted: dedupCollapsed,
|
|
778
|
+
promoted: [],
|
|
779
|
+
contradicted: 0,
|
|
780
|
+
warnings,
|
|
781
|
+
durationMs: Date.now() - startMs,
|
|
782
|
+
};
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
if (opts.limit === undefined && memories.length > 150) {
|
|
786
|
+
warnings.push(`Consolidation: pool has ${memories.length} memories and no limit is set. Consider adding a limit to your consolidate config to prevent timeouts on slow LLM endpoints.`);
|
|
787
|
+
}
|
|
788
|
+
if (opts.limit !== undefined && memories.length > opts.limit) {
|
|
789
|
+
// Order oldest-modified-first before capping so the limit selects the
|
|
790
|
+
// stalest memories rather than a fixed head of the (rowid-ordered) DB
|
|
791
|
+
// query. Consolidation rewrites surviving files, bumping their mtime, so
|
|
792
|
+
// processed memories drift to the back of the queue and the cap rotates
|
|
793
|
+
// across the whole corpus over successive runs instead of revisiting the
|
|
794
|
+
// same slice every time. Fail-open to 0 (front of queue) when a file can
|
|
795
|
+
// no longer be stat'd.
|
|
796
|
+
const mtimeOf = (m) => {
|
|
797
|
+
try {
|
|
798
|
+
return fs.statSync(m.filePath).mtimeMs;
|
|
799
|
+
}
|
|
800
|
+
catch {
|
|
801
|
+
return 0;
|
|
802
|
+
}
|
|
803
|
+
};
|
|
804
|
+
const mtimeCache = new Map(memories.map((m) => [m.filePath, mtimeOf(m)]));
|
|
805
|
+
memories = [...memories].sort((a, b) => (mtimeCache.get(a.filePath) ?? 0) - (mtimeCache.get(b.filePath) ?? 0));
|
|
806
|
+
warnings.push(`Consolidation: pool capped at ${opts.limit} of ${memories.length} memories (limit option, oldest-modified first).`);
|
|
807
|
+
memories = memories.slice(0, opts.limit);
|
|
808
|
+
}
|
|
831
809
|
// Consolidation always uses the HTTP LLM client directly — never the agent
|
|
832
810
|
// CLI. The agent CLI is for interactive agent sessions (reflect, propose);
|
|
833
811
|
// structured JSON generation works better and faster via HTTP.
|
|
@@ -851,16 +829,64 @@ export async function akmConsolidate(opts = {}) {
|
|
|
851
829
|
const chunkSize = computeSafeChunkSize(modelContextLength, bodyTruncation, opts.maxChunkSize);
|
|
852
830
|
// -- Phase A: plan generation -----------------------------------------------
|
|
853
831
|
const sourceName = opts.target ?? stashDir;
|
|
832
|
+
// WS-5: capture llmPoolSize = memories entering the LLM (after all filtering).
|
|
833
|
+
const llmPoolSize = memories.length;
|
|
854
834
|
// C-1 / #380: Pre-cluster memories by embedding similarity before chunking.
|
|
855
835
|
// This ensures that semantically similar memories land in the same LLM
|
|
856
836
|
// context window, allowing the model to detect and merge duplicates that
|
|
857
837
|
// would otherwise be split across chunks and survive indefinitely.
|
|
858
838
|
// mem0 arXiv:2504.19413, A-MEM arXiv:2502.12110.
|
|
859
839
|
// Fails open: if embeddings are unavailable or fail, original order is used.
|
|
860
|
-
const clusteredMemories = await clusterMemoriesBySimilarity(memories, config);
|
|
840
|
+
const { ordered: clusteredMemories, embedTelemetry } = await clusterMemoriesBySimilarity(memories, config, sharedStateDb);
|
|
841
|
+
// WS-3b Anti-collapse step 8c: inject random (non-similar) clusters.
|
|
842
|
+
// A small fraction (default 5%) of the pool is shuffled into random positions
|
|
843
|
+
// so the pipeline isn't PURELY similarity-driven. This prevents rich-get-richer
|
|
844
|
+
// entrenchment where only the most-retrieved assets ever get consolidated.
|
|
845
|
+
// DEFAULT OFF — gated on antiCollapse.enabled.
|
|
846
|
+
let finalClusteredMemories = clusteredMemories;
|
|
847
|
+
{
|
|
848
|
+
const antiCollapseForCluster = config.profiles?.improve?.default?.processes?.consolidate?.antiCollapse ?? {};
|
|
849
|
+
if (antiCollapseForCluster.enabled && clusteredMemories.length > 2) {
|
|
850
|
+
const fraction = antiCollapseForCluster.randomClusterFraction ?? 0.05;
|
|
851
|
+
const randomCount = Math.max(1, Math.floor(clusteredMemories.length * fraction));
|
|
852
|
+
// Pick `randomCount` positions to inject random (un-clustered) members.
|
|
853
|
+
// Use a seeded-ish shuffle: sort by hash of the name so it's deterministic
|
|
854
|
+
// per run but not strictly similarity-driven.
|
|
855
|
+
const shuffled = [...clusteredMemories].sort((a, b) => {
|
|
856
|
+
// Deterministic shuffle: compare sha256-ish (use name hash as proxy).
|
|
857
|
+
const ha = a.name.split("").reduce((acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0, 0);
|
|
858
|
+
const hb = b.name.split("").reduce((acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0, 0);
|
|
859
|
+
return ha - hb;
|
|
860
|
+
});
|
|
861
|
+
const randomSlice = shuffled.slice(0, randomCount);
|
|
862
|
+
const randomSet = new Set(randomSlice.map((m) => m.name));
|
|
863
|
+
// Insert random members at intervals through the clustered sequence.
|
|
864
|
+
const withRandom = [];
|
|
865
|
+
const interval = Math.max(2, Math.floor(clusteredMemories.length / randomCount));
|
|
866
|
+
let randomIdx = 0;
|
|
867
|
+
for (let i = 0; i < clusteredMemories.length; i++) {
|
|
868
|
+
const m = clusteredMemories[i];
|
|
869
|
+
if (m && !randomSet.has(m.name))
|
|
870
|
+
withRandom.push(m);
|
|
871
|
+
if (i > 0 && i % interval === 0 && randomIdx < randomSlice.length) {
|
|
872
|
+
const r = randomSlice[randomIdx++];
|
|
873
|
+
if (r)
|
|
874
|
+
withRandom.push(r);
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
// Append any remaining random members not yet inserted.
|
|
878
|
+
while (randomIdx < randomSlice.length) {
|
|
879
|
+
const r = randomSlice[randomIdx++];
|
|
880
|
+
if (r)
|
|
881
|
+
withRandom.push(r);
|
|
882
|
+
}
|
|
883
|
+
finalClusteredMemories = withRandom;
|
|
884
|
+
warnings.push(`Anti-collapse: injected ${randomCount} random (non-similarity-driven) cluster member(s) into consolidation pool (fraction=${fraction}).`);
|
|
885
|
+
}
|
|
886
|
+
}
|
|
861
887
|
const chunks = [];
|
|
862
|
-
for (let i = 0; i <
|
|
863
|
-
chunks.push(
|
|
888
|
+
for (let i = 0; i < finalClusteredMemories.length; i += chunkSize) {
|
|
889
|
+
chunks.push(finalClusteredMemories.slice(i, i + chunkSize));
|
|
864
890
|
}
|
|
865
891
|
// 2026-05-27 prompt-context fix: precompute body-hashes of pending
|
|
866
892
|
// consolidate proposals once, so the per-chunk prompt can annotate
|
|
@@ -869,8 +895,46 @@ export async function akmConsolidate(opts = {}) {
|
|
|
869
895
|
// 4h on this user's stack. See
|
|
870
896
|
// /tmp/akm-health-investigations/tuning-reasons-investigation.md §Q3.
|
|
871
897
|
const pendingProposalBodyHashes = loadPendingConsolidateProposalHashes(stashDir);
|
|
898
|
+
// ── Cold-start budget estimation ─────────────────────────────────────────────
|
|
899
|
+
// Estimate wall-clock cost BEFORE issuing any LLM calls. When a signal is
|
|
900
|
+
// provided and the estimated cost exceeds ~60% of the remaining budget we
|
|
901
|
+
// auto-reduce the pool and log the reduction so the run never starts work
|
|
902
|
+
// it cannot finish (avoiding SIGTERM mid-LLM-call).
|
|
903
|
+
//
|
|
904
|
+
// Formula: chunks.length × p90_chunk_seconds. The p90 comes from
|
|
905
|
+
// `opts.p90ChunkSecondsDefault` (caller-supplied, typically from the profile
|
|
906
|
+
// config); absent = 30 s (conservative default matching a medium local LLM).
|
|
907
|
+
//
|
|
908
|
+
// "Remaining budget" is read from a custom property on the AbortSignal if
|
|
909
|
+
// the caller (improve.ts) has attached one. Without it no auto-reduction
|
|
910
|
+
// fires but the check is still cheap to run.
|
|
911
|
+
if (chunks.length > 10 && opts.signal) {
|
|
912
|
+
const p90Chunk = opts.p90ChunkSecondsDefault ?? 30;
|
|
913
|
+
const estimatedSeconds = chunks.length * p90Chunk;
|
|
914
|
+
// remainingBudgetMs is a non-standard extension set by improve.ts when it
|
|
915
|
+
// creates the budget AbortController. Undefined = no budget information.
|
|
916
|
+
const budgetMs = opts.signal.remainingBudgetMs;
|
|
917
|
+
if (budgetMs !== undefined && budgetMs > 0) {
|
|
918
|
+
const remainingSeconds = budgetMs / 1000;
|
|
919
|
+
if (estimatedSeconds > remainingSeconds * 0.6) {
|
|
920
|
+
const safeCaps = Math.max(1, Math.floor((remainingSeconds * 0.6) / p90Chunk));
|
|
921
|
+
const removedChunks = chunks.length - safeCaps;
|
|
922
|
+
if (removedChunks > 0) {
|
|
923
|
+
const msg = `[consolidate] cold-start budget: estimated ${estimatedSeconds.toFixed(0)}s > 60% of remaining ${remainingSeconds.toFixed(0)}s; ` +
|
|
924
|
+
`reducing pool from ${chunks.length} to ${safeCaps} chunks (${removedChunks} deferred to next run).`;
|
|
925
|
+
warn(msg);
|
|
926
|
+
warnings.push(msg);
|
|
927
|
+
chunks.splice(safeCaps);
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
}
|
|
872
932
|
warn(`[consolidate] ${memories.length} memories / ${chunks.length} chunk(s) / chunk_size=${chunkSize}` +
|
|
873
933
|
` / pending-proposal hashes: ${pendingProposalBodyHashes.size}`);
|
|
934
|
+
// Consolidate output merges memories (non-wiki) → stash authoring standards.
|
|
935
|
+
// Resolved ONCE per run and passed to each chunk prompt (facts not re-read
|
|
936
|
+
// per chunk).
|
|
937
|
+
const standardsContext = resolveStashStandards(stashDir);
|
|
874
938
|
const chunkOpsArrays = [];
|
|
875
939
|
// Structured skip-reason histogram (2026-05-26): every deterministic
|
|
876
940
|
// post-LLM op rejection site below also calls `pushSkipReason` so the
|
|
@@ -907,6 +971,13 @@ export async function akmConsolidate(opts = {}) {
|
|
|
907
971
|
// judgedNoAction tracks memories the LLM saw inside a chunk but proposed
|
|
908
972
|
// no op for. Computed per chunk as `chunk.length − unique(targetRefs in ops)`.
|
|
909
973
|
let judgedNoAction = 0;
|
|
974
|
+
// Judged-state cache (#581): coarse outcome per memory NAME the LLM actually
|
|
975
|
+
// judged in a successfully-parsed chunk this run. "actioned" = an op targeted
|
|
976
|
+
// it; "no_action" = the LLM saw it and proposed nothing. Populated only when
|
|
977
|
+
// the cache is enabled (otherwise it stays empty and the post-loop recording
|
|
978
|
+
// step is a no-op). Memories in failed/aborted chunks are NOT recorded, so a
|
|
979
|
+
// transient LLM failure never poisons the cache into skipping them next run.
|
|
980
|
+
const judgedOutcomeByName = new Map();
|
|
910
981
|
// 2026-05-27 cross-chunk double-count fix: refs that contributed to
|
|
911
982
|
// judgedNoAction in their own chunk. When a different chunk's op references
|
|
912
983
|
// one of these as a secondary and that op later fails, the ref would land
|
|
@@ -931,6 +1002,19 @@ export async function akmConsolidate(opts = {}) {
|
|
|
931
1002
|
const ABORT_MIN_CHUNKS = 4;
|
|
932
1003
|
const ABORT_FAILURE_RATE = 0.5;
|
|
933
1004
|
for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
|
|
1005
|
+
// Budget-signal check: break cleanly before the next LLM call if the
|
|
1006
|
+
// caller's budget has been exhausted. Commits work done so far.
|
|
1007
|
+
if (opts.signal?.aborted) {
|
|
1008
|
+
const skipped = chunks.length - chunkIdx;
|
|
1009
|
+
const msg = `[consolidate] budget signal aborted before chunk ${chunkIdx + 1}/${chunks.length}; ${skipped} chunk(s) not processed (partial_timeout — work done so far committed).`;
|
|
1010
|
+
warn(msg);
|
|
1011
|
+
warnings.push(msg);
|
|
1012
|
+
// Account for memories in unprocessed chunks.
|
|
1013
|
+
for (let i = chunkIdx; i < chunks.length; i++) {
|
|
1014
|
+
failedChunkMemories += chunks[i].length;
|
|
1015
|
+
}
|
|
1016
|
+
break;
|
|
1017
|
+
}
|
|
934
1018
|
// Abort if failure rate >= 50% over at least 4 processed chunks.
|
|
935
1019
|
if (totalChunksProcessed >= ABORT_MIN_CHUNKS) {
|
|
936
1020
|
const failureRate = totalChunksFailed / totalChunksProcessed;
|
|
@@ -968,7 +1052,7 @@ export async function akmConsolidate(opts = {}) {
|
|
|
968
1052
|
continue;
|
|
969
1053
|
}
|
|
970
1054
|
warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length} (${chunk.length} memories) …`);
|
|
971
|
-
const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation, pendingProposalBodyHashes);
|
|
1055
|
+
const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation, pendingProposalBodyHashes, standardsContext);
|
|
972
1056
|
let raw = await tryLlmFeature("memory_consolidation", config, async () => {
|
|
973
1057
|
if (!llmConfig)
|
|
974
1058
|
return { ok: false, error: "No LLM configured for consolidation" };
|
|
@@ -1075,11 +1159,59 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1075
1159
|
if (!targetRefs.has(memRef)) {
|
|
1076
1160
|
chunkNoAction++;
|
|
1077
1161
|
judgedNoActionRefs.add(memRef);
|
|
1162
|
+
// Judged-state cache (#581): the LLM saw this memory and proposed
|
|
1163
|
+
// nothing → record judged-unchanged so the next run can skip it.
|
|
1164
|
+
if (judgedCacheEnabled)
|
|
1165
|
+
judgedOutcomeByName.set(m.name, "no_action");
|
|
1166
|
+
}
|
|
1167
|
+
else if (judgedCacheEnabled) {
|
|
1168
|
+
// An op targeted this memory → it was judged + actioned.
|
|
1169
|
+
judgedOutcomeByName.set(m.name, "actioned");
|
|
1078
1170
|
}
|
|
1079
1171
|
}
|
|
1080
1172
|
judgedNoAction += chunkNoAction;
|
|
1081
1173
|
chunkOpsArrays.push(ops);
|
|
1082
1174
|
}
|
|
1175
|
+
// ── Judged-state cache recording (#581) ─────────────────────────────────────
|
|
1176
|
+
// Persist judged state for every memory the LLM actually judged this run so
|
|
1177
|
+
// the next run can skip the unchanged ones. Keyed by current content hash so
|
|
1178
|
+
// a later body edit (different hash) re-enters the LLM pool. DEFAULT OFF and
|
|
1179
|
+
// skipped under --dry-run (dry-run mutates nothing). Failed/aborted chunks
|
|
1180
|
+
// contributed no entries to `judgedOutcomeByName`, so a transient LLM outage
|
|
1181
|
+
// never caches a memory as judged.
|
|
1182
|
+
if (judgedCacheEnabled && !opts.dryRun && judgedOutcomeByName.size > 0) {
|
|
1183
|
+
// Use the shared state.db handle; open a local one as fallback.
|
|
1184
|
+
const doRecord = (db) => {
|
|
1185
|
+
const judgedAt = new Date(startMs).toISOString();
|
|
1186
|
+
for (const [name, outcome] of judgedOutcomeByName) {
|
|
1187
|
+
const hash = currentHashByName.get(name);
|
|
1188
|
+
if (hash === undefined)
|
|
1189
|
+
continue;
|
|
1190
|
+
upsertConsolidationJudged(db, {
|
|
1191
|
+
entryKey: `memory:${name}`,
|
|
1192
|
+
contentHash: hash,
|
|
1193
|
+
judgedAt,
|
|
1194
|
+
outcome,
|
|
1195
|
+
});
|
|
1196
|
+
}
|
|
1197
|
+
};
|
|
1198
|
+
if (sharedStateDb) {
|
|
1199
|
+
try {
|
|
1200
|
+
doRecord(sharedStateDb);
|
|
1201
|
+
}
|
|
1202
|
+
catch (e) {
|
|
1203
|
+
warnings.push(`Judged-state cache: failed to record judged state: ${String(e)}`);
|
|
1204
|
+
}
|
|
1205
|
+
}
|
|
1206
|
+
else {
|
|
1207
|
+
try {
|
|
1208
|
+
withStateDb((localDb) => doRecord(localDb));
|
|
1209
|
+
}
|
|
1210
|
+
catch (e) {
|
|
1211
|
+
warnings.push(`Judged-state cache: failed to record judged state: ${String(e)}`);
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1083
1215
|
// Build the known-refs set from the already-filtered memory pool so
|
|
1084
1216
|
// mergePlans() can reject LLM-hallucinated primary refs before execution.
|
|
1085
1217
|
const knownRefs = new Set(memories.map((m) => `memory:${m.name}`));
|
|
@@ -1266,7 +1398,7 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1266
1398
|
emitMergeFailureSkips(mergeResult.error);
|
|
1267
1399
|
continue;
|
|
1268
1400
|
}
|
|
1269
|
-
|
|
1401
|
+
let mergedContent = mergeResult.content;
|
|
1270
1402
|
// Validate frontmatter of merged content — must have a `---` block
|
|
1271
1403
|
// with at minimum a `description` field. We parse via the hand-rolled
|
|
1272
1404
|
// parser (cheap) AND require non-empty description. This guards against
|
|
@@ -1320,6 +1452,62 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1320
1452
|
emitMergeFailureSkips("merge_participant_blocked");
|
|
1321
1453
|
continue;
|
|
1322
1454
|
}
|
|
1455
|
+
// WS-3b: Anti-collapse generation guard (step 8a).
|
|
1456
|
+
// DEFAULT OFF. When antiCollapse.enabled, refuse to merge two assets both
|
|
1457
|
+
// above generation N (default 2). This prevents the pipeline from
|
|
1458
|
+
// building ever-deeper LLM-merged trees that lose the source fidelity
|
|
1459
|
+
// of the original episodes.
|
|
1460
|
+
const antiCollapseConfig = config.profiles?.improve?.default?.processes?.consolidate?.antiCollapse ??
|
|
1461
|
+
{};
|
|
1462
|
+
if (antiCollapseConfig.enabled) {
|
|
1463
|
+
const allParticipants = [op.primary, ...op.secondaries];
|
|
1464
|
+
const sourceGenerations = allParticipants.map((ref) => {
|
|
1465
|
+
const e = memoryByRef.get(ref);
|
|
1466
|
+
if (!e)
|
|
1467
|
+
return 0;
|
|
1468
|
+
try {
|
|
1469
|
+
const raw = fs.readFileSync(e.filePath, "utf8");
|
|
1470
|
+
const parsed = parseFrontmatter(raw);
|
|
1471
|
+
return readAssetGeneration(parsed.data);
|
|
1472
|
+
}
|
|
1473
|
+
catch {
|
|
1474
|
+
return 0;
|
|
1475
|
+
}
|
|
1476
|
+
});
|
|
1477
|
+
const generationCheck = checkGenerationGuard(sourceGenerations, antiCollapseConfig);
|
|
1478
|
+
if (generationCheck.refused) {
|
|
1479
|
+
warnings.push(`Merge: ${generationCheck.reason}`);
|
|
1480
|
+
emitMergeFailureSkips("merge_generation_guard");
|
|
1481
|
+
continue;
|
|
1482
|
+
}
|
|
1483
|
+
// WS-3b: Lexical diversity check (step 8b).
|
|
1484
|
+
// Low n-gram diversity ⇒ likely correlated-extraction artifact; raise merge threshold.
|
|
1485
|
+
if (antiCollapseConfig.lexicalDiversityCheck !== false) {
|
|
1486
|
+
const bodies = allParticipants
|
|
1487
|
+
.map((ref) => {
|
|
1488
|
+
const e = memoryByRef.get(ref);
|
|
1489
|
+
if (!e)
|
|
1490
|
+
return "";
|
|
1491
|
+
try {
|
|
1492
|
+
const raw = fs.readFileSync(e.filePath, "utf8");
|
|
1493
|
+
return stripFrontmatterBody(raw);
|
|
1494
|
+
}
|
|
1495
|
+
catch {
|
|
1496
|
+
return "";
|
|
1497
|
+
}
|
|
1498
|
+
})
|
|
1499
|
+
.filter((b) => b.length > 0);
|
|
1500
|
+
const diversityCheck = checkLexicalDiversity(bodies, antiCollapseConfig);
|
|
1501
|
+
if (diversityCheck.lowDiversity) {
|
|
1502
|
+
// Low-diversity cluster: just warn (don't refuse merge since the dedup
|
|
1503
|
+
// path handles exact twins). The warning surfaces in health telemetry.
|
|
1504
|
+
warnings.push(`Merge: cluster around ${op.primary} has low lexical diversity (${diversityCheck.diversity?.toFixed(2) ?? "?"} < 0.30) — likely correlated extraction; merge proceeds but review is recommended.`);
|
|
1505
|
+
}
|
|
1506
|
+
}
|
|
1507
|
+
// Inject generation counter into merged content frontmatter (step 8a).
|
|
1508
|
+
// merged.generation = max(sourceGenerations) + 1.
|
|
1509
|
+
mergedContent = injectGenerationFrontmatter(mergedContent, sourceGenerations, allParticipants);
|
|
1510
|
+
}
|
|
1323
1511
|
// Backup secondaries before deleting
|
|
1324
1512
|
for (const secRef of op.secondaries) {
|
|
1325
1513
|
const secEntry = memoryByRef.get(secRef);
|
|
@@ -1522,11 +1710,12 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1522
1710
|
// is the load-bearing content, so dedup must hash on body only.
|
|
1523
1711
|
// (b) A prior run created a proposal for the same body under a
|
|
1524
1712
|
// different knowledgeRef slug.
|
|
1525
|
-
|
|
1713
|
+
// Use cacheHash (case-preserving stripped body) to match the canonical
|
|
1714
|
+
// hash domain used by the body-embedding cache and pending-proposal set.
|
|
1715
|
+
const bodyHash = cacheHash(sourceBody);
|
|
1526
1716
|
const allPendingConsolidateProposals = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
|
|
1527
1717
|
const contentDupProposal = allPendingConsolidateProposals.find((p) => {
|
|
1528
|
-
|
|
1529
|
-
return createHash("sha256").update(otherBody, "utf8").digest("hex") === bodyHash;
|
|
1718
|
+
return cacheHash(p.payload.content) === bodyHash;
|
|
1530
1719
|
});
|
|
1531
1720
|
if (contentDupProposal) {
|
|
1532
1721
|
warnings.push(`Skipping promote: identical body already pending as proposal ${contentDupProposal.id} (ref: ${contentDupProposal.ref}); skipping duplicate for ${op.ref} → ${knowledgeRef}`);
|
|
@@ -1608,6 +1797,18 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1608
1797
|
}
|
|
1609
1798
|
}
|
|
1610
1799
|
else if (op.op === "contradict") {
|
|
1800
|
+
// Confidence gate: surface-level topic overlap causes false positives
|
|
1801
|
+
// (investigation 2026-06-18). Require ≥0.92 confidence before writing
|
|
1802
|
+
// contradiction edges. Missing confidence field defaults to 1.0 for
|
|
1803
|
+
// backward compatibility with responses that predate this field.
|
|
1804
|
+
const opConfidence = typeof op.confidence === "number"
|
|
1805
|
+
? op.confidence
|
|
1806
|
+
: 1.0;
|
|
1807
|
+
if (opConfidence < 0.92) {
|
|
1808
|
+
warnings.push(`Contradict: confidence ${opConfidence.toFixed(2)} below 0.92 threshold for ${op.ref} <-> ${op.contradictedByRef} — skipping.`);
|
|
1809
|
+
pushSkipReason("contradict", op.ref, "contradict_low_confidence");
|
|
1810
|
+
continue;
|
|
1811
|
+
}
|
|
1611
1812
|
// C-3 / #382: Write contradictedBy edges so resolveFamilyContradictions
|
|
1612
1813
|
// (the SCC resolver in memory-improve.ts) has edges to work on.
|
|
1613
1814
|
// Zep arXiv:2501.13956 §3 — unified belief-revision with contradiction edges.
|
|
@@ -1647,37 +1848,19 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1647
1848
|
commitWriteTargetBoundary(target, `Consolidate: ${merged} merged, ${deleted} removed`);
|
|
1648
1849
|
}
|
|
1649
1850
|
cleanupJournal(stashDir, timestamp);
|
|
1650
|
-
//
|
|
1651
|
-
//
|
|
1652
|
-
//
|
|
1653
|
-
//
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
// Emit event before deletion so the record survives the purge.
|
|
1664
|
-
appendEvent({
|
|
1665
|
-
eventType: "archive_cleanup",
|
|
1666
|
-
metadata: {
|
|
1667
|
-
file: fname,
|
|
1668
|
-
filePath: fp,
|
|
1669
|
-
ageMs: Date.now() - stat.mtimeMs,
|
|
1670
|
-
retentionMs,
|
|
1671
|
-
},
|
|
1672
|
-
});
|
|
1673
|
-
fs.unlinkSync(fp);
|
|
1674
|
-
}
|
|
1675
|
-
}
|
|
1676
|
-
catch {
|
|
1677
|
-
/* ignore race conditions */
|
|
1678
|
-
}
|
|
1679
|
-
}
|
|
1680
|
-
}
|
|
1851
|
+
// [signoff 2026-06-15] TTL archive cleanup machinery RETIRED (WS-3a).
|
|
1852
|
+
// The elaborate archiveRetentionDays / archive-dir scan existed only to satisfy
|
|
1853
|
+
// the old irrecoverability constraint. Stashes are now git-backed, so git
|
|
1854
|
+
// history is the recovery path — no bespoke archive TTL needed. Any files in
|
|
1855
|
+
// .akm/archive/ will stay there harmlessly until the operator prunes them with
|
|
1856
|
+
// `git rm` or `find .akm/archive -mtime +90 -delete`. Changed N files this
|
|
1857
|
+
// run; recover any via `git show <sha>:<path>` or `git restore <path>`.
|
|
1858
|
+
if (merged > 0 || deleted > 0 || dedupCollapsed > 0) {
|
|
1859
|
+
const totalChanged = merged + deleted + dedupCollapsed;
|
|
1860
|
+
warnings.push(`Changed ${totalChanged} file(s) this run. Recover any via git if needed (git history is the backstop).`);
|
|
1861
|
+
}
|
|
1862
|
+
const runDurationMs = Date.now() - startMs;
|
|
1863
|
+
const budgetFraction = opts.runBudgetMs !== undefined && opts.runBudgetMs > 0 ? runDurationMs / opts.runBudgetMs : undefined;
|
|
1681
1864
|
return {
|
|
1682
1865
|
schemaVersion: 1,
|
|
1683
1866
|
ok: true,
|
|
@@ -1687,7 +1870,10 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1687
1870
|
target: sourceName,
|
|
1688
1871
|
processed: memories.length,
|
|
1689
1872
|
merged,
|
|
1690
|
-
|
|
1873
|
+
// #617: fold the deterministic dedup pre-pass collapses into the reported
|
|
1874
|
+
// deleted count. Each collapse removed exactly one variant file with NO
|
|
1875
|
+
// LLM call before the LLM pass ran on the pruned pool.
|
|
1876
|
+
deleted: deleted + dedupCollapsed,
|
|
1691
1877
|
promoted,
|
|
1692
1878
|
contradicted,
|
|
1693
1879
|
failedChunks: totalChunksFailed,
|
|
@@ -1697,235 +1883,19 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1697
1883
|
mergedSecondaries,
|
|
1698
1884
|
failedChunkMemories,
|
|
1699
1885
|
warnings,
|
|
1700
|
-
durationMs:
|
|
1886
|
+
durationMs: runDurationMs,
|
|
1887
|
+
perfTelemetry: {
|
|
1888
|
+
dedupPoolSize: perfMs.dedupPoolSize,
|
|
1889
|
+
llmPoolSize,
|
|
1890
|
+
judgedCacheSkipped: perfMs.judgedCacheSkipped,
|
|
1891
|
+
embedMs: embedTelemetry.embedMs,
|
|
1892
|
+
embedCacheHits: embedTelemetry.cacheHits,
|
|
1893
|
+
embedCacheMisses: embedTelemetry.cacheMisses,
|
|
1894
|
+
...(budgetFraction !== undefined ? { estimatedBudgetFractionUsed: budgetFraction } : {}),
|
|
1895
|
+
},
|
|
1701
1896
|
};
|
|
1702
1897
|
}
|
|
1703
1898
|
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
1704
|
-
// ── LLM-output sanitization ─────────────────────────────────────────────────
|
|
1705
|
-
//
|
|
1706
|
-
// Three classes of LLM defect have been observed across hundreds of
|
|
1707
|
-
// consolidate proposals (see audit notes in this branch):
|
|
1708
|
-
//
|
|
1709
|
-
// 1. Code-fence leakage: the entire merged asset is wrapped in
|
|
1710
|
-
// ```markdown … ``` (or ```yaml … ```) despite the prompt forbidding
|
|
1711
|
-
// fences. The post-processor used to pass this through verbatim, so the
|
|
1712
|
-
// first character of the asset content became a backtick rather than
|
|
1713
|
-
// `---`, defeating the frontmatter parser.
|
|
1714
|
-
// 2. YAML quote-escaping bugs: descriptions like `'"Specialty intro...:`
|
|
1715
|
-
// with unbalanced quotes that break the YAML reader. The post-processor
|
|
1716
|
-
// historically passed the LLM's raw scalar straight into a manually
|
|
1717
|
-
// assembled `description: <raw>` line.
|
|
1718
|
-
// 3. Truncated descriptions hitting token cutoffs — the model's max_tokens
|
|
1719
|
-
// runs out mid-sentence, leaving things like
|
|
1720
|
-
// `description: "Tables in narrow column containers need max-width:100% +"`
|
|
1721
|
-
// with no closing context.
|
|
1722
|
-
//
|
|
1723
|
-
// `sanitizeMergedContent` and `validateProposalFrontmatter` defend against
|
|
1724
|
-
// all three at the point where LLM output is consumed.
|
|
1725
|
-
/**
|
|
1726
|
-
* Attempt to recover a frontmatter block that is missing its closing `---`.
|
|
1727
|
-
*
|
|
1728
|
-
* Scans lines after the opening `---` for the first blank line or the first
|
|
1729
|
-
* line that cannot be a YAML scalar (i.e. not a key-value, indented
|
|
1730
|
-
* continuation, comment, or list item). Injects `---` before that line so
|
|
1731
|
-
* the normal parser can proceed.
|
|
1732
|
-
*
|
|
1733
|
-
* Returns the patched string on success, or `null` if the structure is too
|
|
1734
|
-
* ambiguous to recover safely (e.g. no opening `---`, or no body content
|
|
1735
|
-
* found after the frontmatter key-value lines).
|
|
1736
|
-
*/
|
|
1737
|
-
function recoverMalformedFrontmatter(raw) {
|
|
1738
|
-
if (!raw.startsWith("---"))
|
|
1739
|
-
return null;
|
|
1740
|
-
const lines = raw.split(/\r?\n/);
|
|
1741
|
-
// Skip the opening `---` line (index 0).
|
|
1742
|
-
let insertAt = -1;
|
|
1743
|
-
for (let i = 1; i < lines.length; i++) {
|
|
1744
|
-
const line = lines[i];
|
|
1745
|
-
// A blank line marks the end of the frontmatter block in many YAML variants.
|
|
1746
|
-
if (line.trim() === "") {
|
|
1747
|
-
insertAt = i;
|
|
1748
|
-
break;
|
|
1749
|
-
}
|
|
1750
|
-
// A line that is clearly body content: doesn't look like a YAML key, an
|
|
1751
|
-
// indented continuation, a comment, or a sequence item.
|
|
1752
|
-
const isYaml = /^\w[\w-]*\s*:/.test(line) || // key: value
|
|
1753
|
-
/^\s+\S/.test(line) || // indented continuation / nested
|
|
1754
|
-
/^\s*#/.test(line) || // YAML comment
|
|
1755
|
-
/^\s*-\s/.test(line); // sequence item
|
|
1756
|
-
if (!isYaml) {
|
|
1757
|
-
insertAt = i;
|
|
1758
|
-
break;
|
|
1759
|
-
}
|
|
1760
|
-
}
|
|
1761
|
-
if (insertAt < 0)
|
|
1762
|
-
return null;
|
|
1763
|
-
const result = [...lines.slice(0, insertAt), "---", ...lines.slice(insertAt)].join("\n");
|
|
1764
|
-
return result;
|
|
1765
|
-
}
|
|
1766
|
-
/**
|
|
1767
|
-
* Outer-fence stripper specific to consolidate. Unlike the shared
|
|
1768
|
-
* `stripMarkdownFences` helper (which only handles markdown fences), this
|
|
1769
|
-
* variant additionally recognises `yaml` and bare-language fences and refuses
|
|
1770
|
-
* to strip an unbalanced fence — i.e. a leading ``` with no trailing ``` is
|
|
1771
|
-
* treated as a malformed response, not partially sanitized.
|
|
1772
|
-
*
|
|
1773
|
-
* Returns `null` when only one half of a fence pair is present (caller
|
|
1774
|
-
* should reject the response entirely).
|
|
1775
|
-
*/
|
|
1776
|
-
export function stripOuterCodeFence(raw) {
|
|
1777
|
-
const trimmed = raw.trim();
|
|
1778
|
-
const leading = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n/i);
|
|
1779
|
-
const trailing = trimmed.match(/\r?\n```\s*$/);
|
|
1780
|
-
if (!leading && !trailing)
|
|
1781
|
-
return { content: trimmed, stripped: false };
|
|
1782
|
-
if (!leading || !trailing)
|
|
1783
|
-
return null; // unbalanced — refuse
|
|
1784
|
-
const inner = trimmed.slice(leading[0].length, trimmed.length - trailing[0].length).trim();
|
|
1785
|
-
return { content: inner, stripped: true };
|
|
1786
|
-
}
|
|
1787
|
-
export function sanitizeMergedContent(raw) {
|
|
1788
|
-
// Step 1: Strip outer code fence.
|
|
1789
|
-
// Recovery path: if only the leading fence is present, strip it and continue
|
|
1790
|
-
// provided the inner content starts with `---`. Trailing-only fences are NOT
|
|
1791
|
-
// recovered — a trailing ``` is more likely a body code block than a forgotten
|
|
1792
|
-
// wrapper, so recovering would silently corrupt the body.
|
|
1793
|
-
let body;
|
|
1794
|
-
{
|
|
1795
|
-
const fenceResult = stripOuterCodeFence(raw);
|
|
1796
|
-
if (fenceResult) {
|
|
1797
|
-
body = fenceResult.content;
|
|
1798
|
-
}
|
|
1799
|
-
else {
|
|
1800
|
-
const trimmed = raw.trim();
|
|
1801
|
-
const leadingMatch = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n([\s\S]*)$/i);
|
|
1802
|
-
const inner = leadingMatch ? leadingMatch[1].trim() : null;
|
|
1803
|
-
if (!inner?.startsWith("---")) {
|
|
1804
|
-
return { ok: false, reason: "UNBALANCED_CODE_FENCE" };
|
|
1805
|
-
}
|
|
1806
|
-
body = inner;
|
|
1807
|
-
}
|
|
1808
|
-
}
|
|
1809
|
-
// Strip <think> blocks (some local models still emit them despite system prompts).
|
|
1810
|
-
body = body.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
1811
|
-
// Step 2: Verify frontmatter sentinel.
|
|
1812
|
-
// Recovery path: LLM sometimes emits 1-2 lines of preamble (e.g. "Here is the
|
|
1813
|
-
// merged content:") before the `---`. Accept if `---` appears within 300 chars.
|
|
1814
|
-
// Beyond that it's more likely a body section divider, not a frontmatter start.
|
|
1815
|
-
if (!body.startsWith("---")) {
|
|
1816
|
-
const nlIdx = body.indexOf("\n---");
|
|
1817
|
-
if (nlIdx >= 0 && nlIdx < 300) {
|
|
1818
|
-
body = body.slice(nlIdx + 1);
|
|
1819
|
-
}
|
|
1820
|
-
else {
|
|
1821
|
-
return { ok: false, reason: "MISSING_FRONTMATTER_SENTINEL" };
|
|
1822
|
-
}
|
|
1823
|
-
}
|
|
1824
|
-
// Extract frontmatter block.
|
|
1825
|
-
// Recovery path: LLM sometimes omits the closing `---` delimiter. Detect this
|
|
1826
|
-
// by scanning lines after the opening `---` for the first blank line or the
|
|
1827
|
-
// first line that isn't a YAML key-value pair, then inject `---` there.
|
|
1828
|
-
let match = body.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
|
|
1829
|
-
if (!match) {
|
|
1830
|
-
const recovered = recoverMalformedFrontmatter(body);
|
|
1831
|
-
if (recovered) {
|
|
1832
|
-
match = recovered.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
|
|
1833
|
-
}
|
|
1834
|
-
if (!match) {
|
|
1835
|
-
return { ok: false, reason: "MALFORMED_FRONTMATTER_BLOCK" };
|
|
1836
|
-
}
|
|
1837
|
-
}
|
|
1838
|
-
// Re-parse via the yaml library so any quote-escaping mistakes either get
|
|
1839
|
-
// normalised or surface as a parse error we can reject.
|
|
1840
|
-
// Recovery: if the strict yaml library fails, fall back to the lenient
|
|
1841
|
-
// hand-rolled parseFrontmatter parser, which tolerates common LLM YAML
|
|
1842
|
-
// quirks (unescaped special chars, bare scalars, etc.). If it recovers
|
|
1843
|
-
// at least one key, proceed — serializeFrontmatter below will re-serialize
|
|
1844
|
-
// cleanly. Only reject if both parsers fail to extract any data.
|
|
1845
|
-
let parsedFm;
|
|
1846
|
-
try {
|
|
1847
|
-
parsedFm = yamlParse(match[1]);
|
|
1848
|
-
}
|
|
1849
|
-
catch (e) {
|
|
1850
|
-
const fallback = parseFrontmatter(`---\n${match[1]}\n---\n${match[2]}`);
|
|
1851
|
-
if (fallback.frontmatter !== null && Object.keys(fallback.data).length > 0) {
|
|
1852
|
-
parsedFm = fallback.data;
|
|
1853
|
-
}
|
|
1854
|
-
else {
|
|
1855
|
-
return { ok: false, reason: `INVALID_YAML: ${e instanceof Error ? e.message : String(e)}` };
|
|
1856
|
-
}
|
|
1857
|
-
}
|
|
1858
|
-
if (parsedFm === null || typeof parsedFm !== "object" || Array.isArray(parsedFm)) {
|
|
1859
|
-
return { ok: false, reason: "FRONTMATTER_NOT_OBJECT" };
|
|
1860
|
-
}
|
|
1861
|
-
const fm = parsedFm;
|
|
1862
|
-
// Normalise placeholder leaks like `updated: today`, `updated: {today: null}`,
|
|
1863
|
-
// `updated: now`, etc. The consolidate prompt instructs the LLM not to emit
|
|
1864
|
-
// these, but small models still do. Replace any such leak with today's ISO
|
|
1865
|
-
// date OR drop the field if we can't safely normalise it.
|
|
1866
|
-
normalizeUpdatedField(fm);
|
|
1867
|
-
// Re-serialise via yaml.stringify to fix any quoting quirks.
|
|
1868
|
-
let serialized;
|
|
1869
|
-
try {
|
|
1870
|
-
serialized = serializeFrontmatter(fm);
|
|
1871
|
-
}
|
|
1872
|
-
catch (e) {
|
|
1873
|
-
return { ok: false, reason: `YAML_STRINGIFY_FAILED: ${e instanceof Error ? e.message : String(e)}` };
|
|
1874
|
-
}
|
|
1875
|
-
const cleaned = assembleAssetFromString(serialized, match[2]);
|
|
1876
|
-
return { ok: true, result: { content: cleaned, frontmatter: fm } };
|
|
1877
|
-
}
|
|
1878
|
-
/**
|
|
1879
|
-
* Mutate `fm.updated` in place to normalise placeholder leaks emitted by the
|
|
1880
|
-
* LLM. The consolidate prompt forbids these, but small models still produce
|
|
1881
|
-
* literal `today` / `{today: null}` / `now` values.
|
|
1882
|
-
*
|
|
1883
|
-
* Rules:
|
|
1884
|
-
* - A real ISO-style date string (YYYY-MM-DD, optionally with time) stays as-is.
|
|
1885
|
-
* - A Date object (some YAML parsers materialise dates) is converted to its
|
|
1886
|
-
* ISO yyyy-mm-dd form.
|
|
1887
|
-
* - A placeholder string ("today", "now", "{today}", "${today}", template
|
|
1888
|
-
* variables) is replaced with today's ISO date.
|
|
1889
|
-
* - A map/object (e.g. `{today: null}`) is replaced with today's ISO date.
|
|
1890
|
-
* - `null`, empty string, missing → left alone (no field added; reviewers
|
|
1891
|
-
* should not silently gain metadata they didn't write).
|
|
1892
|
-
*
|
|
1893
|
-
* Exported for unit testing.
|
|
1894
|
-
*/
|
|
1895
|
-
export function normalizeUpdatedField(fm) {
|
|
1896
|
-
if (!("updated" in fm))
|
|
1897
|
-
return;
|
|
1898
|
-
const v = fm.updated;
|
|
1899
|
-
if (v === null || v === undefined || v === "")
|
|
1900
|
-
return;
|
|
1901
|
-
const todayIso = new Date().toISOString().slice(0, 10);
|
|
1902
|
-
if (v instanceof Date) {
|
|
1903
|
-
fm.updated = v.toISOString().slice(0, 10);
|
|
1904
|
-
return;
|
|
1905
|
-
}
|
|
1906
|
-
if (typeof v === "string") {
|
|
1907
|
-
const trimmed = v.trim().toLowerCase();
|
|
1908
|
-
if (/^\d{4}-\d{2}-\d{2}/.test(v.trim()))
|
|
1909
|
-
return; // already a real date
|
|
1910
|
-
if (trimmed === "today" ||
|
|
1911
|
-
trimmed === "now" ||
|
|
1912
|
-
trimmed === "{today}" ||
|
|
1913
|
-
// biome-ignore lint/suspicious/noTemplateCurlyInString: matches the literal user-typed placeholder text "${today}" so we can normalize it to today's ISO date
|
|
1914
|
-
trimmed === "${today}" ||
|
|
1915
|
-
trimmed === "{{today}}" ||
|
|
1916
|
-
/^\{?\s*today\s*\}?$/.test(trimmed)) {
|
|
1917
|
-
fm.updated = todayIso;
|
|
1918
|
-
return;
|
|
1919
|
-
}
|
|
1920
|
-
// Unknown string format — leave alone so it's visible in the diff.
|
|
1921
|
-
return;
|
|
1922
|
-
}
|
|
1923
|
-
if (typeof v === "object") {
|
|
1924
|
-
// Maps like `{today: null}`, `{now: null}` — clearly a template leak.
|
|
1925
|
-
fm.updated = todayIso;
|
|
1926
|
-
return;
|
|
1927
|
-
}
|
|
1928
|
-
}
|
|
1929
1899
|
/**
|
|
1930
1900
|
* Normalise a knowledge slug for variant-aware deduplication. Collapses:
|
|
1931
1901
|
* - date suffixes (`-may-2026`, `-2026-05-03`, `-2026`)
|
|
@@ -2004,7 +1974,7 @@ function parseSinceToIso(since) {
|
|
|
2004
1974
|
const multiplier = { m: 60_000, h: 3_600_000, d: 86_400_000 }[m[2]];
|
|
2005
1975
|
return new Date(Date.now() - parseInt(m[1], 10) * multiplier).toISOString();
|
|
2006
1976
|
}
|
|
2007
|
-
export function narrowToIncrementalCandidates(memories, since, warnings) {
|
|
1977
|
+
export function narrowToIncrementalCandidates(memories, since, warnings, neighborsPerChanged = 5) {
|
|
2008
1978
|
const sinceIso = parseSinceToIso(since);
|
|
2009
1979
|
const isChanged = (m) => {
|
|
2010
1980
|
try {
|
|
@@ -2019,7 +1989,6 @@ export function narrowToIncrementalCandidates(memories, since, warnings) {
|
|
|
2019
1989
|
return [];
|
|
2020
1990
|
if (changed.length === memories.length)
|
|
2021
1991
|
return memories;
|
|
2022
|
-
const NEIGHBORS_PER_CHANGED = 5;
|
|
2023
1992
|
const byName = new Map(memories.map((m) => [m.name, m]));
|
|
2024
1993
|
const keep = new Set(changed.map((m) => m.name));
|
|
2025
1994
|
let db;
|
|
@@ -2029,7 +1998,7 @@ export function narrowToIncrementalCandidates(memories, since, warnings) {
|
|
|
2029
1998
|
const id = findEntryIdByRef(db, `memory:${m.name}`);
|
|
2030
1999
|
if (id === undefined)
|
|
2031
2000
|
continue;
|
|
2032
|
-
for (const hit of getNeighborsByEntryId(db, id,
|
|
2001
|
+
for (const hit of getNeighborsByEntryId(db, id, neighborsPerChanged + 1)) {
|
|
2033
2002
|
if (hit.id === id)
|
|
2034
2003
|
continue;
|
|
2035
2004
|
const entry = getEntryById(db, hit.id);
|
|
@@ -2067,6 +2036,10 @@ function loadMemoriesForSource(source, stashDir, warnings) {
|
|
|
2067
2036
|
return path.resolve(e.stashDir) === path.resolve(source);
|
|
2068
2037
|
})
|
|
2069
2038
|
.filter((e) => isConsolidationEligibleMemoryName(e.entry.name))
|
|
2039
|
+
// #632 — exclude session-capture telemetry (checkpoints) from the
|
|
2040
|
+
// consolidation pool, mirroring recombine. Their bodies are pipeline
|
|
2041
|
+
// bookkeeping, so consolidating them burns LLM calls on pure noise.
|
|
2042
|
+
.filter((e) => !isSessionCaptureMemoryName(e.entry.name))
|
|
2070
2043
|
// Skip stale DB entries whose file was deleted by a prior run but not yet
|
|
2071
2044
|
// re-indexed. Without this guard the deleted file's ref appears in chunks
|
|
2072
2045
|
// sent to the LLM, which then proposes a second delete → delete_failed
|
|
@@ -2100,6 +2073,8 @@ function loadMemoriesForSource(source, stashDir, warnings) {
|
|
|
2100
2073
|
const name = fname.replace(/\.md$/, "");
|
|
2101
2074
|
if (!isConsolidationEligibleMemoryName(name))
|
|
2102
2075
|
continue;
|
|
2076
|
+
if (isSessionCaptureMemoryName(name))
|
|
2077
|
+
continue; // #632 — skip telemetry checkpoints
|
|
2103
2078
|
memories.push({ name, filePath, description: "", tags: [], stashDir: fsStashDir });
|
|
2104
2079
|
}
|
|
2105
2080
|
}
|
|
@@ -2231,8 +2206,8 @@ async function generateMergedContent(config, primaryRef, primaryBody, secondaryR
|
|
|
2231
2206
|
}
|
|
2232
2207
|
normalizeUpdatedField(repairedFmData);
|
|
2233
2208
|
const repairedYaml = serializeFrontmatter(repairedFmData);
|
|
2234
|
-
const bodyPart = mergedFm.content
|
|
2235
|
-
return { content:
|
|
2209
|
+
const bodyPart = typeof mergedFm.content === "string" ? mergedFm.content : "";
|
|
2210
|
+
return { content: assembleAssetFromString(repairedYaml, bodyPart) };
|
|
2236
2211
|
}
|
|
2237
2212
|
}
|
|
2238
2213
|
catch {
|