akm-cli 0.9.0-beta.9 → 0.9.0-rc.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +592 -0
- package/README.md +12 -4
- package/dist/akm +38 -0
- package/dist/akm-migrate-storage +38 -0
- package/dist/assets/help/help-improve.md +9 -6
- package/dist/assets/hints/cli-hints-full.md +6 -5
- package/dist/assets/profiles/default.json +9 -4
- package/dist/assets/profiles/frequent.json +1 -1
- package/dist/assets/profiles/memory-focus.json +1 -1
- package/dist/assets/profiles/proactive-maintenance.json +25 -0
- package/dist/assets/profiles/quick.json +1 -1
- package/dist/assets/profiles/recombine-only.json +21 -0
- package/dist/assets/profiles/reflect-distill.json +30 -0
- package/dist/assets/profiles/synthesize.json +15 -0
- package/dist/assets/profiles/thorough.json +1 -1
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +11 -3
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/graph-extract-user-prompt.md +1 -1
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/agent.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/command.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/fact.md +39 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/knowledge.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/lesson.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/memory.md +38 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/script.md +43 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/skill.md +40 -0
- package/dist/assets/stash-skeleton/facts/conventions/assets/workflow.md +43 -0
- package/dist/assets/templates/html/health.html +281 -111
- package/dist/assets/wiki/ingest-workflow-template.md +45 -16
- package/dist/assets/wiki/schema-template.md +4 -4
- package/dist/cli/clack.js +56 -0
- package/dist/cli/config-migrate.js +7 -1
- package/dist/cli/confirm.js +1 -1
- package/dist/cli/parse-args.js +46 -1
- package/dist/cli/shared.js +28 -0
- package/dist/cli.js +25 -21
- package/dist/commands/agent/agent-dispatch.js +3 -2
- package/dist/commands/agent/agent-support.js +0 -7
- package/dist/commands/agent/contribute-cli.js +26 -7
- package/dist/commands/config-cli.js +26 -13
- package/dist/commands/env/child-env.js +47 -0
- package/dist/commands/env/env-cli.js +220 -227
- package/dist/commands/env/env.js +14 -67
- package/dist/commands/env/secret-cli.js +140 -138
- package/dist/commands/feedback-cli.js +156 -155
- package/dist/commands/graph/graph-cli.js +5 -13
- package/dist/commands/graph/graph.js +3 -3
- package/dist/commands/health/advisories.js +151 -0
- package/dist/commands/health/checks.js +103 -16
- package/dist/commands/health/html-report.js +447 -81
- package/dist/commands/health/improve-metrics.js +771 -0
- package/dist/commands/health/llm-usage.js +65 -0
- package/dist/commands/health/md-report.js +103 -0
- package/dist/commands/health/metrics.js +278 -0
- package/dist/commands/health/stash-exposure.js +46 -0
- package/dist/commands/health/surfaces.js +216 -0
- package/dist/commands/health/task-runs.js +135 -0
- package/dist/commands/health/types.js +26 -0
- package/dist/commands/health/windows.js +195 -0
- package/dist/commands/health.js +91 -1091
- package/dist/commands/improve/anti-collapse.js +170 -0
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/collapse-detector.js +421 -0
- package/dist/commands/improve/consolidate/chunking.js +141 -0
- package/dist/commands/improve/consolidate/eligibility.js +64 -0
- package/dist/commands/improve/consolidate/merge.js +145 -0
- package/dist/commands/improve/consolidate/sanitize.js +231 -0
- package/dist/commands/{lint.js → improve/consolidate/types.js} +1 -1
- package/dist/commands/improve/consolidate.js +1295 -1277
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill/content-repair.js +202 -0
- package/dist/commands/improve/distill/promote-memory.js +229 -0
- package/dist/commands/improve/distill/quality-gate.js +236 -0
- package/dist/commands/improve/distill-guards.js +127 -0
- package/dist/commands/improve/distill-promotion-policy.js +826 -167
- package/dist/commands/improve/distill.js +228 -605
- package/dist/commands/improve/eligibility.js +434 -0
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +179 -59
- package/dist/commands/improve/extract-prompt.js +54 -3
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +409 -43
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/hot-probation.js +45 -0
- package/dist/commands/improve/improve-auto-accept.js +157 -10
- package/dist/commands/improve/improve-cli.js +115 -73
- package/dist/commands/improve/improve-profiles.js +28 -8
- package/dist/commands/improve/improve-result-file.js +15 -25
- package/dist/commands/improve/improve-session.js +58 -0
- package/dist/commands/improve/improve.js +485 -2764
- package/dist/commands/improve/locks.js +154 -0
- package/dist/commands/improve/loop-stages.js +1100 -0
- package/dist/commands/improve/memory/memory-belief.js +14 -15
- package/dist/commands/improve/memory/memory-contradiction-detect.js +83 -60
- package/dist/commands/improve/memory/memory-improve.js +27 -27
- package/dist/commands/improve/outcome-loop.js +270 -0
- package/dist/commands/improve/preparation.js +2002 -0
- package/dist/commands/improve/proactive-maintenance.js +37 -35
- package/dist/commands/improve/procedural.js +398 -0
- package/dist/commands/improve/recombine.js +818 -0
- package/dist/commands/improve/reflect-noise.js +0 -0
- package/dist/commands/improve/reflect.js +206 -45
- package/dist/commands/improve/salience.js +455 -0
- package/dist/commands/improve/schema-similarity-gate.js +168 -0
- package/dist/commands/improve/shared.js +51 -0
- package/dist/commands/improve/triage.js +93 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +38 -1
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +31 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +7 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/observability-cli.js +4 -4
- package/dist/commands/proposal/drain-policies.js +13 -4
- package/dist/commands/proposal/drain.js +45 -51
- package/dist/commands/proposal/legacy-import.js +115 -0
- package/dist/commands/proposal/proposal-cli.js +24 -34
- package/dist/commands/proposal/proposal.js +2 -1
- package/dist/commands/proposal/propose.js +8 -3
- package/dist/commands/proposal/repository.js +829 -0
- package/dist/commands/proposal/validators/proposal-quality-validators.js +9 -8
- package/dist/commands/proposal/validators/proposals.js +93 -895
- package/dist/commands/read/curate.js +410 -111
- package/dist/commands/read/knowledge.js +10 -3
- package/dist/commands/read/remember-cli.js +133 -138
- package/dist/commands/read/search-cli.js +15 -8
- package/dist/commands/read/search.js +22 -11
- package/dist/commands/read/show.js +106 -14
- package/dist/commands/registry-cli.js +76 -87
- package/dist/commands/remember.js +11 -12
- package/dist/commands/sources/add-cli.js +91 -95
- package/dist/commands/sources/history.js +1 -1
- package/dist/commands/sources/init.js +66 -18
- package/dist/commands/sources/installed-stashes.js +11 -3
- package/dist/commands/sources/schema-repair.js +44 -46
- package/dist/commands/sources/self-update.js +2 -2
- package/dist/commands/sources/source-add.js +7 -3
- package/dist/commands/sources/sources-cli.js +3 -3
- package/dist/commands/sources/stash-cli.js +19 -39
- package/dist/commands/sources/stash-skeleton.js +57 -8
- package/dist/commands/tasks/default-tasks.js +15 -2
- package/dist/commands/tasks/tasks-cli.js +20 -29
- package/dist/commands/tasks/tasks.js +39 -11
- package/dist/commands/wiki-cli.js +23 -38
- package/dist/commands/workflow-cli.js +15 -1
- package/dist/core/asset/asset-registry.js +3 -1
- package/dist/core/asset/asset-spec.js +21 -4
- package/dist/core/asset/frontmatter.js +188 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/authoring-rules.js +92 -0
- package/dist/core/common.js +4 -23
- package/dist/core/concurrent.js +10 -1
- package/dist/core/config/config-io.js +10 -1
- package/dist/core/config/config-migration.js +18 -40
- package/dist/core/config/config-schema.js +382 -62
- package/dist/core/config/config-types.js +3 -3
- package/dist/core/config/config.js +67 -22
- package/dist/core/deep-merge.js +38 -0
- package/dist/core/errors.js +1 -0
- package/dist/core/eval/rank-metrics.js +113 -0
- package/dist/core/events.js +4 -7
- package/dist/core/improve-types.js +47 -8
- package/dist/core/logs-db.js +14 -75
- package/dist/core/parse.js +36 -16
- package/dist/core/paths.js +18 -18
- package/dist/core/standards/resolve-standards-context.js +87 -0
- package/dist/core/standards/resolve-stash-standards.js +99 -0
- package/dist/core/standards/resolve-type-conventions.js +66 -0
- package/dist/core/state/migrations.js +770 -0
- package/dist/core/state-db.js +132 -1126
- package/dist/core/structured.js +69 -0
- package/dist/core/time.js +53 -0
- package/dist/core/warn.js +21 -0
- package/dist/core/write-source.js +37 -0
- package/dist/indexer/db/db.js +259 -769
- package/dist/indexer/db/entry-mapper.js +41 -0
- package/dist/indexer/db/graph-db.js +129 -86
- package/dist/indexer/db/llm-cache.js +2 -2
- package/dist/indexer/db/schema.js +516 -0
- package/dist/indexer/ensure-index.js +36 -92
- package/dist/indexer/feedback/utility-policy.js +75 -0
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/graph/graph-extraction.js +207 -4
- package/dist/indexer/index-writer-lock.js +18 -11
- package/dist/indexer/index-written-assets.js +105 -0
- package/dist/indexer/indexer.js +182 -204
- package/dist/indexer/passes/dir-staleness.js +114 -0
- package/dist/indexer/passes/memory-inference.js +13 -5
- package/dist/indexer/passes/metadata.js +20 -0
- package/dist/indexer/read-preflight.js +23 -0
- package/dist/indexer/search/db-search.js +89 -13
- package/dist/indexer/search/fts-query.js +51 -0
- package/dist/indexer/search/ranking-contributors.js +95 -9
- package/dist/indexer/search/ranking.js +79 -3
- package/dist/indexer/search/search-fields.js +6 -0
- package/dist/indexer/search/search-source.js +32 -21
- package/dist/indexer/search/semantic-status.js +4 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/indexer/walk/walker.js +21 -13
- package/dist/integrations/agent/builders.js +39 -13
- package/dist/integrations/agent/config.js +20 -59
- package/dist/integrations/agent/detect.js +9 -0
- package/dist/integrations/agent/index.js +3 -19
- package/dist/integrations/agent/model-aliases.js +7 -2
- package/dist/integrations/agent/profiles.js +7 -1
- package/dist/integrations/agent/prompts.js +75 -9
- package/dist/integrations/agent/runner-dispatch.js +59 -0
- package/dist/integrations/agent/runner.js +13 -9
- package/dist/integrations/agent/spawn.js +69 -67
- package/dist/integrations/harnesses/claude/agent-builder.js +1 -1
- package/dist/integrations/harnesses/claude/index.js +2 -0
- package/dist/integrations/harnesses/claude/session-log.js +10 -0
- package/dist/integrations/harnesses/index.js +2 -3
- package/dist/integrations/harnesses/opencode/agent-builder.js +1 -1
- package/dist/integrations/harnesses/opencode/index.js +2 -0
- package/dist/integrations/harnesses/opencode/session-log.js +173 -3
- package/dist/integrations/harnesses/opencode-sdk/index.js +2 -2
- package/dist/integrations/harnesses/opencode-sdk/sdk-runner.js +98 -17
- package/dist/integrations/harnesses/types.js +1 -0
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/call-ai.js +2 -2
- package/dist/llm/client.js +34 -11
- package/dist/llm/embedder.js +67 -4
- package/dist/llm/embedders/cache.js +3 -1
- package/dist/llm/embedders/deterministic.js +66 -0
- package/dist/llm/embedders/local.js +73 -3
- package/dist/llm/feature-gate.js +16 -15
- package/dist/llm/graph-extract.js +67 -44
- package/dist/llm/memory-infer-impl.js +138 -0
- package/dist/llm/memory-infer.js +1 -127
- package/dist/llm/metadata-enhance.js +44 -31
- package/dist/llm/structured-call.js +49 -0
- package/dist/migrate-storage-node.mjs +8 -0
- package/dist/output/context.js +5 -5
- package/dist/output/renderers.js +85 -14
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/shapes/helpers.js +0 -3
- package/dist/output/shapes/passthrough.js +2 -1
- package/dist/output/text/helpers.js +29 -1
- package/dist/output/text/workflow.js +1 -0
- package/dist/registry/providers/skills-sh.js +21 -147
- package/dist/registry/providers/static-index.js +15 -157
- package/dist/registry/resolve.js +27 -9
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +2661 -2369
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +883 -596
- package/dist/setup/detect.js +9 -0
- package/dist/setup/legacy-config.js +106 -0
- package/dist/setup/prompt.js +57 -0
- package/dist/setup/providers.js +14 -0
- package/dist/setup/registry-stash-loader.js +12 -0
- package/dist/setup/semantic-assets.js +124 -0
- package/dist/setup/setup.js +52 -1614
- package/dist/setup/steps/connection.js +734 -0
- package/dist/setup/steps/output.js +31 -0
- package/dist/setup/steps/platforms.js +124 -0
- package/dist/setup/steps/semantic.js +27 -0
- package/dist/setup/steps/sources.js +222 -0
- package/dist/setup/steps/stashdir.js +42 -0
- package/dist/setup/steps/tasks.js +152 -0
- package/dist/sources/include.js +6 -2
- package/dist/sources/providers/filesystem.js +0 -1
- package/dist/sources/providers/git-install.js +210 -0
- package/dist/sources/providers/git-provider.js +234 -0
- package/dist/sources/providers/git-stash.js +248 -0
- package/dist/sources/providers/git.js +10 -661
- package/dist/sources/providers/npm.js +2 -6
- package/dist/sources/providers/provider-utils.js +13 -7
- package/dist/sources/providers/sync-from-ref.js +9 -1
- package/dist/sources/providers/website.js +9 -5
- package/dist/sources/website-ingest.js +187 -29
- package/dist/sources/wiki-fetchers/registry.js +53 -0
- package/dist/sources/wiki-fetchers/youtube.js +239 -0
- package/dist/storage/database.js +45 -10
- package/dist/storage/managed-db.js +82 -0
- package/dist/storage/repositories/canaries-repository.js +107 -0
- package/dist/storage/repositories/consolidation-repository.js +38 -0
- package/dist/storage/repositories/embeddings-repository.js +72 -0
- package/dist/storage/repositories/events-repository.js +187 -0
- package/dist/storage/repositories/extract-sessions-repository.js +96 -0
- package/dist/storage/repositories/improve-runs-repository.js +146 -0
- package/dist/storage/repositories/index-db.js +14 -8
- package/dist/storage/repositories/proposals-repository.js +220 -0
- package/dist/storage/repositories/recombine-repository.js +213 -0
- package/dist/storage/repositories/registry-cache.js +93 -0
- package/dist/storage/repositories/registry-index-cache-repository.js +46 -0
- package/dist/storage/repositories/task-history-repository.js +93 -0
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/tasks/backends/cron.js +1 -1
- package/dist/tasks/backends/index.js +9 -0
- package/dist/tasks/backends/launchd.js +1 -1
- package/dist/tasks/backends/schtasks.js +1 -1
- package/dist/tasks/{resolveAkmBin.js → resolve-akm-bin.js} +2 -2
- package/dist/tasks/runner.js +15 -13
- package/dist/text-import-hook.mjs +0 -0
- package/dist/wiki/wiki.js +52 -11
- package/dist/workflows/cli.js +1 -0
- package/dist/workflows/db.js +3 -4
- package/dist/workflows/runtime/runs.js +43 -118
- package/dist/workflows/runtime/workflow-asset-loader.js +125 -0
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/README.md +69 -18
- package/docs/data-and-telemetry.md +5 -4
- package/docs/migration/release-notes/0.7.0.md +1 -1
- package/docs/migration/release-notes/0.9.0.md +39 -0
- package/package.json +10 -10
- package/dist/assets/tasks/core/update-stashes.yml +0 -4
- package/dist/commands/db-cli.js +0 -23
- package/dist/indexer/db/db-backup.js +0 -376
- package/dist/indexer/passes/staleness-detect.js +0 -488
|
@@ -5,52 +5,54 @@ import { createHash } from "node:crypto";
|
|
|
5
5
|
import fs from "node:fs";
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import readline from "node:readline";
|
|
8
|
-
import
|
|
8
|
+
import consolidateSystemPrompt from "../../assets/prompts/consolidate-system.md" with { type: "text" };
|
|
9
9
|
import { parseAssetRef } from "../../core/asset/asset-ref.js";
|
|
10
10
|
import { assembleAssetFromString, serializeFrontmatter } from "../../core/asset/asset-serialize.js";
|
|
11
11
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
12
12
|
import { resolveStashDir, timestampForFilename } from "../../core/common.js";
|
|
13
|
-
import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
|
|
13
|
+
import { getDefaultLlmConfig, getImproveProcessConfig, loadConfig } from "../../core/config/config.js";
|
|
14
14
|
import { ConfigError } from "../../core/errors.js";
|
|
15
|
-
|
|
15
|
+
// Note: appendEvent import removed (WS-3a: archive TTL machinery retired)
|
|
16
16
|
import { parseEmbeddedJsonResponse } from "../../core/parse.js";
|
|
17
|
+
import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
|
|
17
18
|
import { detectTruncatedDescription } from "../../core/text-truncation.js";
|
|
18
|
-
import {
|
|
19
|
-
import { createProposal, isProposalSkipped, listProposals } from "../proposal/
|
|
19
|
+
import { DURATION_UNITS, parseDuration } from "../../core/time.js";
|
|
20
|
+
import { createProposal, isProposalSkipped, listProposals } from "../proposal/repository.js";
|
|
21
|
+
import { hasSupersededStatus, MERGE_ABSOLUTE_FLOOR_CHARS, MERGE_SHRINK_RATIO_MIN, validateProposalFrontmatter, } from "../proposal/validators/proposal-quality-validators.js";
|
|
22
|
+
import { checkGenerationGuard, checkLexicalDiversity, checkMergeInformationFloor, computeMergedGeneration, readAssetGeneration, } from "./anti-collapse.js";
|
|
23
|
+
import { cacheHash, runDeterministicDedup, stripFrontmatterBody } from "./dedup.js";
|
|
24
|
+
import { shouldSkipHotProbationInLlm } from "./hot-probation.js";
|
|
20
25
|
import { writeContradictEdge } from "./memory/memory-belief.js";
|
|
21
26
|
// Re-export the moved helpers so existing test imports continue to resolve.
|
|
22
27
|
export { hasSupersededStatus, validateProposalFrontmatter };
|
|
28
|
+
import { openStateDatabase, withStateDb } from "../../core/state-db.js";
|
|
23
29
|
import { warn } from "../../core/warn.js";
|
|
24
30
|
import { commitWriteTargetBoundary, deleteAssetFromSource, resolveWriteTarget, writeAssetToSource, } from "../../core/write-source.js";
|
|
25
31
|
import { closeDatabase, findEntryIdByRef, getAllEntries, getEntryById, getNeighborsByEntryId, openExistingDatabase, } from "../../indexer/db/db.js";
|
|
26
32
|
import { resolveImproveProcessRunnerFromProfile, runnerIsLlm } from "../../integrations/agent/runner.js";
|
|
27
33
|
import { chatCompletion } from "../../llm/client.js";
|
|
28
|
-
import { cosineSimilarity, embedBatch } from "../../llm/embedder.js";
|
|
34
|
+
import { cosineSimilarity, embedBatch, resolveEmbeddingModelId } from "../../llm/embedder.js";
|
|
29
35
|
import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
|
|
36
|
+
import { getConsolidationJudgedMap, upsertConsolidationJudged, } from "../../storage/repositories/consolidation-repository.js";
|
|
37
|
+
import { getBodyEmbeddings, upsertBodyEmbeddings } from "../../storage/repositories/embeddings-repository.js";
|
|
38
|
+
// Chunk sizing + per-chunk prompt assembly live in ./consolidate/chunking.
|
|
39
|
+
// Imported for internal use by the orchestrator and re-exported for importers.
|
|
40
|
+
import { buildChunkPrompt, computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
|
|
41
|
+
export { buildChunkPrompt, computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
|
|
42
|
+
// LLM-output sanitization (pure string/frontmatter transforms) lives in
|
|
43
|
+
// ./consolidate/sanitize. Imported for internal use + re-exported for importers.
|
|
44
|
+
import { normalizeUpdatedField, sanitizeMergedContent } from "./consolidate/sanitize.js";
|
|
45
|
+
export { normalizeUpdatedField, sanitizeMergedContent, stripOuterCodeFence } from "./consolidate/sanitize.js";
|
|
46
|
+
// Eligibility / safety predicates live in ./consolidate/eligibility. Imported
|
|
47
|
+
// for internal guard use; the two public predicates are re-exported.
|
|
48
|
+
import { consolidateGuardStatus, isConsolidationEligibleMemoryName, isHotCapturedMemory, } from "./consolidate/eligibility.js";
|
|
49
|
+
export { isConsolidationEligibleMemoryName, isHotCapturedMemory } from "./consolidate/eligibility.js";
|
|
50
|
+
// Plan parsing / merging (pure op-reconciliation algebra) lives in
|
|
51
|
+
// ./consolidate/merge. Imported for internal use; mergePlans re-exported.
|
|
52
|
+
import { isValidOp, mergePlans } from "./consolidate/merge.js";
|
|
53
|
+
export { mergePlans } from "./consolidate/merge.js";
|
|
30
54
|
// ── Prompts ─────────────────────────────────────────────────────────────────
|
|
31
|
-
const CONSOLIDATE_SYSTEM_PROMPT =
|
|
32
|
-
|
|
33
|
-
Rules:
|
|
34
|
-
1. MERGE: Two or more memories are substantially duplicated or closely related → propose merging. Return the primary ref to keep and secondary refs to delete. Do NOT include mergedContent — the merge will be executed in a separate step.
|
|
35
|
-
2. DELETE: Memory is clearly outdated, contradicted, or redundant → propose deletion. NEVER propose delete for memories annotated \`(captureMode: hot)\` — they are user-explicit and only the user can retire them. The downstream guard will refuse these regardless, so proposing them just wastes tokens.
|
|
36
|
-
3. PROMOTE: Memory expresses a stable, reusable fact suitable as a \`knowledge:\` asset → propose promotion. Do NOT delete the source memory. NEVER propose promote / merge / contradict for memories annotated \`(already queued)\` — they have a pending proposal whose body matches; a duplicate will be deterministically dropped, so proposing them just wastes tokens.
|
|
37
|
-
4. CONTRADICT: Two memories make mutually exclusive factual claims about the same subject (e.g. "always use VPN" vs "VPN is optional") → mark the older or less authoritative one as contradicted. This writes a contradictedBy edge so the belief-resolution SCC algorithm can resolve the conflict. Do NOT delete contradicted memories — let the belief resolver decide.
|
|
38
|
-
5. KEEP: Memory is unique and current → omit from output.
|
|
39
|
-
|
|
40
|
-
Return ONLY JSON (no prose, no code fences):
|
|
41
|
-
{
|
|
42
|
-
"operations": [
|
|
43
|
-
{ "op": "merge", "primary": "memory:<name>", "secondaries": ["memory:<name>", ...], "mergeStrategy": "synthesize", "confidence": 0.95 },
|
|
44
|
-
{ "op": "delete", "ref": "memory:<name>", "reason": "<brief reason>", "confidence": 0.90 },
|
|
45
|
-
{ "op": "promote", "ref": "memory:<name>", "knowledgeRef": "knowledge:<suggested-slug>", "reason": "<brief reason>", "description": "<one sentence describing the new knowledge asset>", "confidence": 0.92 },
|
|
46
|
-
{ "op": "contradict", "ref": "memory:<name>", "contradictedByRef": "memory:<name>", "reason": "<brief reason>", "confidence": 0.88 }
|
|
47
|
-
],
|
|
48
|
-
"warnings": ["<optional concerns>"]
|
|
49
|
-
}
|
|
50
|
-
|
|
51
|
-
For every operation, emit a \`confidence\` field in [0, 1] expressing your certainty that the operation is correct and safe. Use 0.95+ only when evidence is unambiguous. Omit the field rather than guessing if you are uncertain.
|
|
52
|
-
|
|
53
|
-
When the merged content includes an \`updated\` frontmatter field, the value MUST be a real ISO date string (e.g. \`updated: 2026-05-20\`). NEVER emit \`updated: today\`, \`updated: {today}\`, \`updated: {today: null}\`, \`updated: now\`, or any other literal placeholder/template-variable. If you do not have a real source-of-truth date, OMIT the \`updated\` field entirely — the post-processor will not invent one for you.`;
|
|
55
|
+
const CONSOLIDATE_SYSTEM_PROMPT = consolidateSystemPrompt;
|
|
54
56
|
/**
|
|
55
57
|
* JSON Schema for structured consolidate plans (PR 1 of the asset-writers
|
|
56
58
|
* decision — see knowledge:projects/akm/asset-writers-investigation/00-synthesis).
|
|
@@ -136,140 +138,17 @@ export const CONSOLIDATE_PLAN_JSON_SCHEMA = {
|
|
|
136
138
|
},
|
|
137
139
|
},
|
|
138
140
|
};
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
}
|
|
142
|
-
/**
|
|
143
|
-
* Returns true when the memory file has `captureMode: hot` in its frontmatter.
|
|
144
|
-
*
|
|
145
|
-
* Hot memories are USER-EXPLICIT (written via `akm remember` on the hot path).
|
|
146
|
-
* The consolidate LLM is forbidden from deleting or auto-merging them — the
|
|
147
|
-
* user wrote them on purpose and only the user can decide to retire them.
|
|
148
|
-
*
|
|
149
|
-
* Reads the file once per check; consolidate runs against ~10 memories per
|
|
150
|
-
* chunk so the IO cost is trivial. Returns false on any read/parse error
|
|
151
|
-
* (fail-safe: an unparseable file is treated as not-hot, but the broader
|
|
152
|
-
* consolidate flow already guards against unparseable memories elsewhere).
|
|
153
|
-
*
|
|
154
|
-
* Defends against four observed defect classes (see
|
|
155
|
-
* `memory:akm-improve-critical-review-2026-05-20`):
|
|
156
|
-
* - LLM marks a memory contradicted then deletes (dangling contradictedBy)
|
|
157
|
-
* - LLM merges two unrelated memories sharing a topic keyword
|
|
158
|
-
* - LLM judges a recent durable design memo as "redundant"
|
|
159
|
-
* - Cascade deletes (LLM uses ref:X as `contradictedBy` for ref:Y then deletes both)
|
|
160
|
-
*/
|
|
161
|
-
export function isHotCapturedMemory(filePath) {
|
|
162
|
-
try {
|
|
163
|
-
if (!fs.existsSync(filePath))
|
|
164
|
-
return false;
|
|
165
|
-
const content = fs.readFileSync(filePath, "utf8");
|
|
166
|
-
const parsed = parseFrontmatter(content);
|
|
167
|
-
return hasHotCaptureMode(parsed.data);
|
|
168
|
-
}
|
|
169
|
-
catch {
|
|
170
|
-
return false;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
function consolidateGuardStatus(filePath) {
|
|
174
|
-
if (!fs.existsSync(filePath))
|
|
175
|
-
return "missing";
|
|
176
|
-
let content;
|
|
177
|
-
try {
|
|
178
|
-
content = fs.readFileSync(filePath, "utf8");
|
|
179
|
-
}
|
|
180
|
-
catch {
|
|
181
|
-
return "unparseable";
|
|
182
|
-
}
|
|
183
|
-
let parsed;
|
|
184
|
-
try {
|
|
185
|
-
parsed = parseFrontmatter(content);
|
|
186
|
-
}
|
|
187
|
-
catch {
|
|
188
|
-
return "unparseable";
|
|
189
|
-
}
|
|
190
|
-
const data = parsed.data;
|
|
191
|
-
if (!data || Object.keys(data).length === 0)
|
|
192
|
-
return "unparseable";
|
|
193
|
-
return hasHotCaptureMode(data) ? "hot" : "safe";
|
|
194
|
-
}
|
|
195
|
-
// ── Chunk sizing ─────────────────────────────────────────────────────────────
|
|
196
|
-
/**
|
|
197
|
-
* Conservative chars-per-token estimate used when computing prompt budgets.
|
|
198
|
-
* English text averages roughly 4 chars/token for most LLM tokenizers. We use
|
|
199
|
-
* 3 to stay conservative (shorter tokens = more tokens per char).
|
|
200
|
-
*/
|
|
201
|
-
const CHARS_PER_TOKEN = 3;
|
|
202
|
-
/**
|
|
203
|
-
* Overhead budget reserved for the system prompt, chunk header lines, and per-
|
|
204
|
-
* memory metadata lines (name, description, tags, separator). Measured at
|
|
205
|
-
* roughly 600 chars for the system prompt + ~100 chars of header + ~50 chars
|
|
206
|
-
* per memory × chunk size. We round up to 2 000 tokens to leave room for the
|
|
207
|
-
* model's own output.
|
|
208
|
-
*/
|
|
209
|
-
const PROMPT_OVERHEAD_TOKENS = 2_000;
|
|
210
|
-
/**
|
|
211
|
-
* Default effective token budget used when the default LLM profile's
|
|
212
|
-
* `contextLength` is not set. This is intentionally conservative (4 096)
|
|
213
|
-
* rather than being set to the model's actual context window, because:
|
|
214
|
-
*
|
|
215
|
-
* - When the agent path is used, the agent CLI (e.g. opencode)
|
|
216
|
-
* prepends its own large system prompt + conversation history before
|
|
217
|
-
* forwarding to the model. That overhead easily consumes 30K+ tokens on
|
|
218
|
-
* a model with a 16K context window, leaving very little room for
|
|
219
|
-
* chunk content.
|
|
220
|
-
* - When the HTTP path is used (an LLM profile is selected), only the akm
|
|
221
|
-
* system prompt and user prompt are sent, so the budget can be set to the
|
|
222
|
-
* model's actual context length via profiles.llm[defaults.llm].contextLength.
|
|
223
|
-
*
|
|
224
|
-
* Set profiles.llm[defaults.llm].contextLength in your config file to the
|
|
225
|
-
* model's actual context window to allow larger chunks on the HTTP path.
|
|
226
|
-
*/
|
|
227
|
-
export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
|
|
228
|
-
/**
|
|
229
|
-
* Given the model's context window and the per-memory body truncation limit,
|
|
230
|
-
* return the maximum number of memories that can safely fit in one chunk
|
|
231
|
-
* without the prompt overflowing the context window.
|
|
232
|
-
*
|
|
233
|
-
* The formula is:
|
|
234
|
-
* usableTokens = contextLength - PROMPT_OVERHEAD_TOKENS
|
|
235
|
-
* tokensPerMemory = ceil(bodyTruncation / CHARS_PER_TOKEN)
|
|
236
|
-
* chunkSize = floor(usableTokens / tokensPerMemory)
|
|
237
|
-
*
|
|
238
|
-
* Result is clamped between 1 and 50 to avoid degenerate values.
|
|
239
|
-
*
|
|
240
|
-
* @param contextLength - Model context window in tokens.
|
|
241
|
-
* @param bodyTruncation - Max chars per memory body included in the prompt.
|
|
242
|
-
* @param maxChunkSize - Optional override for the hardcoded cap of 50 (1–50).
|
|
243
|
-
*/
|
|
244
|
-
export function computeSafeChunkSize(contextLength, bodyTruncation, maxChunkSize) {
|
|
245
|
-
const usableTokens = Math.max(contextLength - PROMPT_OVERHEAD_TOKENS, 0);
|
|
246
|
-
const tokensPerMemory = Math.max(Math.ceil(bodyTruncation / CHARS_PER_TOKEN), 1);
|
|
247
|
-
const raw = Math.floor(usableTokens / tokensPerMemory);
|
|
248
|
-
return Math.max(1, Math.min(maxChunkSize ?? 50, raw));
|
|
249
|
-
}
|
|
250
|
-
// ── Similarity clustering (C-1 / #380) ──────────────────────────────────────
|
|
251
|
-
/**
|
|
252
|
-
* Re-order memories so that similar ones are placed adjacent to each other
|
|
253
|
-
* before the memories are sliced into chunks. This ensures high-similarity
|
|
254
|
-
* memories land in the same LLM context window, allowing the consolidate
|
|
255
|
-
* model to detect and merge duplicates that would otherwise be split across
|
|
256
|
-
* chunks and survive indefinitely.
|
|
257
|
-
*
|
|
258
|
-
* Algorithm: greedy nearest-neighbour chain starting from the first memory.
|
|
259
|
-
* Each step selects the unused memory with the highest cosine similarity to
|
|
260
|
-
* the last-placed memory. O(n²) — acceptable for the expected N < 200.
|
|
261
|
-
*
|
|
262
|
-
* mem0 arXiv:2504.19413 — every candidate compared against whole store.
|
|
263
|
-
* A-MEM arXiv:2502.12110 — atomic notes linked by similarity.
|
|
264
|
-
*
|
|
265
|
-
* Returns the original order unchanged when:
|
|
266
|
-
* - The embedding config is not present.
|
|
267
|
-
* - Embedding requests fail (fail-open).
|
|
268
|
-
* - There are fewer than 3 memories (no benefit to reordering).
|
|
269
|
-
*/
|
|
270
|
-
async function clusterMemoriesBySimilarity(memories, config) {
|
|
141
|
+
async function clusterMemoriesBySimilarity(memories, config, stateDb) {
|
|
142
|
+
const noTelemetry = { embedMs: 0, cacheHits: 0, cacheMisses: 0 };
|
|
271
143
|
if (memories.length < 3 || !config.embedding)
|
|
272
|
-
return memories;
|
|
144
|
+
return { ordered: memories, embedTelemetry: noTelemetry };
|
|
145
|
+
// WS-3a: cluster uses description+tags as the embedding input (NOT the raw
|
|
146
|
+
// body) — this is intentionally different from the dedup/body cache because
|
|
147
|
+
// the clustering goal is semantic grouping, not dedup twin detection.
|
|
148
|
+
// The body_embeddings cache is keyed by cacheHash(body); clustering inputs
|
|
149
|
+
// are keyed by cacheHash(description+tags text). Re-use the same table with
|
|
150
|
+
// a distinct hash so the two lookup sets never collide.
|
|
151
|
+
const modelId = resolveEmbeddingModelId(config.embedding);
|
|
273
152
|
const texts = memories.map((m) => {
|
|
274
153
|
const parts = [];
|
|
275
154
|
if (m.description)
|
|
@@ -278,16 +157,91 @@ async function clusterMemoriesBySimilarity(memories, config) {
|
|
|
278
157
|
parts.push(m.tags.join(" "));
|
|
279
158
|
return parts.join(". ") || m.name;
|
|
280
159
|
});
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
160
|
+
// Compute content hashes for the cluster texts (not bodies — different input).
|
|
161
|
+
const contentHashes = texts.map((t) => createHash("sha256").update(t, "utf8").digest("hex"));
|
|
162
|
+
// WS-5: track embed cache hits/misses for perf telemetry.
|
|
163
|
+
let embedMs = 0;
|
|
164
|
+
let cacheHits = 0;
|
|
165
|
+
let cacheMisses = 0;
|
|
166
|
+
let cachedVecs = new Map();
|
|
167
|
+
if (stateDb) {
|
|
168
|
+
try {
|
|
169
|
+
cachedVecs = getBodyEmbeddings(stateDb, contentHashes, modelId);
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
// Fail open.
|
|
173
|
+
cachedVecs = new Map();
|
|
174
|
+
}
|
|
284
175
|
}
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
176
|
+
const missIndices = [];
|
|
177
|
+
const missTexts = [];
|
|
178
|
+
for (let i = 0; i < texts.length; i++) {
|
|
179
|
+
if (!cachedVecs.has(contentHashes[i])) {
|
|
180
|
+
missIndices.push(i);
|
|
181
|
+
missTexts.push(texts[i]);
|
|
182
|
+
cacheMisses++;
|
|
183
|
+
}
|
|
184
|
+
else {
|
|
185
|
+
cacheHits++;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
let missVecs = [];
|
|
189
|
+
if (missTexts.length > 0) {
|
|
190
|
+
const embedStart = Date.now();
|
|
191
|
+
try {
|
|
192
|
+
missVecs = await embedBatch(missTexts, config.embedding);
|
|
193
|
+
}
|
|
194
|
+
catch {
|
|
195
|
+
// Fail open: embedding failures degrade gracefully to original order.
|
|
196
|
+
return { ordered: memories, embedTelemetry: { embedMs, cacheHits, cacheMisses } };
|
|
197
|
+
}
|
|
198
|
+
finally {
|
|
199
|
+
embedMs += Date.now() - embedStart;
|
|
200
|
+
}
|
|
201
|
+
// Upsert newly computed vectors into the cache.
|
|
202
|
+
if (stateDb && missVecs.length === missTexts.length) {
|
|
203
|
+
try {
|
|
204
|
+
const toUpsert = missIndices.map((idx, pos) => ({
|
|
205
|
+
contentHash: contentHashes[idx],
|
|
206
|
+
embedding: missVecs[pos],
|
|
207
|
+
modelId,
|
|
208
|
+
}));
|
|
209
|
+
upsertBodyEmbeddings(stateDb, toUpsert);
|
|
210
|
+
}
|
|
211
|
+
catch {
|
|
212
|
+
// Fail open: cache write errors are non-fatal.
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
// Assemble the full embedding array in memories order.
|
|
217
|
+
let embeddings = null;
|
|
218
|
+
{
|
|
219
|
+
const assembled = [];
|
|
220
|
+
let ok = true;
|
|
221
|
+
for (let i = 0; i < memories.length; i++) {
|
|
222
|
+
const hash = contentHashes[i];
|
|
223
|
+
const cached = cachedVecs.get(hash);
|
|
224
|
+
if (cached) {
|
|
225
|
+
assembled.push(cached);
|
|
226
|
+
continue;
|
|
227
|
+
}
|
|
228
|
+
const missPos = missIndices.indexOf(i);
|
|
229
|
+
const vec = missPos >= 0 ? missVecs[missPos] : undefined;
|
|
230
|
+
if (vec) {
|
|
231
|
+
assembled.push(vec);
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
ok = false;
|
|
235
|
+
break;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
if (ok && assembled.length === memories.length) {
|
|
239
|
+
embeddings = assembled;
|
|
240
|
+
}
|
|
288
241
|
}
|
|
242
|
+
const embedTelemetry = { embedMs, cacheHits, cacheMisses };
|
|
289
243
|
if (!embeddings || embeddings.length !== memories.length)
|
|
290
|
-
return memories;
|
|
244
|
+
return { ordered: memories, embedTelemetry };
|
|
291
245
|
// Greedy nearest-neighbour chain.
|
|
292
246
|
const used = new Array(memories.length).fill(false);
|
|
293
247
|
const ordered = [];
|
|
@@ -313,87 +267,16 @@ async function clusterMemoriesBySimilarity(memories, config) {
|
|
|
313
267
|
used[bestIdx] = true;
|
|
314
268
|
current = bestIdx;
|
|
315
269
|
}
|
|
316
|
-
return ordered;
|
|
270
|
+
return { ordered, embedTelemetry };
|
|
317
271
|
}
|
|
318
272
|
// ── Chunk helpers ────────────────────────────────────────────────────────────
|
|
319
|
-
/**
|
|
320
|
-
* Build the per-chunk user prompt fed to the consolidate LLM.
|
|
321
|
-
*
|
|
322
|
-
* Each memory is annotated with two flags that drive the system-prompt
|
|
323
|
-
* rules at lines 181-186:
|
|
324
|
-
* - `(captureMode: hot)` — user-explicit memory; system prompt rule 2
|
|
325
|
-
* forbids proposing delete. ~60 wasted LLM verdicts/4h on this user's
|
|
326
|
-
* stack before this annotation.
|
|
327
|
-
* - `(already queued)` — the memory's body hash matches a pending
|
|
328
|
-
* consolidate proposal; system prompt rule 3 forbids proposing
|
|
329
|
-
* promote/merge/contradict. ~107/4h before this annotation.
|
|
330
|
-
*
|
|
331
|
-
* Both annotations are visible to the LLM. `pendingProposalBodyHashes`
|
|
332
|
-
* is precomputed once per run by `loadPendingConsolidateProposalHashes`
|
|
333
|
-
* so the cost stays O(memories) inside the chunk loop.
|
|
334
|
-
*/
|
|
335
|
-
export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks, bodyTruncation, pendingProposalBodyHashes = new Set()) {
|
|
336
|
-
const start = memories[0] ? `memory:${memories[0].name}` : "";
|
|
337
|
-
const end = memories[memories.length - 1] ? `memory:${memories[memories.length - 1].name}` : "";
|
|
338
|
-
const annotationsByIndex = [];
|
|
339
|
-
const hotRefs = [];
|
|
340
|
-
for (const m of memories) {
|
|
341
|
-
let body = "";
|
|
342
|
-
try {
|
|
343
|
-
body = fs.readFileSync(m.filePath, "utf8");
|
|
344
|
-
}
|
|
345
|
-
catch {
|
|
346
|
-
body = "(unreadable)";
|
|
347
|
-
}
|
|
348
|
-
const parsed = parseFrontmatter(body);
|
|
349
|
-
const isHot = parsed.data.captureMode === "hot";
|
|
350
|
-
const bodyHash = createHash("sha256").update(parsed.content.trim(), "utf8").digest("hex");
|
|
351
|
-
const isAlreadyQueued = pendingProposalBodyHashes.has(bodyHash);
|
|
352
|
-
annotationsByIndex.push({ isHot, isAlreadyQueued, body });
|
|
353
|
-
if (isHot)
|
|
354
|
-
hotRefs.push(`memory:${m.name}`);
|
|
355
|
-
}
|
|
356
|
-
const lines = [
|
|
357
|
-
`Source: ${sourceName}`,
|
|
358
|
-
`Chunk ${chunkIndex + 1} of ${totalChunks}, memories ${start}–${end}:`,
|
|
359
|
-
"",
|
|
360
|
-
];
|
|
361
|
-
// Top-of-prompt protection block for hot refs. Neutral phrasing — avoid
|
|
362
|
-
// op-words like "promote", "merge", "contradict" so the model doesn't
|
|
363
|
-
// accidentally treat the warning as a hint to use that op elsewhere
|
|
364
|
-
// (variant B leaked the word "contradict" into the control sample
|
|
365
|
-
// during the diagnostic).
|
|
366
|
-
if (hotRefs.length > 0) {
|
|
367
|
-
lines.push("⛔ DO NOT propose any `delete` operation for these refs — they are user-explicit (captureMode: hot) and the downstream guard refuses them regardless. Proposing delete for any of these only wastes tokens.");
|
|
368
|
-
for (const ref of hotRefs)
|
|
369
|
-
lines.push(` - ${ref}`);
|
|
370
|
-
lines.push("");
|
|
371
|
-
}
|
|
372
|
-
for (let i = 0; i < memories.length; i++) {
|
|
373
|
-
const m = memories[i];
|
|
374
|
-
const { isHot, isAlreadyQueued, body } = annotationsByIndex[i];
|
|
375
|
-
const annotations = [];
|
|
376
|
-
if (isHot)
|
|
377
|
-
annotations.push("captureMode: hot");
|
|
378
|
-
if (isAlreadyQueued)
|
|
379
|
-
annotations.push("already queued");
|
|
380
|
-
const annotationSuffix = annotations.length > 0 ? ` (${annotations.join("; ")})` : "";
|
|
381
|
-
lines.push(`[${i + 1}] memory:${m.name}${annotationSuffix}`);
|
|
382
|
-
lines.push(`Description: ${m.description || "(none)"}`);
|
|
383
|
-
lines.push(`Tags: ${m.tags.length > 0 ? m.tags.join(", ") : "(none)"}`);
|
|
384
|
-
lines.push("---");
|
|
385
|
-
lines.push(body.slice(0, bodyTruncation));
|
|
386
|
-
lines.push("");
|
|
387
|
-
}
|
|
388
|
-
return lines.join("\n");
|
|
389
|
-
}
|
|
390
273
|
/**
|
|
391
274
|
* Precompute body-hashes of all currently-pending consolidate proposals so
|
|
392
275
|
* the per-chunk prompt can annotate memories whose body would just produce
|
|
393
|
-
* a deterministic `dedup_pending_proposal` skip.
|
|
394
|
-
*
|
|
395
|
-
*
|
|
396
|
-
* nothing" so the LLM still proposes
|
|
276
|
+
* a deterministic `dedup_pending_proposal` skip. Uses `cacheHash` (case-
|
|
277
|
+
* preserving stripped body) — the same domain used by the body-embedding
|
|
278
|
+
* cache and `computeMemoryContentHash`. Empty set on any read/parse error
|
|
279
|
+
* — fail-safe to "annotate nothing" so the LLM still proposes.
|
|
397
280
|
*/
|
|
398
281
|
function loadPendingConsolidateProposalHashes(stashDir) {
|
|
399
282
|
const hashes = new Set();
|
|
@@ -401,8 +284,7 @@ function loadPendingConsolidateProposalHashes(stashDir) {
|
|
|
401
284
|
const pending = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
|
|
402
285
|
for (const p of pending) {
|
|
403
286
|
try {
|
|
404
|
-
|
|
405
|
-
hashes.add(createHash("sha256").update(body, "utf8").digest("hex"));
|
|
287
|
+
hashes.add(cacheHash(p.payload.content));
|
|
406
288
|
}
|
|
407
289
|
catch {
|
|
408
290
|
// skip malformed payloads — they can't dedup anyway
|
|
@@ -414,148 +296,6 @@ function loadPendingConsolidateProposalHashes(stashDir) {
|
|
|
414
296
|
}
|
|
415
297
|
return hashes;
|
|
416
298
|
}
|
|
417
|
-
function isValidOp(op) {
|
|
418
|
-
if (typeof op !== "object" || op === null)
|
|
419
|
-
return false;
|
|
420
|
-
const o = op;
|
|
421
|
-
if (o.op === "merge") {
|
|
422
|
-
return typeof o.primary === "string" && Array.isArray(o.secondaries);
|
|
423
|
-
}
|
|
424
|
-
if (o.op === "delete") {
|
|
425
|
-
return typeof o.ref === "string";
|
|
426
|
-
}
|
|
427
|
-
if (o.op === "promote") {
|
|
428
|
-
return typeof o.ref === "string" && typeof o.knowledgeRef === "string";
|
|
429
|
-
}
|
|
430
|
-
if (o.op === "contradict") {
|
|
431
|
-
return typeof o.ref === "string" && typeof o.contradictedByRef === "string";
|
|
432
|
-
}
|
|
433
|
-
return false;
|
|
434
|
-
}
|
|
435
|
-
export function mergePlans(chunks, knownRefs) {
|
|
436
|
-
const mergeOps = new Map();
|
|
437
|
-
const deleteOps = new Map();
|
|
438
|
-
const promoteOps = new Map();
|
|
439
|
-
// C-3 / #382: contradict ops keyed by `ref|contradictedByRef` to deduplicate.
|
|
440
|
-
const contradictOps = new Map();
|
|
441
|
-
const warnings = [];
|
|
442
|
-
for (const chunk of chunks) {
|
|
443
|
-
for (const op of chunk) {
|
|
444
|
-
if (op.op === "merge") {
|
|
445
|
-
// Drop ops whose primary the LLM hallucinated (not in the loaded memory
|
|
446
|
-
// pool). Without this guard, a hallucinated primary flows all the way to
|
|
447
|
-
// Phase B where !memoryByRef.has(primary) fires and charges every real
|
|
448
|
-
// secondary with merge_primary_missing — masking LLM hallucinations as
|
|
449
|
-
// filter regressions in health metrics.
|
|
450
|
-
if (knownRefs && !knownRefs.has(op.primary)) {
|
|
451
|
-
warnings.push(`mergePlans: primary ${op.primary} not in loaded memory pool (LLM hallucination) — dropping op before execution.`);
|
|
452
|
-
// Use a dedicated skip reason so dashboards can distinguish
|
|
453
|
-
// hallucinated primaries from stale-DB regressions.
|
|
454
|
-
// Secondaries are real refs; they are NOT charged here — they remain
|
|
455
|
-
// available for other ops to claim.
|
|
456
|
-
continue;
|
|
457
|
-
}
|
|
458
|
-
// Filter hallucinated secondaries while preserving real ones.
|
|
459
|
-
let mergeOp = op;
|
|
460
|
-
if (knownRefs) {
|
|
461
|
-
const filteredSecondaries = op.secondaries.filter((sec) => {
|
|
462
|
-
if (!knownRefs.has(sec)) {
|
|
463
|
-
warnings.push(`mergePlans: secondary ${sec} not in loaded memory pool (LLM hallucination) — dropping from op.`);
|
|
464
|
-
return false;
|
|
465
|
-
}
|
|
466
|
-
return true;
|
|
467
|
-
});
|
|
468
|
-
if (filteredSecondaries.length !== op.secondaries.length) {
|
|
469
|
-
mergeOp = { ...op, secondaries: filteredSecondaries };
|
|
470
|
-
}
|
|
471
|
-
}
|
|
472
|
-
// merge wins over delete
|
|
473
|
-
if (deleteOps.has(mergeOp.primary)) {
|
|
474
|
-
deleteOps.delete(mergeOp.primary);
|
|
475
|
-
}
|
|
476
|
-
for (const sec of mergeOp.secondaries) {
|
|
477
|
-
if (deleteOps.has(sec))
|
|
478
|
-
deleteOps.delete(sec);
|
|
479
|
-
}
|
|
480
|
-
mergeOps.set(mergeOp.primary, mergeOp);
|
|
481
|
-
}
|
|
482
|
-
else if (op.op === "delete") {
|
|
483
|
-
// merge and promote both win over delete. A promote is non-destructive
|
|
484
|
-
// (creates a proposal) but the source memory is counted in `promoted`;
|
|
485
|
-
// if a delete also fires, the ref lands in both `promoted` and
|
|
486
|
-
// `skipReasons`, breaking the invariant by +1.
|
|
487
|
-
if (!mergeOps.has(op.ref) && !promoteOps.has(op.ref)) {
|
|
488
|
-
deleteOps.set(op.ref, op);
|
|
489
|
-
}
|
|
490
|
-
}
|
|
491
|
-
else if (op.op === "promote") {
|
|
492
|
-
// C-2 / #381: when both a promote and a merge target the same ref,
|
|
493
|
-
// queue the promote FIRST rather than discarding it. The promote op
|
|
494
|
-
// routes through createProposal (the human-gated proposal queue), so
|
|
495
|
-
// it is non-destructive. The merge follows after the proposal is
|
|
496
|
-
// created. This preserves the human reviewer's ability to inspect the
|
|
497
|
-
// promotion before the source memory is merged/deleted.
|
|
498
|
-
// AGM K*8 — retain the maximally informative consistent subset.
|
|
499
|
-
promoteOps.set(op.ref, op);
|
|
500
|
-
}
|
|
501
|
-
else if (op.op === "contradict") {
|
|
502
|
-
// Deduplicate by ref+contradictedByRef pair.
|
|
503
|
-
const key = `${op.ref}|${op.contradictedByRef}`;
|
|
504
|
-
if (!contradictOps.has(key)) {
|
|
505
|
-
contradictOps.set(key, op);
|
|
506
|
-
}
|
|
507
|
-
}
|
|
508
|
-
}
|
|
509
|
-
}
|
|
510
|
-
// Second pass: enforce merge-wins-over-delete and deduplicate secondaries.
|
|
511
|
-
//
|
|
512
|
-
// 1. Delete/secondary ordering bug: the per-chunk loop removes delete ops
|
|
513
|
-
// for secondaries that were already in deleteOps, but misses the case
|
|
514
|
-
// where the delete chunk came first. A full sweep here fixes both orders.
|
|
515
|
-
//
|
|
516
|
-
// 2. Cross-merge secondary dedup: if ref A is a secondary in two merge ops,
|
|
517
|
-
// only the first (insertion-order) retains it. Without this, a successful
|
|
518
|
-
// merge credits A to mergedSecondaries and a later merge's emitMerge-
|
|
519
|
-
// FailureSkips also charges A to skipReasons — double-counting A while
|
|
520
|
-
// processed has it only once.
|
|
521
|
-
//
|
|
522
|
-
// 3. Primary-as-secondary dedup: if ref A is a primary in one merge op and
|
|
523
|
-
// a secondary in another, remove A from the secondary list. Both merges
|
|
524
|
-
// would otherwise claim A (merged++ for A, then mergedSecondaries++ for A)
|
|
525
|
-
// breaking the invariant the same way.
|
|
526
|
-
// Also remove delete ops for any ref claimed by a promote op (handles the
|
|
527
|
-
// case where the delete chunk appeared before the promote chunk).
|
|
528
|
-
for (const ref of promoteOps.keys()) {
|
|
529
|
-
deleteOps.delete(ref);
|
|
530
|
-
}
|
|
531
|
-
const claimedSecondaries = new Set();
|
|
532
|
-
for (const mergeOp of mergeOps.values()) {
|
|
533
|
-
deleteOps.delete(mergeOp.primary);
|
|
534
|
-
mergeOp.secondaries = mergeOp.secondaries.filter((sec) => {
|
|
535
|
-
if (mergeOps.has(sec)) {
|
|
536
|
-
warnings.push(`Merge: secondary ${sec} is also a merge primary — removing from secondary list to avoid double-count.`);
|
|
537
|
-
return false;
|
|
538
|
-
}
|
|
539
|
-
if (claimedSecondaries.has(sec)) {
|
|
540
|
-
warnings.push(`Merge: secondary ${sec} appears in multiple merge ops — retaining in first op only.`);
|
|
541
|
-
return false;
|
|
542
|
-
}
|
|
543
|
-
claimedSecondaries.add(sec);
|
|
544
|
-
deleteOps.delete(sec);
|
|
545
|
-
return true;
|
|
546
|
-
});
|
|
547
|
-
}
|
|
548
|
-
// C-2 / #381: promote ops are ordered BEFORE merge ops so that the
|
|
549
|
-
// human-gated proposal queue entry is created before any destructive merge.
|
|
550
|
-
// Phase B processes ops in array order, so promote executes first.
|
|
551
|
-
const ops = [
|
|
552
|
-
...promoteOps.values(),
|
|
553
|
-
...mergeOps.values(),
|
|
554
|
-
...deleteOps.values(),
|
|
555
|
-
...contradictOps.values(),
|
|
556
|
-
];
|
|
557
|
-
return { ops, warnings };
|
|
558
|
-
}
|
|
559
299
|
function getJournalPath(stashDir) {
|
|
560
300
|
return path.join(stashDir, ".akm", "consolidate-journal.json");
|
|
561
301
|
}
|
|
@@ -672,6 +412,32 @@ function backupFile(filePath, backupDir, name) {
|
|
|
672
412
|
// best-effort
|
|
673
413
|
}
|
|
674
414
|
}
|
|
415
|
+
// ── WS-3b: Generation frontmatter injection ───────────────────────────────────
|
|
416
|
+
/**
|
|
417
|
+
* Inject `generation` and `source_refs` into merged content.
|
|
418
|
+
* generation = max(sourceGenerations) + 1.
|
|
419
|
+
* source_refs = UNION of the provided provenance refs (participants + their
|
|
420
|
+
* cited sources) with anything already present in the merged frontmatter —
|
|
421
|
+
* R5 §4.2: the old set-if-absent behavior dropped second-generation
|
|
422
|
+
* provenance whenever the LLM emitted its own (partial) source_refs.
|
|
423
|
+
* Fails open — returns original content if frontmatter can't be parsed.
|
|
424
|
+
*/
|
|
425
|
+
function injectGenerationFrontmatter(mergedContent, sourceGenerations, provenanceRefs) {
|
|
426
|
+
try {
|
|
427
|
+
const parsed = parseFrontmatter(mergedContent);
|
|
428
|
+
const existingFm = parsed.data;
|
|
429
|
+
const existingRefs = Array.isArray(existingFm.source_refs) ? existingFm.source_refs.map(String) : [];
|
|
430
|
+
const updatedFm = {
|
|
431
|
+
...existingFm,
|
|
432
|
+
generation: computeMergedGeneration(sourceGenerations),
|
|
433
|
+
source_refs: [...new Set([...existingRefs, ...provenanceRefs])],
|
|
434
|
+
};
|
|
435
|
+
return assembleAssetFromString(serializeFrontmatter(updatedFm), parsed.content);
|
|
436
|
+
}
|
|
437
|
+
catch {
|
|
438
|
+
return mergedContent; // fail open
|
|
439
|
+
}
|
|
440
|
+
}
|
|
675
441
|
// ── Archive helper (P1-B: soft-invalidation) ─────────────────────────────────
|
|
676
442
|
/**
|
|
677
443
|
* Move a memory asset to `.akm/archive/` with `status: superseded` frontmatter
|
|
@@ -742,8 +508,8 @@ function archiveMemory(filePath, stashDir, ref, reason, opIndex, supersededBy, w
|
|
|
742
508
|
* silent 400s from LM Studio). The investigation lives at
|
|
743
509
|
* `/tmp/akm-health-investigations/consolidation-no-op.md`.
|
|
744
510
|
*/
|
|
745
|
-
function resolveConsolidateLlmConfig(config) {
|
|
746
|
-
const consolidateProcess = config
|
|
511
|
+
function resolveConsolidateLlmConfig(config, activeProfile) {
|
|
512
|
+
const consolidateProcess = getImproveProcessConfig(config, "consolidate", activeProfile);
|
|
747
513
|
const runnerSpec = resolveImproveProcessRunnerFromProfile(consolidateProcess, config);
|
|
748
514
|
if (runnerSpec && runnerIsLlm(runnerSpec)) {
|
|
749
515
|
return runnerSpec.connection;
|
|
@@ -752,6 +518,58 @@ function resolveConsolidateLlmConfig(config) {
|
|
|
752
518
|
// fall back to the default LLM profile rather than disabling the pass.
|
|
753
519
|
return getDefaultLlmConfig(config);
|
|
754
520
|
}
|
|
521
|
+
// ── Judged-state cache (#581) ────────────────────────────────────────────────
|
|
522
|
+
/**
|
|
523
|
+
* Stable content hash for a memory file used by the judged-state cache (#581)
|
|
524
|
+
* and the body-embedding cache (WS-3a). Uses `cacheHash` from dedup.ts
|
|
525
|
+
* (sha256 of the case-preserving stripped body) plus the sorted `tags` list,
|
|
526
|
+
* so semantic-metadata drift re-enters the judge while cosmetic frontmatter
|
|
527
|
+
* touches (`updated:`, `inferenceProcessed:`) still hash identically and never
|
|
528
|
+
* force a needless re-judge. Returns `undefined` on any read/parse error so
|
|
529
|
+
* callers fail open (treat the memory as un-cached → it stays in the LLM pool).
|
|
530
|
+
*/
|
|
531
|
+
function computeMemoryContentHash(filePath) {
|
|
532
|
+
try {
|
|
533
|
+
const raw = fs.readFileSync(filePath, "utf8");
|
|
534
|
+
let tagSuffix = "";
|
|
535
|
+
try {
|
|
536
|
+
const { data } = parseFrontmatter(raw);
|
|
537
|
+
const tags = Array.isArray(data?.tags) ? data.tags.map(String).sort() : [];
|
|
538
|
+
if (tags.length > 0)
|
|
539
|
+
tagSuffix = `\n\u0000tags:${tags.join(",")}`;
|
|
540
|
+
}
|
|
541
|
+
catch {
|
|
542
|
+
// Unparseable frontmatter → body-only hash (prior behaviour).
|
|
543
|
+
}
|
|
544
|
+
return cacheHash(raw + tagSuffix);
|
|
545
|
+
}
|
|
546
|
+
catch {
|
|
547
|
+
return undefined;
|
|
548
|
+
}
|
|
549
|
+
}
|
|
550
|
+
/**
|
|
551
|
+
* Build a {@link ConsolidateResult} from partial overrides, filling the envelope
|
|
552
|
+
* defaults (schemaVersion / ok / shape + the zeroed counters). Collapses the
|
|
553
|
+
* ~7 near-identical result literals that previously appeared verbatim at every
|
|
554
|
+
* early-return site and the final return of `akmConsolidateInner`. Callers pass
|
|
555
|
+
* only the fields that differ from the all-zero, ok, non-preview baseline.
|
|
556
|
+
*/
|
|
557
|
+
export function makeConsolidateResult(overrides) {
|
|
558
|
+
return {
|
|
559
|
+
schemaVersion: 1,
|
|
560
|
+
ok: true,
|
|
561
|
+
shape: "consolidate-result",
|
|
562
|
+
dryRun: false,
|
|
563
|
+
previewOnly: false,
|
|
564
|
+
processed: 0,
|
|
565
|
+
merged: 0,
|
|
566
|
+
deleted: 0,
|
|
567
|
+
promoted: [],
|
|
568
|
+
contradicted: 0,
|
|
569
|
+
warnings: [],
|
|
570
|
+
...overrides,
|
|
571
|
+
};
|
|
572
|
+
}
|
|
755
573
|
// ── Main entry point ─────────────────────────────────────────────────────────
|
|
756
574
|
export async function akmConsolidate(opts = {}) {
|
|
757
575
|
const startMs = Date.now();
|
|
@@ -762,24 +580,75 @@ export async function akmConsolidate(opts = {}) {
|
|
|
762
580
|
const config = opts.config ?? loadConfig();
|
|
763
581
|
const stashDir = opts.stashDir ?? resolveStashDir();
|
|
764
582
|
if (!isLlmFeatureEnabled(config, "memory_consolidation")) {
|
|
765
|
-
return {
|
|
766
|
-
schemaVersion: 1,
|
|
767
|
-
ok: true,
|
|
768
|
-
shape: "consolidate-result",
|
|
583
|
+
return makeConsolidateResult({
|
|
769
584
|
dryRun: opts.dryRun ?? false,
|
|
770
|
-
previewOnly: false,
|
|
771
585
|
target: opts.target ?? stashDir,
|
|
772
|
-
processed: 0,
|
|
773
|
-
merged: 0,
|
|
774
|
-
deleted: 0,
|
|
775
|
-
promoted: [],
|
|
776
|
-
contradicted: 0,
|
|
777
|
-
warnings: [],
|
|
778
586
|
durationMs: Date.now() - startMs,
|
|
779
|
-
};
|
|
587
|
+
});
|
|
780
588
|
}
|
|
781
589
|
const warnings = [];
|
|
782
590
|
checkForIncompleteJournal(stashDir, opts.recoveryMode ?? "abort", warnings);
|
|
591
|
+
// WS-3a: open one state.db handle shared by the body-embedding cache (dedup
|
|
592
|
+
// + cluster) and the judged-state cache. All callers in the function body
|
|
593
|
+
// receive this handle; it is closed in the `finally` block below.
|
|
594
|
+
// Fail-open: any open error leaves it `undefined` and all cache paths skip.
|
|
595
|
+
let sharedStateDb;
|
|
596
|
+
try {
|
|
597
|
+
sharedStateDb = openStateDatabase();
|
|
598
|
+
}
|
|
599
|
+
catch {
|
|
600
|
+
// State DB unavailable → skip the embedding cache for this run.
|
|
601
|
+
}
|
|
602
|
+
try {
|
|
603
|
+
return await akmConsolidateInner(opts, config, stashDir, startMs, sourceRun, warnings, sharedStateDb);
|
|
604
|
+
}
|
|
605
|
+
finally {
|
|
606
|
+
sharedStateDb?.close();
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
/** Fresh, zeroed accounting accumulators for one consolidate run. */
|
|
610
|
+
function createConsolidateAccounting() {
|
|
611
|
+
const acc = {
|
|
612
|
+
judgedNoAction: 0,
|
|
613
|
+
failedChunkMemories: 0,
|
|
614
|
+
totalChunksFailed: 0,
|
|
615
|
+
skipReasons: [],
|
|
616
|
+
skipReasonByRef: new Map(),
|
|
617
|
+
judgedNoActionRefs: new Set(),
|
|
618
|
+
pushSkipReason: () => { },
|
|
619
|
+
};
|
|
620
|
+
acc.pushSkipReason = (op, ref, reason) => {
|
|
621
|
+
// 2026-05-27 cross-chunk double-count fix: if `ref` already contributed
|
|
622
|
+
// to judgedNoAction in its own chunk (a different chunk proposed an op
|
|
623
|
+
// for it that is now being rejected here), promote it from the
|
|
624
|
+
// judgedNoAction bucket into the more specific skipReason bucket.
|
|
625
|
+
// Preserves the invariant: processed == actioned + judgedNoAction +
|
|
626
|
+
// Σ(skipReasons) + failedChunkMemories.
|
|
627
|
+
if (acc.judgedNoActionRefs.delete(ref))
|
|
628
|
+
acc.judgedNoAction--;
|
|
629
|
+
const existing = acc.skipReasonByRef.get(ref);
|
|
630
|
+
if (existing) {
|
|
631
|
+
// Already counted once for accounting. Append the extra skip to the
|
|
632
|
+
// ref's grouped entry for observability without adding a new array
|
|
633
|
+
// entry (which would break the accounting invariant).
|
|
634
|
+
existing.skips.push({ op, reason });
|
|
635
|
+
return;
|
|
636
|
+
}
|
|
637
|
+
const entry = { ref, skips: [{ op, reason }] };
|
|
638
|
+
acc.skipReasonByRef.set(ref, entry);
|
|
639
|
+
acc.skipReasons.push(entry);
|
|
640
|
+
};
|
|
641
|
+
return acc;
|
|
642
|
+
}
|
|
643
|
+
/**
|
|
644
|
+
* Pass 1 — narrow the memory pool before any LLM work: drop stale DB entries,
|
|
645
|
+
* partition hot-probation assets, run the deterministic dedup pre-pass, apply
|
|
646
|
+
* incremental-since and judged-state-cache narrowing, and cap to `opts.limit`
|
|
647
|
+
* (oldest-modified first). Returns an early envelope when the pool empties at
|
|
648
|
+
* any stage; otherwise returns the narrowed pool and the state the plan/apply
|
|
649
|
+
* passes consume. Behavior-identical to the former inlined narrowing block.
|
|
650
|
+
*/
|
|
651
|
+
async function narrowConsolidationPool(opts, config, stashDir, startMs, warnings, sharedStateDb) {
|
|
783
652
|
let memories = loadMemoriesForSource(opts.target, stashDir, warnings);
|
|
784
653
|
// Pre-flight: filter out stale DB entries whose files no longer exist on
|
|
785
654
|
// disk. Without this, memories deleted by a prior run (but not yet
|
|
@@ -791,43 +660,177 @@ export async function akmConsolidate(opts = {}) {
|
|
|
791
660
|
warnings.push(`Pre-flight: filtered ${staleCount} stale DB entr${staleCount === 1 ? "y" : "ies"} (file absent on disk) from memory pool before chunking.`);
|
|
792
661
|
}
|
|
793
662
|
memories = memories.filter((m) => fs.existsSync(m.filePath));
|
|
663
|
+
// (The former WS-3b Step 0a homeostatic demotion pass was removed — R4:
|
|
664
|
+
// it was default-off and self-undoing (the next salience recompute
|
|
665
|
+
// unconditionally overwrote the demoted values). Continuous decay now lives
|
|
666
|
+
// in computeSalience's recency term, whose floor decays on a long half-life.)
|
|
667
|
+
// ── WS-3b Step 0c: Filter hot-probation assets from LLM merge pool ─────────
|
|
668
|
+
// Hot-probation assets (system-generated, not yet graduated from intake pass)
|
|
669
|
+
// are processed by the dedup pre-pass but excluded from the LLM clustering.
|
|
670
|
+
// This prevents noisy extractions from polluting LLM context. The dedup pass
|
|
671
|
+
// below still runs against them so they're cleaned up deterministically.
|
|
672
|
+
// DEFAULT OFF — only active when `processes.extract.hotProbation.enabled === true`
|
|
673
|
+
// (the flag that causes extract to tag new extractions as hot-probation).
|
|
674
|
+
// Without that flag no assets will ever carry the hot-probation marker, so
|
|
675
|
+
// running the filter loop would be pure unnecessary I/O over the full corpus.
|
|
676
|
+
const hotProbationEnabled = getImproveProcessConfig(config, "extract", opts.improveProfile)?.hotProbation
|
|
677
|
+
?.enabled === true;
|
|
678
|
+
let hotProbationCount = 0;
|
|
679
|
+
if (hotProbationEnabled) {
|
|
680
|
+
const hotProbationMemories = [];
|
|
681
|
+
const nonProbationMemories = [];
|
|
682
|
+
for (const m of memories) {
|
|
683
|
+
try {
|
|
684
|
+
const raw = fs.readFileSync(m.filePath, "utf8");
|
|
685
|
+
const parsed = parseFrontmatter(raw);
|
|
686
|
+
if (shouldSkipHotProbationInLlm(parsed.data)) {
|
|
687
|
+
hotProbationMemories.push(m);
|
|
688
|
+
hotProbationCount++;
|
|
689
|
+
}
|
|
690
|
+
else {
|
|
691
|
+
nonProbationMemories.push(m);
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
catch {
|
|
695
|
+
nonProbationMemories.push(m); // fail open
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
if (hotProbationCount > 0) {
|
|
699
|
+
warnings.push(`Hot-probation: ${hotProbationCount} hot-probation asset(s) routed to dedup-only pass (excluded from LLM merge pool).`);
|
|
700
|
+
memories = nonProbationMemories;
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
// ── Deterministic dedup pre-pass (#617) ─────────────────────────────────────
|
|
704
|
+
// Cheap, no-LLM fast path that collapses the obvious near-duplicates
|
|
705
|
+
// (`.derived` ↔ origin pairs + content twins) BEFORE the embedding-clustered
|
|
706
|
+
// LLM consolidation. DEFAULT OFF — when `dedup.enabled !== true` this is a
|
|
707
|
+
// no-op and the pass behaves byte-identically to today. Collapsed variants
|
|
708
|
+
// are pruned from the LLM pool so the model only ever sees genuinely
|
|
709
|
+
// distinct-but-related memories. Each dropped variant is archived (soft
|
|
710
|
+
// invalidation) before deletion, matching the LLM merge path.
|
|
711
|
+
// Dry-run never mutates the filesystem, so the dedup pre-pass is skipped
|
|
712
|
+
// entirely under `--dry-run` (the LLM plan preview below is unaffected).
|
|
713
|
+
let dedupCollapsed = 0;
|
|
714
|
+
if (opts.dedup?.enabled && !opts.dryRun) {
|
|
715
|
+
const dedupTimestamp = timestampForFilename();
|
|
716
|
+
const dedupResult = await runDeterministicDedup(stashDir, opts.dedup, config, (variantFilePath, variantName) => {
|
|
717
|
+
archiveMemory(variantFilePath, stashDir, `memory:${variantName}`, "collapsed by deterministic dedup pre-pass", -1, undefined, warnings);
|
|
718
|
+
backupFile(variantFilePath, getBackupDir(stashDir, dedupTimestamp), variantName);
|
|
719
|
+
}, opts.signal, sharedStateDb);
|
|
720
|
+
dedupCollapsed = dedupResult.collapsed;
|
|
721
|
+
warnings.push(...dedupResult.warnings);
|
|
722
|
+
if (dedupResult.consumedRefs.length > 0) {
|
|
723
|
+
const consumed = new Set(dedupResult.consumedRefs);
|
|
724
|
+
memories = memories.filter((m) => !consumed.has(`memory:${m.name}`));
|
|
725
|
+
warnings.push(`Deterministic dedup: collapsed ${dedupResult.collapsed} near-duplicate memor${dedupResult.collapsed === 1 ? "y" : "ies"} (no LLM) before chunking.`);
|
|
726
|
+
}
|
|
727
|
+
}
|
|
794
728
|
if (memories.length === 0) {
|
|
795
729
|
return {
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
warnings,
|
|
808
|
-
durationMs: Date.now() - startMs,
|
|
730
|
+
done: true,
|
|
731
|
+
result: makeConsolidateResult({
|
|
732
|
+
dryRun: opts.dryRun ?? false,
|
|
733
|
+
target: opts.target ?? stashDir,
|
|
734
|
+
// #617: the deterministic dedup pre-pass may have emptied the pool by
|
|
735
|
+
// collapsing every remaining memory into a canonical. Surface those
|
|
736
|
+
// collapses in `deleted` so the run reports the work it actually did.
|
|
737
|
+
deleted: dedupCollapsed,
|
|
738
|
+
warnings,
|
|
739
|
+
durationMs: Date.now() - startMs,
|
|
740
|
+
}),
|
|
809
741
|
};
|
|
810
742
|
}
|
|
811
743
|
if (opts.incrementalSince) {
|
|
812
744
|
memories = narrowToIncrementalCandidates(memories, opts.incrementalSince, warnings, opts.neighborsPerChanged);
|
|
813
745
|
if (memories.length === 0) {
|
|
814
746
|
return {
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
747
|
+
done: true,
|
|
748
|
+
result: makeConsolidateResult({
|
|
749
|
+
dryRun: opts.dryRun ?? false,
|
|
750
|
+
target: opts.target ?? stashDir,
|
|
751
|
+
warnings,
|
|
752
|
+
durationMs: Date.now() - startMs,
|
|
753
|
+
}),
|
|
754
|
+
};
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
// WS-5 perf telemetry accumulators. These are collected throughout the run and
|
|
758
|
+
// merged into `perfTelemetry` on the final ConsolidateResult.
|
|
759
|
+
// `dedupPoolSize` = memories entering judgedCache narrowing (after dedup+incremental+limit).
|
|
760
|
+
// `judgedCacheSkipped` = memories skipped by the cache.
|
|
761
|
+
// `llmPoolSize` = memories actually sent to the LLM.
|
|
762
|
+
// `embedMs/cacheHits/cacheMisses` = accumulated from clusterMemoriesBySimilarity.
|
|
763
|
+
const perfMs = { dedupPoolSize: memories.length, judgedCacheSkipped: 0 };
|
|
764
|
+
// ── Judged-state cache narrowing (#581) ─────────────────────────────────────
|
|
765
|
+
// DEFAULT OFF. When enabled, skip every memory whose current content hash
|
|
766
|
+
// equals the hash recorded the last time the consolidate LLM judged it
|
|
767
|
+
// (judged-unchanged → no re-judge). This converts coverage from O(window) to
|
|
768
|
+
// O(changed/new) so one run can sweep the whole corpus while the LLM only
|
|
769
|
+
// sees genuinely new/changed memories. `currentHashByName` is populated for
|
|
770
|
+
// EVERY surviving memory (whether or not the cache is on) so the post-LLM
|
|
771
|
+
// recording step can upsert judged state without re-reading the files; when
|
|
772
|
+
// the cache is off it stays empty and the recording step is a no-op.
|
|
773
|
+
const judgedCacheEnabled = opts.judgedCache?.enabled !== false;
|
|
774
|
+
const currentHashByName = new Map();
|
|
775
|
+
if (judgedCacheEnabled) {
|
|
776
|
+
for (const m of memories) {
|
|
777
|
+
const h = computeMemoryContentHash(m.filePath);
|
|
778
|
+
if (h !== undefined)
|
|
779
|
+
currentHashByName.set(m.name, h);
|
|
780
|
+
}
|
|
781
|
+
let cachedMap = new Map();
|
|
782
|
+
{
|
|
783
|
+
// Use the shared state.db handle if available; open a local one otherwise.
|
|
784
|
+
const dbForJudged = sharedStateDb;
|
|
785
|
+
if (dbForJudged) {
|
|
786
|
+
try {
|
|
787
|
+
cachedMap = getConsolidationJudgedMap(dbForJudged, memories.map((m) => `memory:${m.name}`));
|
|
788
|
+
}
|
|
789
|
+
catch {
|
|
790
|
+
cachedMap = new Map();
|
|
791
|
+
}
|
|
792
|
+
}
|
|
793
|
+
else {
|
|
794
|
+
try {
|
|
795
|
+
cachedMap = withStateDb((localDb) => getConsolidationJudgedMap(localDb, memories.map((m) => `memory:${m.name}`)));
|
|
796
|
+
}
|
|
797
|
+
catch {
|
|
798
|
+
// State DB unavailable → fail open: judge the full pool this run.
|
|
799
|
+
cachedMap = new Map();
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
const beforeCount = memories.length;
|
|
804
|
+
memories = memories.filter((m) => {
|
|
805
|
+
const cur = currentHashByName.get(m.name);
|
|
806
|
+
// No readable hash → keep (fail open; let the LLM judge it).
|
|
807
|
+
if (cur === undefined)
|
|
808
|
+
return true;
|
|
809
|
+
const cached = cachedMap.get(`memory:${m.name}`);
|
|
810
|
+
// Skip only when previously judged AND content is byte-identical since.
|
|
811
|
+
return !(cached !== undefined && cached.content_hash === cur);
|
|
812
|
+
});
|
|
813
|
+
const skipped = beforeCount - memories.length;
|
|
814
|
+
perfMs.judgedCacheSkipped = skipped; // WS-5 perf telemetry
|
|
815
|
+
if (skipped > 0) {
|
|
816
|
+
warnings.push(`Judged-state cache: skipped ${skipped} memor${skipped === 1 ? "y" : "ies"} judged-unchanged (no LLM); ${memories.length} remain for judging.`);
|
|
817
|
+
}
|
|
818
|
+
if (memories.length === 0) {
|
|
819
|
+
return {
|
|
820
|
+
done: true,
|
|
821
|
+
result: makeConsolidateResult({
|
|
822
|
+
dryRun: opts.dryRun ?? false,
|
|
823
|
+
target: opts.target ?? stashDir,
|
|
824
|
+
deleted: dedupCollapsed,
|
|
825
|
+
warnings,
|
|
826
|
+
durationMs: Date.now() - startMs,
|
|
827
|
+
}),
|
|
828
828
|
};
|
|
829
829
|
}
|
|
830
830
|
}
|
|
831
|
+
if (opts.limit === undefined && memories.length > 150) {
|
|
832
|
+
warnings.push(`Consolidation: pool has ${memories.length} memories and no limit is set. Consider adding a limit to your consolidate config to prevent timeouts on slow LLM endpoints.`);
|
|
833
|
+
}
|
|
831
834
|
if (opts.limit !== undefined && memories.length > opts.limit) {
|
|
832
835
|
// Order oldest-modified-first before capping so the limit selects the
|
|
833
836
|
// stalest memories rather than a fixed head of the (rowid-ordered) DB
|
|
@@ -849,13 +852,25 @@ export async function akmConsolidate(opts = {}) {
|
|
|
849
852
|
warnings.push(`Consolidation: pool capped at ${opts.limit} of ${memories.length} memories (limit option, oldest-modified first).`);
|
|
850
853
|
memories = memories.slice(0, opts.limit);
|
|
851
854
|
}
|
|
855
|
+
return { done: false, memories, dedupCollapsed, perfMs, judgedCacheEnabled, currentHashByName };
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Pass 2 — turn the narrowed pool into an executable plan. Sizes chunks to the
|
|
859
|
+
* model context window, clusters by embedding similarity, injects the
|
|
860
|
+
* anti-collapse random fraction, applies the cold-start budget cap, runs the
|
|
861
|
+
* per-chunk LLM calls (with retry + failure-rate abort), records judged-state
|
|
862
|
+
* cache outcomes, and reconciles the per-chunk op arrays via {@link mergePlans}.
|
|
863
|
+
* Populates `accounting` in place. Behavior-identical to the former inlined
|
|
864
|
+
* plan-generation block.
|
|
865
|
+
*/
|
|
866
|
+
async function planConsolidation(opts, config, stashDir, startMs, memories, warnings, sharedStateDb, judgedCacheEnabled, currentHashByName, accounting) {
|
|
852
867
|
// Consolidation always uses the HTTP LLM client directly — never the agent
|
|
853
868
|
// CLI. The agent CLI is for interactive agent sessions (reflect, propose);
|
|
854
869
|
// structured JSON generation works better and faster via HTTP.
|
|
855
870
|
//
|
|
856
871
|
// Honor `profiles.improve.default.processes.consolidate.profile` first; fall
|
|
857
872
|
// back to the default LLM. See {@link resolveConsolidateLlmConfig}.
|
|
858
|
-
const llmConfig = resolveConsolidateLlmConfig(config);
|
|
873
|
+
const llmConfig = resolveConsolidateLlmConfig(config, opts.improveProfile);
|
|
859
874
|
const isHttpPath = !!llmConfig;
|
|
860
875
|
// Chunk sizing: derive a safe chunk size from the configured model context
|
|
861
876
|
// window so that the full prompt (system prompt + chunk user prompt) never
|
|
@@ -872,16 +887,64 @@ export async function akmConsolidate(opts = {}) {
|
|
|
872
887
|
const chunkSize = computeSafeChunkSize(modelContextLength, bodyTruncation, opts.maxChunkSize);
|
|
873
888
|
// -- Phase A: plan generation -----------------------------------------------
|
|
874
889
|
const sourceName = opts.target ?? stashDir;
|
|
890
|
+
// WS-5: capture llmPoolSize = memories entering the LLM (after all filtering).
|
|
891
|
+
const llmPoolSize = memories.length;
|
|
875
892
|
// C-1 / #380: Pre-cluster memories by embedding similarity before chunking.
|
|
876
893
|
// This ensures that semantically similar memories land in the same LLM
|
|
877
894
|
// context window, allowing the model to detect and merge duplicates that
|
|
878
895
|
// would otherwise be split across chunks and survive indefinitely.
|
|
879
896
|
// mem0 arXiv:2504.19413, A-MEM arXiv:2502.12110.
|
|
880
897
|
// Fails open: if embeddings are unavailable or fail, original order is used.
|
|
881
|
-
const clusteredMemories = await clusterMemoriesBySimilarity(memories, config);
|
|
898
|
+
const { ordered: clusteredMemories, embedTelemetry } = await clusterMemoriesBySimilarity(memories, config, sharedStateDb);
|
|
899
|
+
// WS-3b Anti-collapse step 8c: inject random (non-similar) clusters.
|
|
900
|
+
// A small fraction (default 5%) of the pool is shuffled into random positions
|
|
901
|
+
// so the pipeline isn't PURELY similarity-driven. This prevents rich-get-richer
|
|
902
|
+
// entrenchment where only the most-retrieved assets ever get consolidated.
|
|
903
|
+
// DEFAULT ON since R5 — opt out via antiCollapse.enabled: false.
|
|
904
|
+
let finalClusteredMemories = clusteredMemories;
|
|
905
|
+
{
|
|
906
|
+
const antiCollapseForCluster = getImproveProcessConfig(config, "consolidate", opts.improveProfile)?.antiCollapse ?? {};
|
|
907
|
+
if (antiCollapseForCluster.enabled !== false && clusteredMemories.length > 2) {
|
|
908
|
+
const fraction = antiCollapseForCluster.randomClusterFraction ?? 0.05;
|
|
909
|
+
const randomCount = Math.max(1, Math.floor(clusteredMemories.length * fraction));
|
|
910
|
+
// Pick `randomCount` positions to inject random (un-clustered) members.
|
|
911
|
+
// Use a seeded-ish shuffle: sort by hash of the name so it's deterministic
|
|
912
|
+
// per run but not strictly similarity-driven.
|
|
913
|
+
const shuffled = [...clusteredMemories].sort((a, b) => {
|
|
914
|
+
// Deterministic shuffle: compare sha256-ish (use name hash as proxy).
|
|
915
|
+
const ha = a.name.split("").reduce((acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0, 0);
|
|
916
|
+
const hb = b.name.split("").reduce((acc, c) => ((acc << 5) - acc + c.charCodeAt(0)) | 0, 0);
|
|
917
|
+
return ha - hb;
|
|
918
|
+
});
|
|
919
|
+
const randomSlice = shuffled.slice(0, randomCount);
|
|
920
|
+
const randomSet = new Set(randomSlice.map((m) => m.name));
|
|
921
|
+
// Insert random members at intervals through the clustered sequence.
|
|
922
|
+
const withRandom = [];
|
|
923
|
+
const interval = Math.max(2, Math.floor(clusteredMemories.length / randomCount));
|
|
924
|
+
let randomIdx = 0;
|
|
925
|
+
for (let i = 0; i < clusteredMemories.length; i++) {
|
|
926
|
+
const m = clusteredMemories[i];
|
|
927
|
+
if (m && !randomSet.has(m.name))
|
|
928
|
+
withRandom.push(m);
|
|
929
|
+
if (i > 0 && i % interval === 0 && randomIdx < randomSlice.length) {
|
|
930
|
+
const r = randomSlice[randomIdx++];
|
|
931
|
+
if (r)
|
|
932
|
+
withRandom.push(r);
|
|
933
|
+
}
|
|
934
|
+
}
|
|
935
|
+
// Append any remaining random members not yet inserted.
|
|
936
|
+
while (randomIdx < randomSlice.length) {
|
|
937
|
+
const r = randomSlice[randomIdx++];
|
|
938
|
+
if (r)
|
|
939
|
+
withRandom.push(r);
|
|
940
|
+
}
|
|
941
|
+
finalClusteredMemories = withRandom;
|
|
942
|
+
warnings.push(`Anti-collapse: injected ${randomCount} random (non-similarity-driven) cluster member(s) into consolidation pool (fraction=${fraction}).`);
|
|
943
|
+
}
|
|
944
|
+
}
|
|
882
945
|
const chunks = [];
|
|
883
|
-
for (let i = 0; i <
|
|
884
|
-
chunks.push(
|
|
946
|
+
for (let i = 0; i < finalClusteredMemories.length; i += chunkSize) {
|
|
947
|
+
chunks.push(finalClusteredMemories.slice(i, i + chunkSize));
|
|
885
948
|
}
|
|
886
949
|
// 2026-05-27 prompt-context fix: precompute body-hashes of pending
|
|
887
950
|
// consolidate proposals once, so the per-chunk prompt can annotate
|
|
@@ -890,71 +953,86 @@ export async function akmConsolidate(opts = {}) {
|
|
|
890
953
|
// 4h on this user's stack. See
|
|
891
954
|
// /tmp/akm-health-investigations/tuning-reasons-investigation.md §Q3.
|
|
892
955
|
const pendingProposalBodyHashes = loadPendingConsolidateProposalHashes(stashDir);
|
|
956
|
+
// ── Cold-start budget estimation ─────────────────────────────────────────────
|
|
957
|
+
// Estimate wall-clock cost BEFORE issuing any LLM calls. When a signal is
|
|
958
|
+
// provided and the estimated cost exceeds ~60% of the remaining budget we
|
|
959
|
+
// auto-reduce the pool and log the reduction so the run never starts work
|
|
960
|
+
// it cannot finish (avoiding SIGTERM mid-LLM-call).
|
|
961
|
+
//
|
|
962
|
+
// Formula: chunks.length × p90_chunk_seconds. The p90 comes from
|
|
963
|
+
// `opts.p90ChunkSecondsDefault` (caller-supplied, typically from the profile
|
|
964
|
+
// config); absent = 30 s (conservative default matching a medium local LLM).
|
|
965
|
+
//
|
|
966
|
+
// "Remaining budget" is read from a custom property on the AbortSignal if
|
|
967
|
+
// the caller (improve.ts) has attached one. Without it no auto-reduction
|
|
968
|
+
// fires but the check is still cheap to run.
|
|
969
|
+
if (chunks.length > 10 && opts.signal) {
|
|
970
|
+
const p90Chunk = opts.p90ChunkSecondsDefault ?? 30;
|
|
971
|
+
const estimatedSeconds = chunks.length * p90Chunk;
|
|
972
|
+
// remainingBudgetMs is a non-standard extension set by improve.ts when it
|
|
973
|
+
// creates the budget AbortController. Undefined = no budget information.
|
|
974
|
+
const budgetMs = opts.signal.remainingBudgetMs;
|
|
975
|
+
if (budgetMs !== undefined && budgetMs > 0) {
|
|
976
|
+
const remainingSeconds = budgetMs / 1000;
|
|
977
|
+
if (estimatedSeconds > remainingSeconds * 0.6) {
|
|
978
|
+
const safeCaps = Math.max(1, Math.floor((remainingSeconds * 0.6) / p90Chunk));
|
|
979
|
+
const removedChunks = chunks.length - safeCaps;
|
|
980
|
+
if (removedChunks > 0) {
|
|
981
|
+
const msg = `[consolidate] cold-start budget: estimated ${estimatedSeconds.toFixed(0)}s > 60% of remaining ${remainingSeconds.toFixed(0)}s; ` +
|
|
982
|
+
`reducing pool from ${chunks.length} to ${safeCaps} chunks (${removedChunks} deferred to next run).`;
|
|
983
|
+
warn(msg);
|
|
984
|
+
warnings.push(msg);
|
|
985
|
+
chunks.splice(safeCaps);
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
}
|
|
893
990
|
warn(`[consolidate] ${memories.length} memories / ${chunks.length} chunk(s) / chunk_size=${chunkSize}` +
|
|
894
991
|
` / pending-proposal hashes: ${pendingProposalBodyHashes.size}`);
|
|
992
|
+
// Consolidate output merges memories (non-wiki) → stash authoring standards.
|
|
993
|
+
// Resolved ONCE per run and passed to each chunk prompt (facts not re-read
|
|
994
|
+
// per chunk).
|
|
995
|
+
const standardsContext = resolveStashStandards(stashDir);
|
|
895
996
|
const chunkOpsArrays = [];
|
|
896
|
-
// Structured skip-reason histogram (2026-05-26): every deterministic
|
|
897
|
-
// post-LLM op rejection site below also calls `pushSkipReason` so the
|
|
898
|
-
// health rollup can aggregate without regex-parsing English warning
|
|
899
|
-
// strings. See `/tmp/akm-health-investigations/tuning-reasons-investigation.md` §Q2.
|
|
900
|
-
const skipReasons = [];
|
|
901
|
-
// Per-ref grouping of skipReasons entries. A ref occupies exactly one
|
|
902
|
-
// accounting bucket and therefore exactly one skipReasons array entry;
|
|
903
|
-
// subsequent skip ops for the same ref append to that entry's `skips[]`
|
|
904
|
-
// rather than pushing a second array entry (that would inflate
|
|
905
|
-
// Σ(skipReasons) and break the invariant by +1 per duplicate).
|
|
906
|
-
const skipReasonByRef = new Map();
|
|
907
|
-
const pushSkipReason = (op, ref, reason) => {
|
|
908
|
-
// 2026-05-27 cross-chunk double-count fix: if `ref` already contributed
|
|
909
|
-
// to judgedNoAction in its own chunk (a different chunk proposed an op
|
|
910
|
-
// for it that is now being rejected here), promote it from the
|
|
911
|
-
// judgedNoAction bucket into the more specific skipReason bucket.
|
|
912
|
-
// Preserves the invariant: processed == actioned + judgedNoAction +
|
|
913
|
-
// Σ(skipReasons) + failedChunkMemories.
|
|
914
|
-
if (judgedNoActionRefs.delete(ref))
|
|
915
|
-
judgedNoAction--;
|
|
916
|
-
const existing = skipReasonByRef.get(ref);
|
|
917
|
-
if (existing) {
|
|
918
|
-
// Already counted once for accounting. Append the extra skip to the
|
|
919
|
-
// ref's grouped entry for observability without adding a new array
|
|
920
|
-
// entry (which would break the accounting invariant).
|
|
921
|
-
existing.skips.push({ op, reason });
|
|
922
|
-
return;
|
|
923
|
-
}
|
|
924
|
-
const entry = { ref, skips: [{ op, reason }] };
|
|
925
|
-
skipReasonByRef.set(ref, entry);
|
|
926
|
-
skipReasons.push(entry);
|
|
927
|
-
};
|
|
928
997
|
// judgedNoAction tracks memories the LLM saw inside a chunk but proposed
|
|
929
998
|
// no op for. Computed per chunk as `chunk.length − unique(targetRefs in ops)`.
|
|
930
|
-
|
|
931
|
-
//
|
|
932
|
-
//
|
|
933
|
-
//
|
|
934
|
-
//
|
|
935
|
-
//
|
|
936
|
-
|
|
937
|
-
//
|
|
938
|
-
//
|
|
939
|
-
//
|
|
940
|
-
//
|
|
941
|
-
|
|
942
|
-
// 2026-05-26 accounting-leak fix: per-secondary tally so successful merges
|
|
943
|
-
// account for `1 + secondaries.length` memories instead of 1.
|
|
944
|
-
let mergedSecondaries = 0;
|
|
999
|
+
// The structured skip-reason histogram (2026-05-26) plus the cross-chunk
|
|
1000
|
+
// double-count fixes now live on `accounting`; every deterministic post-LLM
|
|
1001
|
+
// op rejection site calls `accounting.pushSkipReason`. See
|
|
1002
|
+
// `/tmp/akm-health-investigations/tuning-reasons-investigation.md` §Q2.
|
|
1003
|
+
//
|
|
1004
|
+
// Judged-state cache (#581): coarse outcome per memory NAME the LLM actually
|
|
1005
|
+
// judged in a successfully-parsed chunk this run. "actioned" = an op targeted
|
|
1006
|
+
// it; "no_action" = the LLM saw it and proposed nothing. Populated only when
|
|
1007
|
+
// the cache is enabled (otherwise it stays empty and the post-loop recording
|
|
1008
|
+
// step is a no-op). Memories in failed/aborted chunks are NOT recorded, so a
|
|
1009
|
+
// transient LLM failure never poisons the cache into skipping them next run.
|
|
1010
|
+
const judgedOutcomeByName = new Map();
|
|
945
1011
|
// C-6 / #392: Replace two-consecutive-failures abort with failure-rate threshold.
|
|
946
1012
|
// Consecutive-count policies are brittle against transient LM Studio reloads:
|
|
947
1013
|
// two transient failures abort the run even though the next chunk would succeed.
|
|
948
1014
|
// Rate-based abort (≥50% failure over ≥4 chunks) is more robust.
|
|
949
1015
|
// Tanenbaum, Distributed Systems §8 — rate-based policies with minimum sample sizes.
|
|
950
1016
|
let totalChunksProcessed = 0;
|
|
951
|
-
let totalChunksFailed = 0;
|
|
952
1017
|
const ABORT_MIN_CHUNKS = 4;
|
|
953
1018
|
const ABORT_FAILURE_RATE = 0.5;
|
|
954
1019
|
for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
|
|
1020
|
+
// Budget-signal check: break cleanly before the next LLM call if the
|
|
1021
|
+
// caller's budget has been exhausted. Commits work done so far.
|
|
1022
|
+
if (opts.signal?.aborted) {
|
|
1023
|
+
const skipped = chunks.length - chunkIdx;
|
|
1024
|
+
const msg = `[consolidate] budget signal aborted before chunk ${chunkIdx + 1}/${chunks.length}; ${skipped} chunk(s) not processed (partial_timeout — work done so far committed).`;
|
|
1025
|
+
warn(msg);
|
|
1026
|
+
warnings.push(msg);
|
|
1027
|
+
// Account for memories in unprocessed chunks.
|
|
1028
|
+
for (let i = chunkIdx; i < chunks.length; i++) {
|
|
1029
|
+
accounting.failedChunkMemories += chunks[i].length;
|
|
1030
|
+
}
|
|
1031
|
+
break;
|
|
1032
|
+
}
|
|
955
1033
|
// Abort if failure rate >= 50% over at least 4 processed chunks.
|
|
956
1034
|
if (totalChunksProcessed >= ABORT_MIN_CHUNKS) {
|
|
957
|
-
const failureRate = totalChunksFailed / totalChunksProcessed;
|
|
1035
|
+
const failureRate = accounting.totalChunksFailed / totalChunksProcessed;
|
|
958
1036
|
if (failureRate >= ABORT_FAILURE_RATE) {
|
|
959
1037
|
const skipped = chunks.length - chunkIdx;
|
|
960
1038
|
const abortMsg = `Consolidation aborted — failure rate ${(failureRate * 100).toFixed(0)}% over ${totalChunksProcessed} chunks (>= ${ABORT_FAILURE_RATE * 100}% threshold). LLM may be unavailable. ${skipped} chunk(s) skipped.`;
|
|
@@ -965,7 +1043,7 @@ export async function akmConsolidate(opts = {}) {
|
|
|
965
1043
|
// rejected). Without this, the accounting invariant fails by
|
|
966
1044
|
// `Σ(unattempted_chunk.length)` whenever the abort fires.
|
|
967
1045
|
for (let i = chunkIdx; i < chunks.length; i++) {
|
|
968
|
-
failedChunkMemories += chunks[i].length;
|
|
1046
|
+
accounting.failedChunkMemories += chunks[i].length;
|
|
969
1047
|
}
|
|
970
1048
|
break;
|
|
971
1049
|
}
|
|
@@ -983,22 +1061,23 @@ export async function akmConsolidate(opts = {}) {
|
|
|
983
1061
|
// LLM-failure-rate abort policy — no request was attempted.
|
|
984
1062
|
if (chunk.length > 0 && chunk.every((m) => isHotCapturedMemory(m.filePath))) {
|
|
985
1063
|
for (const m of chunk)
|
|
986
|
-
judgedNoActionRefs.add(`memory:${m.name}`);
|
|
987
|
-
judgedNoAction += chunk.length;
|
|
1064
|
+
accounting.judgedNoActionRefs.add(`memory:${m.name}`);
|
|
1065
|
+
accounting.judgedNoAction += chunk.length;
|
|
988
1066
|
warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length}: all ${chunk.length} memories are captureMode: hot — skipping LLM (judged no-action).`);
|
|
989
1067
|
continue;
|
|
990
1068
|
}
|
|
991
1069
|
warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length} (${chunk.length} memories) …`);
|
|
992
|
-
const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation, pendingProposalBodyHashes);
|
|
993
|
-
|
|
1070
|
+
const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation, pendingProposalBodyHashes, standardsContext);
|
|
1071
|
+
// Single chunk LLM call, wrapped in the feature gate. Deduplicated across
|
|
1072
|
+
// the first attempt and the retry below (the two blocks were byte-identical
|
|
1073
|
+
// apart from their fallback error string). responseSchema lift (PR 1,
|
|
1074
|
+
// asset-writers-investigation §5): providers with `supportsJsonSchema: true`
|
|
1075
|
+
// enforce the shape upstream; others fall through to
|
|
1076
|
+
// `parseEmbeddedJsonResponse` on the response side.
|
|
1077
|
+
const callChunkLlm = (fallbackError) => tryLlmFeature("memory_consolidation", config, async () => {
|
|
994
1078
|
if (!llmConfig)
|
|
995
1079
|
return { ok: false, error: "No LLM configured for consolidation" };
|
|
996
1080
|
try {
|
|
997
|
-
// responseSchema lift (PR 1, asset-writers-investigation §5): pass
|
|
998
|
-
// the consolidate plan schema so providers with
|
|
999
|
-
// `supportsJsonSchema: true` enforce shape upstream. Providers that
|
|
1000
|
-
// ignore the option fall through to the existing
|
|
1001
|
-
// `parseEmbeddedJsonResponse` path on the response side.
|
|
1002
1081
|
const content = await chatCompletion(llmConfig, [
|
|
1003
1082
|
{ role: "system", content: CONSOLIDATE_SYSTEM_PROMPT },
|
|
1004
1083
|
{ role: "user", content: userPrompt },
|
|
@@ -1008,36 +1087,24 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1008
1087
|
catch (e) {
|
|
1009
1088
|
return { ok: false, error: String(e) };
|
|
1010
1089
|
}
|
|
1011
|
-
}, { ok: false, error:
|
|
1090
|
+
}, { ok: false, error: fallbackError });
|
|
1091
|
+
let raw = await callChunkLlm(`chunk ${chunkIdx + 1} failed`);
|
|
1012
1092
|
if (!raw.ok) {
|
|
1013
1093
|
// Single retry with 2s backoff before recording chunk as lost.
|
|
1014
1094
|
// Recovers transient Shredder LM Studio timeouts without significantly
|
|
1015
1095
|
// extending run time. Only marks failed if both attempts fail.
|
|
1016
1096
|
await new Promise((r) => setTimeout(r, 2_000));
|
|
1017
|
-
const retry = await
|
|
1018
|
-
if (!llmConfig)
|
|
1019
|
-
return { ok: false, error: "No LLM configured for consolidation" };
|
|
1020
|
-
try {
|
|
1021
|
-
const content = await chatCompletion(llmConfig, [
|
|
1022
|
-
{ role: "system", content: CONSOLIDATE_SYSTEM_PROMPT },
|
|
1023
|
-
{ role: "user", content: userPrompt },
|
|
1024
|
-
], { responseSchema: CONSOLIDATE_PLAN_JSON_SCHEMA, enableThinking: false });
|
|
1025
|
-
return { ok: true, content };
|
|
1026
|
-
}
|
|
1027
|
-
catch (e) {
|
|
1028
|
-
return { ok: false, error: String(e) };
|
|
1029
|
-
}
|
|
1030
|
-
}, { ok: false, error: `chunk ${chunkIdx + 1} retry failed` });
|
|
1097
|
+
const retry = await callChunkLlm(`chunk ${chunkIdx + 1} retry failed`);
|
|
1031
1098
|
if (!retry.ok) {
|
|
1032
1099
|
warn(retry.error ?? `chunk ${chunkIdx + 1} failed after retry`);
|
|
1033
1100
|
warnings.push(retry.error ?? `chunk ${chunkIdx + 1} failed after retry`);
|
|
1034
1101
|
totalChunksProcessed++;
|
|
1035
|
-
totalChunksFailed++;
|
|
1102
|
+
accounting.totalChunksFailed++;
|
|
1036
1103
|
// Account for the chunk's memories under the failed-chunk bucket.
|
|
1037
1104
|
// judgedNoAction does NOT run on this path (it's after the success
|
|
1038
1105
|
// guards) so without this the accounting invariant breaks on every
|
|
1039
1106
|
// chunk-level transport/parse failure.
|
|
1040
|
-
failedChunkMemories += chunk.length;
|
|
1107
|
+
accounting.failedChunkMemories += chunk.length;
|
|
1041
1108
|
continue;
|
|
1042
1109
|
}
|
|
1043
1110
|
raw = retry;
|
|
@@ -1054,8 +1121,8 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1054
1121
|
warn(`Chunk ${chunkIdx + 1}: invalid plan from AI — skipping.${hint}`);
|
|
1055
1122
|
warnings.push(`Chunk ${chunkIdx + 1}: invalid plan from AI — skipping.${hint}`);
|
|
1056
1123
|
totalChunksProcessed++;
|
|
1057
|
-
totalChunksFailed++;
|
|
1058
|
-
failedChunkMemories += chunk.length;
|
|
1124
|
+
accounting.totalChunksFailed++;
|
|
1125
|
+
accounting.failedChunkMemories += chunk.length;
|
|
1059
1126
|
continue;
|
|
1060
1127
|
}
|
|
1061
1128
|
totalChunksProcessed++; // success
|
|
@@ -1095,41 +1162,185 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1095
1162
|
const memRef = `memory:${m.name}`;
|
|
1096
1163
|
if (!targetRefs.has(memRef)) {
|
|
1097
1164
|
chunkNoAction++;
|
|
1098
|
-
judgedNoActionRefs.add(memRef);
|
|
1165
|
+
accounting.judgedNoActionRefs.add(memRef);
|
|
1166
|
+
// Judged-state cache (#581): the LLM saw this memory and proposed
|
|
1167
|
+
// nothing → record judged-unchanged so the next run can skip it.
|
|
1168
|
+
if (judgedCacheEnabled)
|
|
1169
|
+
judgedOutcomeByName.set(m.name, "no_action");
|
|
1170
|
+
}
|
|
1171
|
+
else if (judgedCacheEnabled) {
|
|
1172
|
+
// An op targeted this memory → it was judged + actioned.
|
|
1173
|
+
judgedOutcomeByName.set(m.name, "actioned");
|
|
1099
1174
|
}
|
|
1100
1175
|
}
|
|
1101
|
-
judgedNoAction += chunkNoAction;
|
|
1176
|
+
accounting.judgedNoAction += chunkNoAction;
|
|
1102
1177
|
chunkOpsArrays.push(ops);
|
|
1103
1178
|
}
|
|
1179
|
+
// ── Judged-state cache recording (#581) ─────────────────────────────────────
|
|
1180
|
+
// Persist judged state for every memory the LLM actually judged this run so
|
|
1181
|
+
// the next run can skip the unchanged ones. Keyed by current content hash so
|
|
1182
|
+
// a later body edit (different hash) re-enters the LLM pool. DEFAULT OFF and
|
|
1183
|
+
// skipped under --dry-run (dry-run mutates nothing). Failed/aborted chunks
|
|
1184
|
+
// contributed no entries to `judgedOutcomeByName`, so a transient LLM outage
|
|
1185
|
+
// never caches a memory as judged.
|
|
1186
|
+
if (judgedCacheEnabled && !opts.dryRun && judgedOutcomeByName.size > 0) {
|
|
1187
|
+
// Use the shared state.db handle; open a local one as fallback.
|
|
1188
|
+
const doRecord = (db) => {
|
|
1189
|
+
const judgedAt = new Date(startMs).toISOString();
|
|
1190
|
+
for (const [name, outcome] of judgedOutcomeByName) {
|
|
1191
|
+
const hash = currentHashByName.get(name);
|
|
1192
|
+
if (hash === undefined)
|
|
1193
|
+
continue;
|
|
1194
|
+
upsertConsolidationJudged(db, {
|
|
1195
|
+
entryKey: `memory:${name}`,
|
|
1196
|
+
contentHash: hash,
|
|
1197
|
+
judgedAt,
|
|
1198
|
+
outcome,
|
|
1199
|
+
});
|
|
1200
|
+
}
|
|
1201
|
+
};
|
|
1202
|
+
if (sharedStateDb) {
|
|
1203
|
+
try {
|
|
1204
|
+
doRecord(sharedStateDb);
|
|
1205
|
+
}
|
|
1206
|
+
catch (e) {
|
|
1207
|
+
warnings.push(`Judged-state cache: failed to record judged state: ${String(e)}`);
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
else {
|
|
1211
|
+
try {
|
|
1212
|
+
withStateDb((localDb) => doRecord(localDb));
|
|
1213
|
+
}
|
|
1214
|
+
catch (e) {
|
|
1215
|
+
warnings.push(`Judged-state cache: failed to record judged state: ${String(e)}`);
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1104
1219
|
// Build the known-refs set from the already-filtered memory pool so
|
|
1105
1220
|
// mergePlans() can reject LLM-hallucinated primary refs before execution.
|
|
1106
1221
|
const knownRefs = new Set(memories.map((m) => `memory:${m.name}`));
|
|
1107
1222
|
const { ops: allOps, warnings: mergeWarnings } = mergePlans(chunkOpsArrays, knownRefs);
|
|
1108
1223
|
warnings.push(...mergeWarnings);
|
|
1224
|
+
return { allOps, totalChunks: chunks.length, llmPoolSize, embedTelemetry, isHttpPath, sourceName };
|
|
1225
|
+
}
|
|
1226
|
+
/**
|
|
1227
|
+
* Pass 3 — execute the reconciled plan against the filesystem: resolve the
|
|
1228
|
+
* write target, journal the batch, dispatch each op to its handler, then commit
|
|
1229
|
+
* the batch at the boundary and clean up the journal. Mutates `accounting` via
|
|
1230
|
+
* the op-handlers' `pushSkipReason`. Behavior-identical to the former inlined
|
|
1231
|
+
* write block. Never invoked on the dry-run or aborted-confirm paths.
|
|
1232
|
+
*/
|
|
1233
|
+
async function applyConsolidationPlan(config, stashDir, sourceRun, memories, warnings, allOps, accounting, dedupCollapsed, activeProfile) {
|
|
1234
|
+
// -- Phase B + writes -------------------------------------------------------
|
|
1235
|
+
const target = resolveWriteTarget(config);
|
|
1236
|
+
const timestamp = timestampForFilename();
|
|
1237
|
+
const backupDir = getBackupDir(stashDir, timestamp);
|
|
1238
|
+
// Write journal before any mutations
|
|
1239
|
+
writeJournal(stashDir, allOps, timestamp);
|
|
1240
|
+
const counts = {
|
|
1241
|
+
merged: 0,
|
|
1242
|
+
deleted: 0,
|
|
1243
|
+
contradicted: 0, // C-3 / #382: count of contradiction edges written
|
|
1244
|
+
mergeFloorViolations: 0, // R5 §4.2: advisory merge-information-floor failures
|
|
1245
|
+
mergedSecondaries: 0,
|
|
1246
|
+
};
|
|
1247
|
+
const promoted = [];
|
|
1248
|
+
// Within-run dedup: track source refs for which a promote proposal was
|
|
1249
|
+
// already created this run. The LLM can return multiple promote ops for
|
|
1250
|
+
// different source memories that happen to have identical content (all are
|
|
1251
|
+
// duplicate memories), so we also need a content-hash guard below.
|
|
1252
|
+
const promotedSourceRefs = new Set();
|
|
1253
|
+
// Build a lookup map: ref → MemoryEntry
|
|
1254
|
+
const memoryByRef = new Map();
|
|
1255
|
+
for (const m of memories) {
|
|
1256
|
+
memoryByRef.set(`memory:${m.name}`, m);
|
|
1257
|
+
}
|
|
1258
|
+
const opCtx = {
|
|
1259
|
+
config,
|
|
1260
|
+
improveProfile: activeProfile,
|
|
1261
|
+
stashDir,
|
|
1262
|
+
sourceRun,
|
|
1263
|
+
target,
|
|
1264
|
+
backupDir,
|
|
1265
|
+
memoryByRef,
|
|
1266
|
+
promoted,
|
|
1267
|
+
promotedSourceRefs,
|
|
1268
|
+
warnings,
|
|
1269
|
+
counts,
|
|
1270
|
+
pushSkipReason: accounting.pushSkipReason,
|
|
1271
|
+
};
|
|
1272
|
+
// Thin dispatch over the op discriminator — each branch is now an isolated,
|
|
1273
|
+
// independently-testable handler that mutates `opCtx`.
|
|
1274
|
+
for (let opIndex = 0; opIndex < allOps.length; opIndex++) {
|
|
1275
|
+
const op = allOps[opIndex];
|
|
1276
|
+
const opDisplayRef = op.op === "merge" ? op.primary : op.op === "contradict" ? `${op.ref} ↔ ${op.contradictedByRef}` : op.ref;
|
|
1277
|
+
warn(`[consolidate] ${opIndex + 1}/${allOps.length} ${op.op} ${opDisplayRef}`);
|
|
1278
|
+
switch (op.op) {
|
|
1279
|
+
case "merge":
|
|
1280
|
+
await handleMergeOp(op, opIndex, opCtx);
|
|
1281
|
+
break;
|
|
1282
|
+
case "delete":
|
|
1283
|
+
await handleDeleteOp(op, opIndex, opCtx);
|
|
1284
|
+
break;
|
|
1285
|
+
case "promote":
|
|
1286
|
+
await handlePromoteOp(op, opCtx);
|
|
1287
|
+
break;
|
|
1288
|
+
case "contradict":
|
|
1289
|
+
await handleContradictOp(op, opCtx);
|
|
1290
|
+
break;
|
|
1291
|
+
}
|
|
1292
|
+
}
|
|
1293
|
+
const { merged, deleted, contradicted, mergeFloorViolations, mergedSecondaries } = counts;
|
|
1294
|
+
// 0.9.0 (issue #507): batch-at-boundary commit. The merge/delete loop above
|
|
1295
|
+
// wrote one merged primary and deleted N secondaries to the resolved target
|
|
1296
|
+
// with NO per-asset commit. If the target is a writable git source and any
|
|
1297
|
+
// asset was mutated, commit the whole batch ONCE here (stages .akm/ +
|
|
1298
|
+
// siblings together). No-op for filesystem/primary-stash targets.
|
|
1299
|
+
if (merged > 0 || deleted > 0) {
|
|
1300
|
+
commitWriteTargetBoundary(target, `Consolidate: ${merged} merged, ${deleted} removed`);
|
|
1301
|
+
}
|
|
1302
|
+
cleanupJournal(stashDir, timestamp);
|
|
1303
|
+
// [signoff 2026-06-15] TTL archive cleanup machinery RETIRED (WS-3a).
|
|
1304
|
+
// The elaborate archiveRetentionDays / archive-dir scan existed only to satisfy
|
|
1305
|
+
// the old irrecoverability constraint. Stashes are now git-backed, so git
|
|
1306
|
+
// history is the recovery path — no bespoke archive TTL needed. Any files in
|
|
1307
|
+
// .akm/archive/ will stay there harmlessly until the operator prunes them with
|
|
1308
|
+
// `git rm` or `find .akm/archive -mtime +90 -delete`. Changed N files this
|
|
1309
|
+
// run; recover any via `git show <sha>:<path>` or `git restore <path>`.
|
|
1310
|
+
if (merged > 0 || deleted > 0 || dedupCollapsed > 0) {
|
|
1311
|
+
const totalChanged = merged + deleted + dedupCollapsed;
|
|
1312
|
+
warnings.push(`Changed ${totalChanged} file(s) this run. Recover any via git if needed (git history is the backstop).`);
|
|
1313
|
+
}
|
|
1314
|
+
return { merged, deleted, contradicted, mergeFloorViolations, mergedSecondaries, promoted };
|
|
1315
|
+
}
|
|
1316
|
+
async function akmConsolidateInner(opts, config, stashDir, startMs, sourceRun, warnings, sharedStateDb) {
|
|
1317
|
+
// -- Pass 1: narrow the memory pool (may early-return an envelope) ----------
|
|
1318
|
+
const narrowed = await narrowConsolidationPool(opts, config, stashDir, startMs, warnings, sharedStateDb);
|
|
1319
|
+
if (narrowed.done)
|
|
1320
|
+
return narrowed.result;
|
|
1321
|
+
const { memories, dedupCollapsed, perfMs, judgedCacheEnabled, currentHashByName } = narrowed;
|
|
1322
|
+
// -- Pass 2: build the LLM plan (populates the shared accounting counters) ---
|
|
1323
|
+
const accounting = createConsolidateAccounting();
|
|
1324
|
+
const { allOps, totalChunks, llmPoolSize, embedTelemetry, isHttpPath, sourceName } = await planConsolidation(opts, config, stashDir, startMs, memories, warnings, sharedStateDb, judgedCacheEnabled, currentHashByName, accounting);
|
|
1109
1325
|
// -- Dry-run: show AI plan without executing any writes --------------------
|
|
1110
1326
|
if (opts.dryRun) {
|
|
1111
|
-
return {
|
|
1112
|
-
schemaVersion: 1,
|
|
1113
|
-
ok: true,
|
|
1114
|
-
shape: "consolidate-result",
|
|
1327
|
+
return makeConsolidateResult({
|
|
1115
1328
|
dryRun: true,
|
|
1116
1329
|
previewOnly: true,
|
|
1117
1330
|
target: sourceName,
|
|
1118
1331
|
processed: memories.length,
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
mergedSecondaries,
|
|
1128
|
-
failedChunkMemories,
|
|
1332
|
+
failedChunks: accounting.totalChunksFailed,
|
|
1333
|
+
totalChunks,
|
|
1334
|
+
judgedNoAction: accounting.judgedNoAction,
|
|
1335
|
+
skipReasons: accounting.skipReasons,
|
|
1336
|
+
// No merge has executed on the preview path — the per-secondary tally is
|
|
1337
|
+
// provably still 0 here (it only increments in the op-execution loop).
|
|
1338
|
+
mergedSecondaries: 0,
|
|
1339
|
+
failedChunkMemories: accounting.failedChunkMemories,
|
|
1129
1340
|
planned: allOps,
|
|
1130
1341
|
warnings,
|
|
1131
1342
|
durationMs: Date.now() - startMs,
|
|
1132
|
-
};
|
|
1343
|
+
});
|
|
1133
1344
|
}
|
|
1134
1345
|
warn(`[consolidate] plan: ${allOps.length} operation(s)`);
|
|
1135
1346
|
// -- HTTP path: warn about quality and confirm unless auto-accepted --------
|
|
@@ -1150,555 +1361,28 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1150
1361
|
const nonInteractive = process.stdin.isTTY === false || process.env.AKM_NON_INTERACTIVE === "1";
|
|
1151
1362
|
const answer = nonInteractive ? false : await promptConfirm(`Apply ${n} operations? [y/N] `);
|
|
1152
1363
|
if (!answer) {
|
|
1153
|
-
return {
|
|
1154
|
-
schemaVersion: 1,
|
|
1155
|
-
ok: true,
|
|
1156
|
-
shape: "consolidate-result",
|
|
1157
|
-
dryRun: false,
|
|
1364
|
+
return makeConsolidateResult({
|
|
1158
1365
|
previewOnly: true,
|
|
1159
1366
|
target: sourceName,
|
|
1160
1367
|
processed: memories.length,
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
skipReasons,
|
|
1169
|
-
mergedSecondaries,
|
|
1170
|
-
failedChunkMemories,
|
|
1368
|
+
failedChunks: accounting.totalChunksFailed,
|
|
1369
|
+
totalChunks,
|
|
1370
|
+
judgedNoAction: accounting.judgedNoAction,
|
|
1371
|
+
skipReasons: accounting.skipReasons,
|
|
1372
|
+
// No merge executed on the abort path — mergedSecondaries is still 0.
|
|
1373
|
+
mergedSecondaries: 0,
|
|
1374
|
+
failedChunkMemories: accounting.failedChunkMemories,
|
|
1171
1375
|
planned: allOps,
|
|
1172
1376
|
warnings: [...warnings, nonInteractive ? "Non-interactive context: skipped apply." : "Aborted by user."],
|
|
1173
1377
|
durationMs: Date.now() - startMs,
|
|
1174
|
-
};
|
|
1175
|
-
}
|
|
1176
|
-
}
|
|
1177
|
-
}
|
|
1178
|
-
// -- Phase B + writes -------------------------------------------------------
|
|
1179
|
-
const target = resolveWriteTarget(config);
|
|
1180
|
-
const timestamp = timestampForFilename();
|
|
1181
|
-
const backupDir = getBackupDir(stashDir, timestamp);
|
|
1182
|
-
// Write journal before any mutations
|
|
1183
|
-
writeJournal(stashDir, allOps, timestamp);
|
|
1184
|
-
let merged = 0;
|
|
1185
|
-
let deleted = 0;
|
|
1186
|
-
const promoted = [];
|
|
1187
|
-
let contradicted = 0; // C-3 / #382: count of contradiction edges written
|
|
1188
|
-
// Within-run dedup: track source refs for which a promote proposal was
|
|
1189
|
-
// already created this run. The LLM can return multiple promote ops for
|
|
1190
|
-
// different source memories that happen to have identical content (all are
|
|
1191
|
-
// duplicate memories), so we also need a content-hash guard below.
|
|
1192
|
-
const promotedSourceRefs = new Set();
|
|
1193
|
-
// Build a lookup map: ref → MemoryEntry
|
|
1194
|
-
const memoryByRef = new Map();
|
|
1195
|
-
for (const m of memories) {
|
|
1196
|
-
memoryByRef.set(`memory:${m.name}`, m);
|
|
1197
|
-
}
|
|
1198
|
-
for (let opIndex = 0; opIndex < allOps.length; opIndex++) {
|
|
1199
|
-
const op = allOps[opIndex];
|
|
1200
|
-
const opDisplayRef = op.op === "merge" ? op.primary : op.op === "contradict" ? `${op.ref} ↔ ${op.contradictedByRef}` : op.ref;
|
|
1201
|
-
warn(`[consolidate] ${opIndex + 1}/${allOps.length} ${op.op} ${opDisplayRef}`);
|
|
1202
|
-
if (op.op === "merge") {
|
|
1203
|
-
// Accounting helper: emit a per-participant skipReason for failed
|
|
1204
|
-
// merges so primary + every loaded-memory secondary land in the
|
|
1205
|
-
// structured skip histogram. Pre-2026-05-26 only the primary was
|
|
1206
|
-
// counted (1 skipReason per failed merge), leaving N secondaries
|
|
1207
|
-
// unaccounted for in the `processed == actioned + noAction + Σskips`
|
|
1208
|
-
// invariant — the source of the 4–11 silent leaks per run.
|
|
1209
|
-
const emitMergeFailureSkips = (reason) => {
|
|
1210
|
-
if (memoryByRef.has(op.primary))
|
|
1211
|
-
pushSkipReason("merge", op.primary, reason);
|
|
1212
|
-
for (const secRef of op.secondaries) {
|
|
1213
|
-
if (memoryByRef.has(secRef))
|
|
1214
|
-
pushSkipReason("merge", secRef, reason);
|
|
1215
|
-
}
|
|
1216
|
-
};
|
|
1217
|
-
const primaryEntry = memoryByRef.get(op.primary);
|
|
1218
|
-
if (!primaryEntry) {
|
|
1219
|
-
// This fires when a prior op in the same run consumed this ref as a
|
|
1220
|
-
// secondary and Fix-A pruned it from memoryByRef. It should NOT fire
|
|
1221
|
-
// for hallucinated primaries (those are dropped by mergePlans() before
|
|
1222
|
-
// reaching here). If this counter is non-zero, suspect an intra-run
|
|
1223
|
-
// cross-chunk race, not a filter regression.
|
|
1224
|
-
warnings.push(`Merge: primary ${op.primary} not found in loaded memories (pruned by prior op this run) — skipping.`);
|
|
1225
|
-
emitMergeFailureSkips("merge_primary_missing");
|
|
1226
|
-
continue;
|
|
1227
|
-
}
|
|
1228
|
-
// Defense-in-depth: even if the entry is in memoryByRef (pre-flight ran
|
|
1229
|
-
// before this run's own ops), the file may have been deleted by a
|
|
1230
|
-
// concurrent process or an edge case the pre-flight filter missed.
|
|
1231
|
-
if (!fs.existsSync(primaryEntry.filePath)) {
|
|
1232
|
-
warnings.push(`Merge: primary ${op.primary} file gone at execution time (stale entry) — skipping.`);
|
|
1233
|
-
emitMergeFailureSkips("merge_primary_file_gone");
|
|
1234
|
-
continue;
|
|
1235
|
-
}
|
|
1236
|
-
// Phase B: generate merged content
|
|
1237
|
-
const secondaryBodies = [];
|
|
1238
|
-
for (const secRef of op.secondaries) {
|
|
1239
|
-
const secEntry = memoryByRef.get(secRef);
|
|
1240
|
-
if (!secEntry) {
|
|
1241
|
-
warnings.push(`Merge: secondary ${secRef} not found — skipping merge op.`);
|
|
1242
|
-
// No accounting impact: a missing secondary is a phantom ref and
|
|
1243
|
-
// never contributed to any chunk's targetRefs reduction. We still
|
|
1244
|
-
// continue the loop to gather the remaining valid secondaries.
|
|
1245
|
-
continue;
|
|
1246
|
-
}
|
|
1247
|
-
secondaryBodies.push(secRef);
|
|
1248
|
-
}
|
|
1249
|
-
if (secondaryBodies.length === 0) {
|
|
1250
|
-
warnings.push(`Merge: ${op.primary} has no valid secondaries — skipping.`);
|
|
1251
|
-
emitMergeFailureSkips("merge_no_valid_secondaries");
|
|
1252
|
-
continue;
|
|
1253
|
-
}
|
|
1254
|
-
// Pre-flight hot guard — skip the LLM call entirely if any participant
|
|
1255
|
-
// is hot or unparseable. Without this, mixed chunks still send hot merges
|
|
1256
|
-
// to the planner which proposes them; generateMergedContent() is then
|
|
1257
|
-
// called, produces output without `description`, and the skip is
|
|
1258
|
-
// misattributed to merge_missing_description instead of the real cause.
|
|
1259
|
-
const preflightParticipants = [op.primary, ...op.secondaries];
|
|
1260
|
-
const preflightBlocked = preflightParticipants.flatMap((ref) => {
|
|
1261
|
-
const e = memoryByRef.get(ref);
|
|
1262
|
-
if (!e)
|
|
1263
|
-
return [];
|
|
1264
|
-
const verdict = consolidateGuardStatus(e.filePath);
|
|
1265
|
-
if (verdict === "hot" || verdict === "unparseable")
|
|
1266
|
-
return [{ ref, verdict }];
|
|
1267
|
-
return [];
|
|
1268
|
-
});
|
|
1269
|
-
if (preflightBlocked.length > 0) {
|
|
1270
|
-
const detail = preflightBlocked.map((p) => `${p.ref} (${p.verdict})`).join(", ");
|
|
1271
|
-
warnings.push(`Merge: refused for ${op.primary} — ${preflightBlocked.length} participant(s) blocked by hot/unparseable frontmatter guard (pre-flight): ${detail}`);
|
|
1272
|
-
emitMergeFailureSkips("merge_participant_blocked");
|
|
1273
|
-
continue;
|
|
1274
|
-
}
|
|
1275
|
-
let primaryBody = "";
|
|
1276
|
-
try {
|
|
1277
|
-
primaryBody = fs.readFileSync(primaryEntry.filePath, "utf8");
|
|
1278
|
-
}
|
|
1279
|
-
catch {
|
|
1280
|
-
warnings.push(`Merge: could not read primary ${op.primary} — skipping.`);
|
|
1281
|
-
emitMergeFailureSkips("merge_read_failed");
|
|
1282
|
-
continue;
|
|
1283
|
-
}
|
|
1284
|
-
const mergeResult = await generateMergedContent(config, op.primary, primaryBody, op.secondaries, memoryByRef);
|
|
1285
|
-
if ("error" in mergeResult) {
|
|
1286
|
-
warnings.push(`Merge: ${mergeResult.error} for ${mergeResult.detail}.`);
|
|
1287
|
-
emitMergeFailureSkips(mergeResult.error);
|
|
1288
|
-
continue;
|
|
1289
|
-
}
|
|
1290
|
-
const mergedContent = mergeResult.content;
|
|
1291
|
-
// Validate frontmatter of merged content — must have a `---` block
|
|
1292
|
-
// with at minimum a `description` field. We parse via the hand-rolled
|
|
1293
|
-
// parser (cheap) AND require non-empty description. This guards against
|
|
1294
|
-
// the historical defect where merged memories were written back with
|
|
1295
|
-
// empty `description` and later polluted the promote path.
|
|
1296
|
-
let parsedMerged;
|
|
1297
|
-
try {
|
|
1298
|
-
parsedMerged = parseFrontmatter(mergedContent);
|
|
1299
|
-
}
|
|
1300
|
-
catch {
|
|
1301
|
-
warnings.push(`Merge: merged content for ${op.primary} has invalid frontmatter — skipping.`);
|
|
1302
|
-
emitMergeFailureSkips("merge_invalid_frontmatter");
|
|
1303
|
-
continue;
|
|
1304
|
-
}
|
|
1305
|
-
if (parsedMerged.frontmatter === null) {
|
|
1306
|
-
warnings.push(`Merge: merged content for ${op.primary} has no frontmatter block — skipping.`);
|
|
1307
|
-
emitMergeFailureSkips("merge_invalid_frontmatter");
|
|
1308
|
-
continue;
|
|
1309
|
-
}
|
|
1310
|
-
const mergedDesc = parsedMerged.data.description;
|
|
1311
|
-
if (typeof mergedDesc !== "string" || mergedDesc.trim().length === 0) {
|
|
1312
|
-
warnings.push(`Merge: merged content for ${op.primary} missing description — skipping.`);
|
|
1313
|
-
emitMergeFailureSkips("merge_missing_description");
|
|
1314
|
-
continue;
|
|
1315
|
-
}
|
|
1316
|
-
const truncReason = detectTruncatedDescription(mergedDesc);
|
|
1317
|
-
if (truncReason) {
|
|
1318
|
-
warnings.push(`Merge: merged content for ${op.primary} has truncated description (${truncReason}) — skipping.`);
|
|
1319
|
-
emitMergeFailureSkips("merge_truncated_description");
|
|
1320
|
-
continue;
|
|
1321
|
-
}
|
|
1322
|
-
// captureMode:hot guard — refuse the merge if ANY participating memory
|
|
1323
|
-
// (primary or secondary) was user-captured or has unparseable frontmatter
|
|
1324
|
-
// (could have hidden a hot flag). Hot memories are user-explicit and
|
|
1325
|
-
// must not be deleted/overwritten by the consolidate LLM. 14 user
|
|
1326
|
-
// memories were silent-deleted by consolidate before this guard landed;
|
|
1327
|
-
// recovery required copying from .akm/archive/ by hand.
|
|
1328
|
-
const mergeParticipants = [op.primary, ...op.secondaries];
|
|
1329
|
-
const blockedParticipants = mergeParticipants.flatMap((ref) => {
|
|
1330
|
-
const e = memoryByRef.get(ref);
|
|
1331
|
-
if (!e)
|
|
1332
|
-
return [];
|
|
1333
|
-
const verdict = consolidateGuardStatus(e.filePath);
|
|
1334
|
-
if (verdict === "hot" || verdict === "unparseable")
|
|
1335
|
-
return [{ ref, verdict }];
|
|
1336
|
-
return [];
|
|
1337
|
-
});
|
|
1338
|
-
if (blockedParticipants.length > 0) {
|
|
1339
|
-
const detail = blockedParticipants.map((p) => `${p.ref} (${p.verdict})`).join(", ");
|
|
1340
|
-
warnings.push(`Merge: refused for ${op.primary} — ${blockedParticipants.length} participant(s) blocked by hot/unparseable frontmatter guard: ${detail}`);
|
|
1341
|
-
emitMergeFailureSkips("merge_participant_blocked");
|
|
1342
|
-
continue;
|
|
1343
|
-
}
|
|
1344
|
-
// Backup secondaries before deleting
|
|
1345
|
-
for (const secRef of op.secondaries) {
|
|
1346
|
-
const secEntry = memoryByRef.get(secRef);
|
|
1347
|
-
if (secEntry && fs.existsSync(secEntry.filePath)) {
|
|
1348
|
-
backupFile(secEntry.filePath, backupDir, secEntry.name);
|
|
1349
|
-
}
|
|
1350
|
-
}
|
|
1351
|
-
// Write merged primary
|
|
1352
|
-
try {
|
|
1353
|
-
const parsedPrimary = parseAssetRef(op.primary);
|
|
1354
|
-
await writeAssetToSource(target.source, target.config, parsedPrimary, mergedContent);
|
|
1355
|
-
}
|
|
1356
|
-
catch (e) {
|
|
1357
|
-
warnings.push(`Merge: write failed for ${op.primary}: ${String(e)}`);
|
|
1358
|
-
emitMergeFailureSkips("merge_write_failed");
|
|
1359
|
-
continue;
|
|
1360
|
-
}
|
|
1361
|
-
// Archive and delete secondaries (P1-B: soft-invalidation)
|
|
1362
|
-
for (const secRef of op.secondaries) {
|
|
1363
|
-
const secEntry = memoryByRef.get(secRef);
|
|
1364
|
-
if (!secEntry)
|
|
1365
|
-
continue;
|
|
1366
|
-
if (fs.existsSync(secEntry.filePath)) {
|
|
1367
|
-
archiveMemory(secEntry.filePath, stashDir, secRef, "merged into primary", opIndex, op.primary, warnings);
|
|
1368
|
-
}
|
|
1369
|
-
try {
|
|
1370
|
-
const parsedSec = parseAssetRef(secRef);
|
|
1371
|
-
await deleteAssetFromSource(target.source, target.config, parsedSec);
|
|
1372
|
-
markJournalCompleted(stashDir, secRef);
|
|
1373
|
-
}
|
|
1374
|
-
catch (e) {
|
|
1375
|
-
warnings.push(`Merge: delete failed for ${secRef}: ${String(e)}`);
|
|
1376
|
-
}
|
|
1377
|
-
}
|
|
1378
|
-
markJournalCompleted(stashDir, op.primary);
|
|
1379
|
-
merged++;
|
|
1380
|
-
// 2026-05-26 accounting-leak fix: `merged` is op-level, but each
|
|
1381
|
-
// successful merge actions `1 + secondaries.length` memories. Without
|
|
1382
|
-
// this counter the accounting invariant breaks by `secondaries.length`
|
|
1383
|
-
// per successful merge (chunk loop excluded all secondaries from
|
|
1384
|
-
// judgedNoAction via targetRefs, but only the primary is credited to
|
|
1385
|
-
// `merged`). Count only loaded-memory secondaries; phantom secondary
|
|
1386
|
-
// refs never affected any chunk's targetRefs in the first place.
|
|
1387
|
-
for (const secRef of op.secondaries) {
|
|
1388
|
-
if (memoryByRef.has(secRef))
|
|
1389
|
-
mergedSecondaries++;
|
|
1390
|
-
}
|
|
1391
|
-
// Prune consumed refs from memoryByRef so later ops in this run cannot
|
|
1392
|
-
// reference an absorbed secondary as a merge primary and proceed with a
|
|
1393
|
-
// stale entry. Primary is rewritten (not deleted), so we only remove
|
|
1394
|
-
// secondaries; the primary ref remains valid under its new content.
|
|
1395
|
-
for (const secRef of op.secondaries) {
|
|
1396
|
-
memoryByRef.delete(secRef);
|
|
1397
|
-
}
|
|
1398
|
-
}
|
|
1399
|
-
else if (op.op === "delete") {
|
|
1400
|
-
const entry = memoryByRef.get(op.ref);
|
|
1401
|
-
if (!entry) {
|
|
1402
|
-
warnings.push(`Delete: ${op.ref} not found in loaded memories — skipping.`);
|
|
1403
|
-
// Phantom ref: not in the batch so not in processed. Pushing to
|
|
1404
|
-
// skipReasons would inflate Σ(skipReasons) without a matching processed
|
|
1405
|
-
// entry, breaking the accounting invariant. Visibility is preserved via
|
|
1406
|
-
// the warnings array above.
|
|
1407
|
-
continue;
|
|
1408
|
-
}
|
|
1409
|
-
// captureMode:hot guard — refuse to delete user-captured memories OR
|
|
1410
|
-
// memories whose frontmatter is unparseable (could have hidden the hot
|
|
1411
|
-
// flag). The consolidate LLM was deleting hot-captured user memos as
|
|
1412
|
-
// "redundant" — 14 such deletes were silently archived between
|
|
1413
|
-
// 2026-05-19 and 2026-05-20 before this guard. Hot memories are
|
|
1414
|
-
// user-explicit and may only be deleted by the user.
|
|
1415
|
-
const guard = consolidateGuardStatus(entry.filePath);
|
|
1416
|
-
if (guard === "hot" || guard === "unparseable") {
|
|
1417
|
-
warnings.push(`Delete: refused for ${op.ref} — ${guard === "hot" ? "captureMode:hot (user-explicit; never auto-delete)" : "frontmatter unparseable (cannot verify hot flag absent)"}. Reason from LLM: "${op.reason ?? "n/a"}"`);
|
|
1418
|
-
pushSkipReason("delete", op.ref, "captureMode_hot_refused");
|
|
1419
|
-
continue;
|
|
1420
|
-
}
|
|
1421
|
-
if (fs.existsSync(entry.filePath)) {
|
|
1422
|
-
backupFile(entry.filePath, backupDir, entry.name);
|
|
1423
|
-
// P1-B: soft-invalidation archive before hard delete
|
|
1424
|
-
archiveMemory(entry.filePath, stashDir, op.ref, op.reason, opIndex, undefined, warnings);
|
|
1425
|
-
}
|
|
1426
|
-
try {
|
|
1427
|
-
const parsedRef = parseAssetRef(op.ref);
|
|
1428
|
-
await deleteAssetFromSource(target.source, target.config, parsedRef);
|
|
1429
|
-
markJournalCompleted(stashDir, op.ref);
|
|
1430
|
-
deleted++;
|
|
1431
|
-
// Prune from memoryByRef so later ops in this run cannot reference a
|
|
1432
|
-
// deleted memory as a merge primary or secondary.
|
|
1433
|
-
memoryByRef.delete(op.ref);
|
|
1434
|
-
}
|
|
1435
|
-
catch (e) {
|
|
1436
|
-
// Distinguish "file already absent" from genuine failures. A prior run
|
|
1437
|
-
// may have deleted the file but the DB was not yet re-indexed, so the
|
|
1438
|
-
// ref still appeared in memoryByRef. The delete goal is already met.
|
|
1439
|
-
const msg = e instanceof Error ? e.message : String(e);
|
|
1440
|
-
if (msg.includes("not found in source")) {
|
|
1441
|
-
warnings.push(`Delete: ${op.ref} — file already absent (stale DB entry); skipping.`);
|
|
1442
|
-
pushSkipReason("delete", op.ref, "delete_already_gone");
|
|
1443
|
-
}
|
|
1444
|
-
else {
|
|
1445
|
-
warnings.push(`Delete: failed for ${op.ref}: ${String(e)}`);
|
|
1446
|
-
pushSkipReason("delete", op.ref, "delete_failed");
|
|
1447
|
-
}
|
|
1448
|
-
}
|
|
1449
|
-
}
|
|
1450
|
-
else if (op.op === "promote") {
|
|
1451
|
-
const entry = memoryByRef.get(op.ref);
|
|
1452
|
-
if (!entry) {
|
|
1453
|
-
warnings.push(`Promote: ${op.ref} not found in loaded memories — skipping.`);
|
|
1454
|
-
// Phantom ref: not in processed, so no skipReason (same rationale as
|
|
1455
|
-
// delete_ref_missing above).
|
|
1456
|
-
continue;
|
|
1457
|
-
}
|
|
1458
|
-
// Within-run source-ref dedup: skip if this source memory was already
|
|
1459
|
-
// promoted earlier in this run (safety belt — mergePlans already
|
|
1460
|
-
// deduplicates promote ops by source ref via Map, but this guard also
|
|
1461
|
-
// catches any future code paths that bypass mergePlans).
|
|
1462
|
-
if (promotedSourceRefs.has(op.ref)) {
|
|
1463
|
-
warnings.push(`Skipping promote: ${op.ref} already promoted in this run`);
|
|
1464
|
-
pushSkipReason("promote", op.ref, "promote_already_promoted_this_run");
|
|
1465
|
-
continue;
|
|
1466
|
-
}
|
|
1467
|
-
let knowledgeRef = op.knowledgeRef;
|
|
1468
|
-
try {
|
|
1469
|
-
parseAssetRef(knowledgeRef);
|
|
1470
|
-
}
|
|
1471
|
-
catch {
|
|
1472
|
-
const slug = op.knowledgeRef
|
|
1473
|
-
.replace(/^knowledge:/, "")
|
|
1474
|
-
.replace(/[^a-z0-9-]/gi, "-")
|
|
1475
|
-
.toLowerCase();
|
|
1476
|
-
knowledgeRef = `knowledge:${slug}`;
|
|
1477
|
-
warnings.push(`Normalized invalid ref "${op.knowledgeRef}" → "${knowledgeRef}"`);
|
|
1478
|
-
}
|
|
1479
|
-
// Idempotency: check pending proposals by target ref
|
|
1480
|
-
const existingProposals = listProposals(stashDir, { ref: knowledgeRef });
|
|
1481
|
-
if (existingProposals.some((p) => p.status === "pending")) {
|
|
1482
|
-
warnings.push(`Skipping promote: pending proposal already exists for ${knowledgeRef}`);
|
|
1483
|
-
pushSkipReason("promote", op.ref, "promote_pending_proposal_exists");
|
|
1484
|
-
continue;
|
|
1485
|
-
}
|
|
1486
|
-
// Idempotency: check if knowledge asset already exists
|
|
1487
|
-
const parsedKnowledgeRef = parseAssetRef(knowledgeRef);
|
|
1488
|
-
const destPath = path.join(target.source.path, "knowledge", `${parsedKnowledgeRef.name}.md`);
|
|
1489
|
-
if (fs.existsSync(destPath)) {
|
|
1490
|
-
warnings.push(`Skipping promote: ${knowledgeRef} already exists in source`);
|
|
1491
|
-
pushSkipReason("promote", op.ref, "promote_already_exists");
|
|
1492
|
-
continue;
|
|
1493
|
-
}
|
|
1494
|
-
let memoryContent = "";
|
|
1495
|
-
try {
|
|
1496
|
-
memoryContent = fs.readFileSync(entry.filePath, "utf8");
|
|
1497
|
-
}
|
|
1498
|
-
catch (e) {
|
|
1499
|
-
warnings.push(`Promote: could not read ${op.ref}: ${String(e)}`);
|
|
1500
|
-
pushSkipReason("promote", op.ref, "promote_read_failed");
|
|
1501
|
-
continue;
|
|
1502
|
-
}
|
|
1503
|
-
// Defensive sanitization: legacy memory files written by older
|
|
1504
|
-
// consolidate runs may still carry outer code fences or broken YAML.
|
|
1505
|
-
// Strip them here so we never propose a polluted asset.
|
|
1506
|
-
const promoteSanitized = sanitizeMergedContent(memoryContent);
|
|
1507
|
-
if (!promoteSanitized.ok) {
|
|
1508
|
-
warnings.push(`Promote: rejected ${op.ref} — source memory failed sanitization (${promoteSanitized.reason}).`);
|
|
1509
|
-
pushSkipReason("promote", op.ref, "promote_sanitization_failed");
|
|
1510
|
-
continue;
|
|
1511
|
-
}
|
|
1512
|
-
memoryContent = promoteSanitized.result.content;
|
|
1513
|
-
// SOURCE_SUPERSEDED guard: refuse to promote a memory whose source
|
|
1514
|
-
// frontmatter carries `status: superseded`. Predicate at module top
|
|
1515
|
-
// (`hasSupersededStatus`) so tests can exercise it directly.
|
|
1516
|
-
if (hasSupersededStatus(promoteSanitized.result.frontmatter)) {
|
|
1517
|
-
warnings.push(`Promote: refused for ${op.ref} → ${knowledgeRef} — source memory has status:superseded; superseded memories are not promotable knowledge.`);
|
|
1518
|
-
pushSkipReason("promote", op.ref, "promote_superseded");
|
|
1519
|
-
continue;
|
|
1520
|
-
}
|
|
1521
|
-
// Parse the source memory up-front so the body/frontmatter checks below
|
|
1522
|
-
// share the same parsed view.
|
|
1523
|
-
const parsedMemory = parseFrontmatter(memoryContent);
|
|
1524
|
-
// Reject sources whose body is too small to make useful knowledge.
|
|
1525
|
-
// Observed failure: memory files whose body is literally a tags string
|
|
1526
|
-
// ("discord,notification,send-notification") get promoted to knowledge
|
|
1527
|
-
// proposals that no reviewer would accept. Threshold is conservative —
|
|
1528
|
-
// 100 chars catches single-line tag dumps without rejecting genuinely
|
|
1529
|
-
// terse but valid notes.
|
|
1530
|
-
const PROMOTE_BODY_MIN_CHARS = 100;
|
|
1531
|
-
const sourceBody = parsedMemory.content.trim();
|
|
1532
|
-
if (sourceBody.length < PROMOTE_BODY_MIN_CHARS) {
|
|
1533
|
-
warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — source memory body is too small (${sourceBody.length} chars; need ≥${PROMOTE_BODY_MIN_CHARS}) to make useful knowledge.`);
|
|
1534
|
-
pushSkipReason("promote", op.ref, "promote_source_too_small");
|
|
1535
|
-
continue;
|
|
1536
|
-
}
|
|
1537
|
-
// Cross-run + within-run content dedup: if an identical body already
|
|
1538
|
-
// exists in ANY pending consolidate proposal (regardless of target ref),
|
|
1539
|
-
// skip. This prevents duplicate proposals when:
|
|
1540
|
-
// (a) Multiple source memories have identical bodies but differ only
|
|
1541
|
-
// in noise frontmatter (`inferenceProcessed: true` twin alongside
|
|
1542
|
-
// the original; differing `updated:` timestamps; etc.) — the body
|
|
1543
|
-
// is the load-bearing content, so dedup must hash on body only.
|
|
1544
|
-
// (b) A prior run created a proposal for the same body under a
|
|
1545
|
-
// different knowledgeRef slug.
|
|
1546
|
-
const bodyHash = createHash("sha256").update(sourceBody, "utf8").digest("hex");
|
|
1547
|
-
const allPendingConsolidateProposals = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
|
|
1548
|
-
const contentDupProposal = allPendingConsolidateProposals.find((p) => {
|
|
1549
|
-
const otherBody = parseFrontmatter(p.payload.content).content.trim();
|
|
1550
|
-
return createHash("sha256").update(otherBody, "utf8").digest("hex") === bodyHash;
|
|
1551
|
-
});
|
|
1552
|
-
if (contentDupProposal) {
|
|
1553
|
-
warnings.push(`Skipping promote: identical body already pending as proposal ${contentDupProposal.id} (ref: ${contentDupProposal.ref}); skipping duplicate for ${op.ref} → ${knowledgeRef}`);
|
|
1554
|
-
pushSkipReason("promote", op.ref, "dedup_pending_proposal");
|
|
1555
|
-
continue;
|
|
1556
|
-
}
|
|
1557
|
-
try {
|
|
1558
|
-
// Use LLM-provided description; fall back to memory's own description
|
|
1559
|
-
// (post-sanitization frontmatter is authoritative).
|
|
1560
|
-
const description = (typeof op.description === "string" && op.description.trim()
|
|
1561
|
-
? op.description.trim()
|
|
1562
|
-
: parsedMemory.data?.description?.trim()) ?? "";
|
|
1563
|
-
// Validate the resolved frontmatter before emitting a proposal.
|
|
1564
|
-
// Required field: non-empty description. Reject obvious truncation
|
|
1565
|
-
// markers (description ends with `,`/`;`/`:`/`...`/hanging connector)
|
|
1566
|
-
// so the queue never sees half-formed metadata that the reviewer
|
|
1567
|
-
// would only reject.
|
|
1568
|
-
const fmCheck = validateProposalFrontmatter({ description });
|
|
1569
|
-
if (!fmCheck.ok) {
|
|
1570
|
-
warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — ${fmCheck.reason}.`);
|
|
1571
|
-
pushSkipReason("promote", op.ref, "promote_invalid_frontmatter");
|
|
1572
|
-
continue;
|
|
1573
|
-
}
|
|
1574
|
-
// Merge `description` INTO the body's YAML frontmatter so it lands in
|
|
1575
|
-
// the on-disk asset when the proposal is accepted. The descriptionQuality
|
|
1576
|
-
// validator parses `payload.content` body (not the envelope
|
|
1577
|
-
// `payload.frontmatter`), and a memory's native frontmatter has
|
|
1578
|
-
// `captureMode`/`beliefState`/etc. but never `description` — without
|
|
1579
|
-
// this merge, 60+ pending proposals were blocked at accept-time with
|
|
1580
|
-
// MISSING_FRONTMATTER_DESCRIPTION even though the envelope had it.
|
|
1581
|
-
// (The body-frontmatter assumption baked into the 2026-05-20 comment
|
|
1582
|
-
// below was wrong: body fm and envelope fm only converge when the
|
|
1583
|
-
// writer explicitly merges them, which it now does.)
|
|
1584
|
-
const mergedBodyFm = {
|
|
1585
|
-
...(parsedMemory.data ?? {}),
|
|
1586
|
-
description,
|
|
1587
|
-
};
|
|
1588
|
-
const serializedMergedFm = serializeFrontmatter(mergedBodyFm);
|
|
1589
|
-
const proposalContent = assembleAssetFromString(serializedMergedFm, parsedMemory.content);
|
|
1590
|
-
// Pre-emit dedup against pending consolidate proposals from the
|
|
1591
|
-
// same improve run (slug-variant match). The cross-run content-hash
|
|
1592
|
-
// dedup inside `mergePlans` handles duplicates against existing
|
|
1593
|
-
// stash assets — see commit history for the deletion of the
|
|
1594
|
-
// unbounded embedding + cross-type slug branches.
|
|
1595
|
-
const dedup = await checkPreEmitDedup({
|
|
1596
|
-
candidateRef: knowledgeRef,
|
|
1597
|
-
candidateText: `${description}. ${memoryContent}`,
|
|
1598
|
-
stashDir,
|
|
1599
|
-
config,
|
|
1600
1378
|
});
|
|
1601
|
-
if (dedup.duplicate) {
|
|
1602
|
-
warnings.push(`Promote: skipped ${op.ref} → ${knowledgeRef} — ${dedup.reason}.`);
|
|
1603
|
-
pushSkipReason("promote", op.ref, "promote_dedup_window");
|
|
1604
|
-
continue;
|
|
1605
|
-
}
|
|
1606
|
-
const proposalResult = createProposal(stashDir, {
|
|
1607
|
-
ref: knowledgeRef,
|
|
1608
|
-
source: "consolidate",
|
|
1609
|
-
sourceRun,
|
|
1610
|
-
payload: {
|
|
1611
|
-
content: proposalContent,
|
|
1612
|
-
frontmatter: { description },
|
|
1613
|
-
},
|
|
1614
|
-
...(typeof op.confidence === "number" ? { confidence: op.confidence } : {}),
|
|
1615
|
-
});
|
|
1616
|
-
if (isProposalSkipped(proposalResult)) {
|
|
1617
|
-
warnings.push(`Promote: skipped proposal for ${op.ref} (${proposalResult.reason}): ${proposalResult.message}`);
|
|
1618
|
-
pushSkipReason("promote", op.ref, `promote_proposal_${proposalResult.reason}`);
|
|
1619
|
-
}
|
|
1620
|
-
else {
|
|
1621
|
-
promoted.push(proposalResult.id);
|
|
1622
|
-
promotedSourceRefs.add(op.ref);
|
|
1623
|
-
markJournalCompleted(stashDir, op.ref);
|
|
1624
|
-
}
|
|
1625
|
-
}
|
|
1626
|
-
catch (e) {
|
|
1627
|
-
warnings.push(`Promote: createProposal failed for ${op.ref}: ${String(e)}`);
|
|
1628
|
-
pushSkipReason("promote", op.ref, "promote_create_failed");
|
|
1629
|
-
}
|
|
1630
|
-
}
|
|
1631
|
-
else if (op.op === "contradict") {
|
|
1632
|
-
// C-3 / #382: Write contradictedBy edges so resolveFamilyContradictions
|
|
1633
|
-
// (the SCC resolver in memory-improve.ts) has edges to work on.
|
|
1634
|
-
// Zep arXiv:2501.13956 §3 — unified belief-revision with contradiction edges.
|
|
1635
|
-
const entry = memoryByRef.get(op.ref);
|
|
1636
|
-
const contradictorEntry = memoryByRef.get(op.contradictedByRef);
|
|
1637
|
-
if (!entry) {
|
|
1638
|
-
warnings.push(`Contradict: ${op.ref} not found in loaded memories — skipping.`);
|
|
1639
|
-
// Phantom ref: not in processed, so no skipReason (same rationale as
|
|
1640
|
-
// delete_ref_missing).
|
|
1641
|
-
continue;
|
|
1642
|
-
}
|
|
1643
|
-
if (!contradictorEntry) {
|
|
1644
|
-
warnings.push(`Contradict: ${op.contradictedByRef} not found — skipping.`);
|
|
1645
|
-
// op.ref IS in the batch (entry found above) so the skipReason is
|
|
1646
|
-
// correctly charged against a real processed memory.
|
|
1647
|
-
pushSkipReason("contradict", op.ref, "contradict_target_missing");
|
|
1648
|
-
continue;
|
|
1649
|
-
}
|
|
1650
|
-
try {
|
|
1651
|
-
// Write the contradiction edge: op.ref is contradicted by op.contradictedByRef
|
|
1652
|
-
writeContradictEdge(entry.filePath, op.contradictedByRef);
|
|
1653
|
-
contradicted++;
|
|
1654
|
-
markJournalCompleted(stashDir, op.ref);
|
|
1655
|
-
}
|
|
1656
|
-
catch (e) {
|
|
1657
|
-
warnings.push(`Contradict: failed to write edge for ${op.ref}: ${String(e)}`);
|
|
1658
|
-
pushSkipReason("contradict", op.ref, "contradict_write_failed");
|
|
1659
|
-
}
|
|
1660
|
-
}
|
|
1661
|
-
}
|
|
1662
|
-
// 0.9.0 (issue #507): batch-at-boundary commit. The merge/delete loop above
|
|
1663
|
-
// wrote one merged primary and deleted N secondaries to the resolved target
|
|
1664
|
-
// with NO per-asset commit. If the target is a writable git source and any
|
|
1665
|
-
// asset was mutated, commit the whole batch ONCE here (stages .akm/ +
|
|
1666
|
-
// siblings together). No-op for filesystem/primary-stash targets.
|
|
1667
|
-
if (merged > 0 || deleted > 0) {
|
|
1668
|
-
commitWriteTargetBoundary(target, `Consolidate: ${merged} merged, ${deleted} removed`);
|
|
1669
|
-
}
|
|
1670
|
-
cleanupJournal(stashDir, timestamp);
|
|
1671
|
-
// TTL cleanup: remove archive entries older than archiveRetentionDays (default 90).
|
|
1672
|
-
// C-5 / #391: emit an `archive_cleanup` event before each deletion so the
|
|
1673
|
-
// audit trail records what was lost. Outbox pattern (EIP, Hohpe-Woolf) —
|
|
1674
|
-
// any event that is recorded must be queryable; silent deletes are an anti-pattern.
|
|
1675
|
-
const archiveDir = path.join(stashDir, ".akm", "archive");
|
|
1676
|
-
if (fs.existsSync(archiveDir)) {
|
|
1677
|
-
const retentionMs = (config.archiveRetentionDays ?? 90) * 86_400_000;
|
|
1678
|
-
const cutoff = Date.now() - retentionMs;
|
|
1679
|
-
for (const fname of fs.readdirSync(archiveDir)) {
|
|
1680
|
-
const fp = path.join(archiveDir, fname);
|
|
1681
|
-
try {
|
|
1682
|
-
const stat = fs.statSync(fp);
|
|
1683
|
-
if (stat.mtimeMs < cutoff) {
|
|
1684
|
-
// Emit event before deletion so the record survives the purge.
|
|
1685
|
-
appendEvent({
|
|
1686
|
-
eventType: "archive_cleanup",
|
|
1687
|
-
metadata: {
|
|
1688
|
-
file: fname,
|
|
1689
|
-
filePath: fp,
|
|
1690
|
-
ageMs: Date.now() - stat.mtimeMs,
|
|
1691
|
-
retentionMs,
|
|
1692
|
-
},
|
|
1693
|
-
});
|
|
1694
|
-
fs.unlinkSync(fp);
|
|
1695
|
-
}
|
|
1696
|
-
}
|
|
1697
|
-
catch {
|
|
1698
|
-
/* ignore race conditions */
|
|
1699
1379
|
}
|
|
1700
1380
|
}
|
|
1701
1381
|
}
|
|
1382
|
+
// -- Pass 3: execute the plan against the filesystem ------------------------
|
|
1383
|
+
const { merged, deleted, contradicted, mergeFloorViolations, mergedSecondaries, promoted } = await applyConsolidationPlan(config, stashDir, sourceRun, memories, warnings, allOps, accounting, dedupCollapsed, opts.improveProfile);
|
|
1384
|
+
const runDurationMs = Date.now() - startMs;
|
|
1385
|
+
const budgetFraction = opts.runBudgetMs !== undefined && opts.runBudgetMs > 0 ? runDurationMs / opts.runBudgetMs : undefined;
|
|
1702
1386
|
return {
|
|
1703
1387
|
schemaVersion: 1,
|
|
1704
1388
|
ok: true,
|
|
@@ -1708,245 +1392,577 @@ export async function akmConsolidate(opts = {}) {
|
|
|
1708
1392
|
target: sourceName,
|
|
1709
1393
|
processed: memories.length,
|
|
1710
1394
|
merged,
|
|
1711
|
-
|
|
1395
|
+
// #617: fold the deterministic dedup pre-pass collapses into the reported
|
|
1396
|
+
// deleted count. Each collapse removed exactly one variant file with NO
|
|
1397
|
+
// LLM call before the LLM pass ran on the pruned pool.
|
|
1398
|
+
deleted: deleted + dedupCollapsed,
|
|
1712
1399
|
promoted,
|
|
1713
1400
|
contradicted,
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1401
|
+
mergeFloorViolations,
|
|
1402
|
+
failedChunks: accounting.totalChunksFailed,
|
|
1403
|
+
totalChunks,
|
|
1404
|
+
judgedNoAction: accounting.judgedNoAction,
|
|
1405
|
+
skipReasons: accounting.skipReasons,
|
|
1718
1406
|
mergedSecondaries,
|
|
1719
|
-
failedChunkMemories,
|
|
1407
|
+
failedChunkMemories: accounting.failedChunkMemories,
|
|
1720
1408
|
warnings,
|
|
1721
|
-
durationMs:
|
|
1409
|
+
durationMs: runDurationMs,
|
|
1410
|
+
perfTelemetry: {
|
|
1411
|
+
dedupPoolSize: perfMs.dedupPoolSize,
|
|
1412
|
+
llmPoolSize,
|
|
1413
|
+
judgedCacheSkipped: perfMs.judgedCacheSkipped,
|
|
1414
|
+
embedMs: embedTelemetry.embedMs,
|
|
1415
|
+
embedCacheHits: embedTelemetry.cacheHits,
|
|
1416
|
+
embedCacheMisses: embedTelemetry.cacheMisses,
|
|
1417
|
+
...(budgetFraction !== undefined ? { estimatedBudgetFractionUsed: budgetFraction } : {}),
|
|
1418
|
+
},
|
|
1722
1419
|
};
|
|
1723
1420
|
}
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
//
|
|
1728
|
-
//
|
|
1729
|
-
//
|
|
1730
|
-
//
|
|
1731
|
-
//
|
|
1732
|
-
//
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
// 3. Truncated descriptions hitting token cutoffs — the model's max_tokens
|
|
1740
|
-
// runs out mid-sentence, leaving things like
|
|
1741
|
-
// `description: "Tables in narrow column containers need max-width:100% +"`
|
|
1742
|
-
// with no closing context.
|
|
1743
|
-
//
|
|
1744
|
-
// `sanitizeMergedContent` and `validateProposalFrontmatter` defend against
|
|
1745
|
-
// all three at the point where LLM output is consumed.
|
|
1746
|
-
/**
|
|
1747
|
-
* Attempt to recover a frontmatter block that is missing its closing `---`.
|
|
1748
|
-
*
|
|
1749
|
-
* Scans lines after the opening `---` for the first blank line or the first
|
|
1750
|
-
* line that cannot be a YAML scalar (i.e. not a key-value, indented
|
|
1751
|
-
* continuation, comment, or list item). Injects `---` before that line so
|
|
1752
|
-
* the normal parser can proceed.
|
|
1753
|
-
*
|
|
1754
|
-
* Returns the patched string on success, or `null` if the structure is too
|
|
1755
|
-
* ambiguous to recover safely (e.g. no opening `---`, or no body content
|
|
1756
|
-
* found after the frontmatter key-value lines).
|
|
1757
|
-
*/
|
|
1758
|
-
function recoverMalformedFrontmatter(raw) {
|
|
1759
|
-
if (!raw.startsWith("---"))
|
|
1760
|
-
return null;
|
|
1761
|
-
const lines = raw.split(/\r?\n/);
|
|
1762
|
-
// Skip the opening `---` line (index 0).
|
|
1763
|
-
let insertAt = -1;
|
|
1764
|
-
for (let i = 1; i < lines.length; i++) {
|
|
1765
|
-
const line = lines[i];
|
|
1766
|
-
// A blank line marks the end of the frontmatter block in many YAML variants.
|
|
1767
|
-
if (line.trim() === "") {
|
|
1768
|
-
insertAt = i;
|
|
1769
|
-
break;
|
|
1421
|
+
/** Execute one `merge` op (behavior-identical to the former inlined branch). */
|
|
1422
|
+
export async function handleMergeOp(op, opIndex, ctx) {
|
|
1423
|
+
const { config, stashDir, target, backupDir, memoryByRef, warnings, pushSkipReason, counts } = ctx;
|
|
1424
|
+
// Accounting helper: emit a per-participant skipReason for failed
|
|
1425
|
+
// merges so primary + every loaded-memory secondary land in the
|
|
1426
|
+
// structured skip histogram. Pre-2026-05-26 only the primary was
|
|
1427
|
+
// counted (1 skipReason per failed merge), leaving N secondaries
|
|
1428
|
+
// unaccounted for in the `processed == actioned + noAction + Σskips`
|
|
1429
|
+
// invariant — the source of the 4–11 silent leaks per run.
|
|
1430
|
+
const emitMergeFailureSkips = (reason) => {
|
|
1431
|
+
if (memoryByRef.has(op.primary))
|
|
1432
|
+
pushSkipReason("merge", op.primary, reason);
|
|
1433
|
+
for (const secRef of op.secondaries) {
|
|
1434
|
+
if (memoryByRef.has(secRef))
|
|
1435
|
+
pushSkipReason("merge", secRef, reason);
|
|
1770
1436
|
}
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1437
|
+
};
|
|
1438
|
+
const primaryEntry = memoryByRef.get(op.primary);
|
|
1439
|
+
if (!primaryEntry) {
|
|
1440
|
+
// This fires when a prior op in the same run consumed this ref as a
|
|
1441
|
+
// secondary and Fix-A pruned it from memoryByRef. It should NOT fire
|
|
1442
|
+
// for hallucinated primaries (those are dropped by mergePlans() before
|
|
1443
|
+
// reaching here). If this counter is non-zero, suspect an intra-run
|
|
1444
|
+
// cross-chunk race, not a filter regression.
|
|
1445
|
+
warnings.push(`Merge: primary ${op.primary} not found in loaded memories (pruned by prior op this run) — skipping.`);
|
|
1446
|
+
emitMergeFailureSkips("merge_primary_missing");
|
|
1447
|
+
return;
|
|
1448
|
+
}
|
|
1449
|
+
// Defense-in-depth: even if the entry is in memoryByRef (pre-flight ran
|
|
1450
|
+
// before this run's own ops), the file may have been deleted by a
|
|
1451
|
+
// concurrent process or an edge case the pre-flight filter missed.
|
|
1452
|
+
if (!fs.existsSync(primaryEntry.filePath)) {
|
|
1453
|
+
warnings.push(`Merge: primary ${op.primary} file gone at execution time (stale entry) — skipping.`);
|
|
1454
|
+
emitMergeFailureSkips("merge_primary_file_gone");
|
|
1455
|
+
return;
|
|
1456
|
+
}
|
|
1457
|
+
// Phase B: generate merged content
|
|
1458
|
+
const secondaryBodies = [];
|
|
1459
|
+
for (const secRef of op.secondaries) {
|
|
1460
|
+
const secEntry = memoryByRef.get(secRef);
|
|
1461
|
+
if (!secEntry) {
|
|
1462
|
+
warnings.push(`Merge: secondary ${secRef} not found — skipping merge op.`);
|
|
1463
|
+
// No accounting impact: a missing secondary is a phantom ref and
|
|
1464
|
+
// never contributed to any chunk's targetRefs reduction. We still
|
|
1465
|
+
// continue the loop to gather the remaining valid secondaries.
|
|
1466
|
+
continue;
|
|
1780
1467
|
}
|
|
1468
|
+
secondaryBodies.push(secRef);
|
|
1781
1469
|
}
|
|
1782
|
-
if (
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
}
|
|
1787
|
-
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
|
|
1798
|
-
|
|
1799
|
-
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
}
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1470
|
+
if (secondaryBodies.length === 0) {
|
|
1471
|
+
warnings.push(`Merge: ${op.primary} has no valid secondaries — skipping.`);
|
|
1472
|
+
emitMergeFailureSkips("merge_no_valid_secondaries");
|
|
1473
|
+
return;
|
|
1474
|
+
}
|
|
1475
|
+
// Pre-flight hot guard — skip the LLM call entirely if any participant
|
|
1476
|
+
// is hot or unparseable. Without this, mixed chunks still send hot merges
|
|
1477
|
+
// to the planner which proposes them; generateMergedContent() is then
|
|
1478
|
+
// called, produces output without `description`, and the skip is
|
|
1479
|
+
// misattributed to merge_missing_description instead of the real cause.
|
|
1480
|
+
const preflightParticipants = [op.primary, ...op.secondaries];
|
|
1481
|
+
const preflightBlocked = preflightParticipants.flatMap((ref) => {
|
|
1482
|
+
const e = memoryByRef.get(ref);
|
|
1483
|
+
if (!e)
|
|
1484
|
+
return [];
|
|
1485
|
+
const verdict = consolidateGuardStatus(e.filePath);
|
|
1486
|
+
if (verdict === "hot" || verdict === "unparseable")
|
|
1487
|
+
return [{ ref, verdict }];
|
|
1488
|
+
return [];
|
|
1489
|
+
});
|
|
1490
|
+
if (preflightBlocked.length > 0) {
|
|
1491
|
+
const detail = preflightBlocked.map((p) => `${p.ref} (${p.verdict})`).join(", ");
|
|
1492
|
+
warnings.push(`Merge: refused for ${op.primary} — ${preflightBlocked.length} participant(s) blocked by hot/unparseable frontmatter guard (pre-flight): ${detail}`);
|
|
1493
|
+
emitMergeFailureSkips("merge_participant_blocked");
|
|
1494
|
+
return;
|
|
1495
|
+
}
|
|
1496
|
+
let primaryBody = "";
|
|
1497
|
+
try {
|
|
1498
|
+
primaryBody = fs.readFileSync(primaryEntry.filePath, "utf8");
|
|
1499
|
+
}
|
|
1500
|
+
catch {
|
|
1501
|
+
warnings.push(`Merge: could not read primary ${op.primary} — skipping.`);
|
|
1502
|
+
emitMergeFailureSkips("merge_read_failed");
|
|
1503
|
+
return;
|
|
1504
|
+
}
|
|
1505
|
+
const mergeResult = await generateMergedContent(config, op.primary, primaryBody, op.secondaries, memoryByRef, ctx.improveProfile);
|
|
1506
|
+
if ("error" in mergeResult) {
|
|
1507
|
+
warnings.push(`Merge: ${mergeResult.error} for ${mergeResult.detail}.`);
|
|
1508
|
+
emitMergeFailureSkips(mergeResult.error);
|
|
1509
|
+
return;
|
|
1510
|
+
}
|
|
1511
|
+
let mergedContent = mergeResult.content;
|
|
1512
|
+
// Validate frontmatter of merged content — must have a `---` block
|
|
1513
|
+
// with at minimum a `description` field. We parse via the hand-rolled
|
|
1514
|
+
// parser (cheap) AND require non-empty description. This guards against
|
|
1515
|
+
// the historical defect where merged memories were written back with
|
|
1516
|
+
// empty `description` and later polluted the promote path.
|
|
1517
|
+
let parsedMerged;
|
|
1518
|
+
try {
|
|
1519
|
+
parsedMerged = parseFrontmatter(mergedContent);
|
|
1520
|
+
}
|
|
1521
|
+
catch {
|
|
1522
|
+
warnings.push(`Merge: merged content for ${op.primary} has invalid frontmatter — skipping.`);
|
|
1523
|
+
emitMergeFailureSkips("merge_invalid_frontmatter");
|
|
1524
|
+
return;
|
|
1525
|
+
}
|
|
1526
|
+
if (parsedMerged.frontmatter === null) {
|
|
1527
|
+
warnings.push(`Merge: merged content for ${op.primary} has no frontmatter block — skipping.`);
|
|
1528
|
+
emitMergeFailureSkips("merge_invalid_frontmatter");
|
|
1529
|
+
return;
|
|
1530
|
+
}
|
|
1531
|
+
const mergedDesc = parsedMerged.data.description;
|
|
1532
|
+
if (typeof mergedDesc !== "string" || mergedDesc.trim().length === 0) {
|
|
1533
|
+
warnings.push(`Merge: merged content for ${op.primary} missing description — skipping.`);
|
|
1534
|
+
emitMergeFailureSkips("merge_missing_description");
|
|
1535
|
+
return;
|
|
1536
|
+
}
|
|
1537
|
+
const truncReason = detectTruncatedDescription(mergedDesc);
|
|
1538
|
+
if (truncReason) {
|
|
1539
|
+
warnings.push(`Merge: merged content for ${op.primary} has truncated description (${truncReason}) — skipping.`);
|
|
1540
|
+
emitMergeFailureSkips("merge_truncated_description");
|
|
1541
|
+
return;
|
|
1542
|
+
}
|
|
1543
|
+
// captureMode:hot guard — refuse the merge if ANY participating memory
|
|
1544
|
+
// (primary or secondary) was user-captured or has unparseable frontmatter
|
|
1545
|
+
// (could have hidden a hot flag). Hot memories are user-explicit and
|
|
1546
|
+
// must not be deleted/overwritten by the consolidate LLM. 14 user
|
|
1547
|
+
// memories were silent-deleted by consolidate before this guard landed;
|
|
1548
|
+
// recovery required copying from .akm/archive/ by hand.
|
|
1549
|
+
const mergeParticipants = [op.primary, ...op.secondaries];
|
|
1550
|
+
const blockedParticipants = mergeParticipants.flatMap((ref) => {
|
|
1551
|
+
const e = memoryByRef.get(ref);
|
|
1552
|
+
if (!e)
|
|
1553
|
+
return [];
|
|
1554
|
+
const verdict = consolidateGuardStatus(e.filePath);
|
|
1555
|
+
if (verdict === "hot" || verdict === "unparseable")
|
|
1556
|
+
return [{ ref, verdict }];
|
|
1557
|
+
return [];
|
|
1558
|
+
});
|
|
1559
|
+
if (blockedParticipants.length > 0) {
|
|
1560
|
+
const detail = blockedParticipants.map((p) => `${p.ref} (${p.verdict})`).join(", ");
|
|
1561
|
+
warnings.push(`Merge: refused for ${op.primary} — ${blockedParticipants.length} participant(s) blocked by hot/unparseable frontmatter guard: ${detail}`);
|
|
1562
|
+
emitMergeFailureSkips("merge_participant_blocked");
|
|
1563
|
+
return;
|
|
1564
|
+
}
|
|
1565
|
+
// WS-3b: Anti-collapse generation guard (step 8a).
|
|
1566
|
+
// DEFAULT ON since R5 (opt out via antiCollapse.enabled: false). Refuses
|
|
1567
|
+
// to merge two assets both above generation N (default 2) — prevents the
|
|
1568
|
+
// pipeline from building ever-deeper LLM-merged trees that lose the
|
|
1569
|
+
// source fidelity of the original episodes.
|
|
1570
|
+
const antiCollapseConfig = getImproveProcessConfig(config, "consolidate", ctx.improveProfile)?.antiCollapse ?? {};
|
|
1571
|
+
if (antiCollapseConfig.enabled !== false) {
|
|
1572
|
+
const allParticipants = [op.primary, ...op.secondaries];
|
|
1573
|
+
// One read per participant: generation counter, stripped body (for the
|
|
1574
|
+
// information floor), and existing source_refs (for the provenance union).
|
|
1575
|
+
const participantInfo = allParticipants.map((ref) => {
|
|
1576
|
+
const e = memoryByRef.get(ref);
|
|
1577
|
+
if (!e)
|
|
1578
|
+
return { ref, generation: 0, body: "", sourceRefs: [] };
|
|
1579
|
+
try {
|
|
1580
|
+
const raw = fs.readFileSync(e.filePath, "utf8");
|
|
1581
|
+
const parsed = parseFrontmatter(raw);
|
|
1582
|
+
const fm = parsed.data;
|
|
1583
|
+
const sourceRefs = Array.isArray(fm.source_refs) ? fm.source_refs.map(String) : [];
|
|
1584
|
+
return { ref, generation: readAssetGeneration(fm), body: stripFrontmatterBody(raw), sourceRefs };
|
|
1585
|
+
}
|
|
1586
|
+
catch {
|
|
1587
|
+
return { ref, generation: 0, body: "", sourceRefs: [] };
|
|
1588
|
+
}
|
|
1589
|
+
});
|
|
1590
|
+
const sourceGenerations = participantInfo.map((p) => p.generation);
|
|
1591
|
+
const generationCheck = checkGenerationGuard(sourceGenerations, antiCollapseConfig);
|
|
1592
|
+
if (generationCheck.refused) {
|
|
1593
|
+
warnings.push(`Merge: ${generationCheck.reason}`);
|
|
1594
|
+
emitMergeFailureSkips("merge_generation_guard");
|
|
1595
|
+
return;
|
|
1819
1596
|
}
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
const
|
|
1824
|
-
|
|
1825
|
-
|
|
1597
|
+
// WS-3b: Lexical diversity check (step 8b).
|
|
1598
|
+
// Low n-gram diversity ⇒ likely correlated-extraction artifact; raise merge threshold.
|
|
1599
|
+
if (antiCollapseConfig.lexicalDiversityCheck !== false) {
|
|
1600
|
+
const bodies = participantInfo.map((p) => p.body).filter((b) => b.length > 0);
|
|
1601
|
+
const diversityCheck = checkLexicalDiversity(bodies, antiCollapseConfig);
|
|
1602
|
+
if (diversityCheck.lowDiversity) {
|
|
1603
|
+
// Low-diversity cluster: just warn (don't refuse merge since the dedup
|
|
1604
|
+
// path handles exact twins). The warning surfaces in health telemetry.
|
|
1605
|
+
warnings.push(`Merge: cluster around ${op.primary} has low lexical diversity (${diversityCheck.diversity?.toFixed(2) ?? "?"} < 0.30) — likely correlated extraction; merge proceeds but review is recommended.`);
|
|
1826
1606
|
}
|
|
1827
|
-
|
|
1607
|
+
}
|
|
1608
|
+
// Inject generation counter into merged content frontmatter (step 8a).
|
|
1609
|
+
// merged.generation = max(sourceGenerations) + 1. source_refs is the
|
|
1610
|
+
// UNION of participants + everything they already cited (R5 §4.2 —
|
|
1611
|
+
// the old set-if-absent behavior dropped second-generation provenance).
|
|
1612
|
+
const provenanceUnion = [...new Set([...allParticipants, ...participantInfo.flatMap((p) => p.sourceRefs)])];
|
|
1613
|
+
mergedContent = injectGenerationFrontmatter(mergedContent, sourceGenerations, provenanceUnion);
|
|
1614
|
+
// R5 §4.2: merge-information floor — ADVISORY in v1. A merge that
|
|
1615
|
+
// shrinks provenance or genericizes below the retention floor is
|
|
1616
|
+
// counted + warned, never refused (promotion path: design doc §7).
|
|
1617
|
+
try {
|
|
1618
|
+
const mergedParsed = parseFrontmatter(mergedContent);
|
|
1619
|
+
const mergedFm = mergedParsed.data;
|
|
1620
|
+
const mergedSourceRefs = Array.isArray(mergedFm.source_refs) ? mergedFm.source_refs.map(String) : [];
|
|
1621
|
+
const floorCheck = checkMergeInformationFloor(mergedParsed.content, mergedSourceRefs, participantInfo, antiCollapseConfig);
|
|
1622
|
+
if (!floorCheck.passed) {
|
|
1623
|
+
counts.mergeFloorViolations++;
|
|
1624
|
+
warnings.push(`Merge: information floor advisory for ${op.primary}: ${floorCheck.reason ?? "unspecified"} — merge proceeds (v1 observe-only).`);
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
catch {
|
|
1628
|
+
// Floor measurement is best-effort; never blocks the merge path.
|
|
1828
1629
|
}
|
|
1829
1630
|
}
|
|
1830
|
-
//
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
// Beyond that it's more likely a body section divider, not a frontmatter start.
|
|
1836
|
-
if (!body.startsWith("---")) {
|
|
1837
|
-
const nlIdx = body.indexOf("\n---");
|
|
1838
|
-
if (nlIdx >= 0 && nlIdx < 300) {
|
|
1839
|
-
body = body.slice(nlIdx + 1);
|
|
1631
|
+
// Backup secondaries before deleting
|
|
1632
|
+
for (const secRef of op.secondaries) {
|
|
1633
|
+
const secEntry = memoryByRef.get(secRef);
|
|
1634
|
+
if (secEntry && fs.existsSync(secEntry.filePath)) {
|
|
1635
|
+
backupFile(secEntry.filePath, backupDir, secEntry.name);
|
|
1840
1636
|
}
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
}
|
|
1844
|
-
}
|
|
1845
|
-
// Extract frontmatter block.
|
|
1846
|
-
// Recovery path: LLM sometimes omits the closing `---` delimiter. Detect this
|
|
1847
|
-
// by scanning lines after the opening `---` for the first blank line or the
|
|
1848
|
-
// first line that isn't a YAML key-value pair, then inject `---` there.
|
|
1849
|
-
let match = body.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
|
|
1850
|
-
if (!match) {
|
|
1851
|
-
const recovered = recoverMalformedFrontmatter(body);
|
|
1852
|
-
if (recovered) {
|
|
1853
|
-
match = recovered.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
|
|
1854
|
-
}
|
|
1855
|
-
if (!match) {
|
|
1856
|
-
return { ok: false, reason: "MALFORMED_FRONTMATTER_BLOCK" };
|
|
1857
|
-
}
|
|
1858
|
-
}
|
|
1859
|
-
// Re-parse via the yaml library so any quote-escaping mistakes either get
|
|
1860
|
-
// normalised or surface as a parse error we can reject.
|
|
1861
|
-
// Recovery: if the strict yaml library fails, fall back to the lenient
|
|
1862
|
-
// hand-rolled parseFrontmatter parser, which tolerates common LLM YAML
|
|
1863
|
-
// quirks (unescaped special chars, bare scalars, etc.). If it recovers
|
|
1864
|
-
// at least one key, proceed — serializeFrontmatter below will re-serialize
|
|
1865
|
-
// cleanly. Only reject if both parsers fail to extract any data.
|
|
1866
|
-
let parsedFm;
|
|
1637
|
+
}
|
|
1638
|
+
// Write merged primary
|
|
1867
1639
|
try {
|
|
1868
|
-
|
|
1640
|
+
const parsedPrimary = parseAssetRef(op.primary);
|
|
1641
|
+
await writeAssetToSource(target.source, target.config, parsedPrimary, mergedContent);
|
|
1869
1642
|
}
|
|
1870
1643
|
catch (e) {
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1644
|
+
warnings.push(`Merge: write failed for ${op.primary}: ${String(e)}`);
|
|
1645
|
+
emitMergeFailureSkips("merge_write_failed");
|
|
1646
|
+
return;
|
|
1647
|
+
}
|
|
1648
|
+
// Archive and delete secondaries (P1-B: soft-invalidation)
|
|
1649
|
+
for (const secRef of op.secondaries) {
|
|
1650
|
+
const secEntry = memoryByRef.get(secRef);
|
|
1651
|
+
if (!secEntry)
|
|
1652
|
+
continue;
|
|
1653
|
+
if (fs.existsSync(secEntry.filePath)) {
|
|
1654
|
+
archiveMemory(secEntry.filePath, stashDir, secRef, "merged into primary", opIndex, op.primary, warnings);
|
|
1874
1655
|
}
|
|
1875
|
-
|
|
1876
|
-
|
|
1656
|
+
try {
|
|
1657
|
+
const parsedSec = parseAssetRef(secRef);
|
|
1658
|
+
await deleteAssetFromSource(target.source, target.config, parsedSec);
|
|
1659
|
+
markJournalCompleted(stashDir, secRef);
|
|
1877
1660
|
}
|
|
1661
|
+
catch (e) {
|
|
1662
|
+
warnings.push(`Merge: delete failed for ${secRef}: ${String(e)}`);
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
markJournalCompleted(stashDir, op.primary);
|
|
1666
|
+
counts.merged++;
|
|
1667
|
+
// 2026-05-26 accounting-leak fix: `merged` is op-level, but each
|
|
1668
|
+
// successful merge actions `1 + secondaries.length` memories. Without
|
|
1669
|
+
// this counter the accounting invariant breaks by `secondaries.length`
|
|
1670
|
+
// per successful merge (chunk loop excluded all secondaries from
|
|
1671
|
+
// judgedNoAction via targetRefs, but only the primary is credited to
|
|
1672
|
+
// `merged`). Count only loaded-memory secondaries; phantom secondary
|
|
1673
|
+
// refs never affected any chunk's targetRefs in the first place.
|
|
1674
|
+
for (const secRef of op.secondaries) {
|
|
1675
|
+
if (memoryByRef.has(secRef))
|
|
1676
|
+
counts.mergedSecondaries++;
|
|
1677
|
+
}
|
|
1678
|
+
// Prune consumed refs from memoryByRef so later ops in this run cannot
|
|
1679
|
+
// reference an absorbed secondary as a merge primary and proceed with a
|
|
1680
|
+
// stale entry. Primary is rewritten (not deleted), so we only remove
|
|
1681
|
+
// secondaries; the primary ref remains valid under its new content.
|
|
1682
|
+
for (const secRef of op.secondaries) {
|
|
1683
|
+
memoryByRef.delete(secRef);
|
|
1878
1684
|
}
|
|
1879
|
-
|
|
1880
|
-
|
|
1685
|
+
}
|
|
1686
|
+
/** Execute one `delete` op (behavior-identical to the former inlined branch). */
|
|
1687
|
+
export async function handleDeleteOp(op, opIndex, ctx) {
|
|
1688
|
+
const { stashDir, target, backupDir, memoryByRef, warnings, pushSkipReason, counts } = ctx;
|
|
1689
|
+
const entry = memoryByRef.get(op.ref);
|
|
1690
|
+
if (!entry) {
|
|
1691
|
+
warnings.push(`Delete: ${op.ref} not found in loaded memories — skipping.`);
|
|
1692
|
+
// Phantom ref: not in the batch so not in processed. Pushing to
|
|
1693
|
+
// skipReasons would inflate Σ(skipReasons) without a matching processed
|
|
1694
|
+
// entry, breaking the accounting invariant. Visibility is preserved via
|
|
1695
|
+
// the warnings array above.
|
|
1696
|
+
return;
|
|
1697
|
+
}
|
|
1698
|
+
// captureMode:hot guard — refuse to delete user-captured memories OR
|
|
1699
|
+
// memories whose frontmatter is unparseable (could have hidden the hot
|
|
1700
|
+
// flag). The consolidate LLM was deleting hot-captured user memos as
|
|
1701
|
+
// "redundant" — 14 such deletes were silently archived between
|
|
1702
|
+
// 2026-05-19 and 2026-05-20 before this guard. Hot memories are
|
|
1703
|
+
// user-explicit and may only be deleted by the user.
|
|
1704
|
+
const guard = consolidateGuardStatus(entry.filePath);
|
|
1705
|
+
if (guard === "hot" || guard === "unparseable") {
|
|
1706
|
+
warnings.push(`Delete: refused for ${op.ref} — ${guard === "hot" ? "captureMode:hot (user-explicit; never auto-delete)" : "frontmatter unparseable (cannot verify hot flag absent)"}. Reason from LLM: "${op.reason ?? "n/a"}"`);
|
|
1707
|
+
pushSkipReason("delete", op.ref, "captureMode_hot_refused");
|
|
1708
|
+
return;
|
|
1709
|
+
}
|
|
1710
|
+
if (fs.existsSync(entry.filePath)) {
|
|
1711
|
+
backupFile(entry.filePath, backupDir, entry.name);
|
|
1712
|
+
// P1-B: soft-invalidation archive before hard delete
|
|
1713
|
+
archiveMemory(entry.filePath, stashDir, op.ref, op.reason, opIndex, undefined, warnings);
|
|
1881
1714
|
}
|
|
1882
|
-
const fm = parsedFm;
|
|
1883
|
-
// Normalise placeholder leaks like `updated: today`, `updated: {today: null}`,
|
|
1884
|
-
// `updated: now`, etc. The consolidate prompt instructs the LLM not to emit
|
|
1885
|
-
// these, but small models still do. Replace any such leak with today's ISO
|
|
1886
|
-
// date OR drop the field if we can't safely normalise it.
|
|
1887
|
-
normalizeUpdatedField(fm);
|
|
1888
|
-
// Re-serialise via yaml.stringify to fix any quoting quirks.
|
|
1889
|
-
let serialized;
|
|
1890
1715
|
try {
|
|
1891
|
-
|
|
1716
|
+
const parsedRef = parseAssetRef(op.ref);
|
|
1717
|
+
await deleteAssetFromSource(target.source, target.config, parsedRef);
|
|
1718
|
+
markJournalCompleted(stashDir, op.ref);
|
|
1719
|
+
counts.deleted++;
|
|
1720
|
+
// Prune from memoryByRef so later ops in this run cannot reference a
|
|
1721
|
+
// deleted memory as a merge primary or secondary.
|
|
1722
|
+
memoryByRef.delete(op.ref);
|
|
1892
1723
|
}
|
|
1893
1724
|
catch (e) {
|
|
1894
|
-
|
|
1725
|
+
// Distinguish "file already absent" from genuine failures. A prior run
|
|
1726
|
+
// may have deleted the file but the DB was not yet re-indexed, so the
|
|
1727
|
+
// ref still appeared in memoryByRef. The delete goal is already met.
|
|
1728
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
1729
|
+
if (msg.includes("not found in source")) {
|
|
1730
|
+
warnings.push(`Delete: ${op.ref} — file already absent (stale DB entry); skipping.`);
|
|
1731
|
+
pushSkipReason("delete", op.ref, "delete_already_gone");
|
|
1732
|
+
}
|
|
1733
|
+
else {
|
|
1734
|
+
warnings.push(`Delete: failed for ${op.ref}: ${String(e)}`);
|
|
1735
|
+
pushSkipReason("delete", op.ref, "delete_failed");
|
|
1736
|
+
}
|
|
1895
1737
|
}
|
|
1896
|
-
const cleaned = assembleAssetFromString(serialized, match[2]);
|
|
1897
|
-
return { ok: true, result: { content: cleaned, frontmatter: fm } };
|
|
1898
1738
|
}
|
|
1899
|
-
/**
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
* ISO yyyy-mm-dd form.
|
|
1908
|
-
* - A placeholder string ("today", "now", "{today}", "${today}", template
|
|
1909
|
-
* variables) is replaced with today's ISO date.
|
|
1910
|
-
* - A map/object (e.g. `{today: null}`) is replaced with today's ISO date.
|
|
1911
|
-
* - `null`, empty string, missing → left alone (no field added; reviewers
|
|
1912
|
-
* should not silently gain metadata they didn't write).
|
|
1913
|
-
*
|
|
1914
|
-
* Exported for unit testing.
|
|
1915
|
-
*/
|
|
1916
|
-
export function normalizeUpdatedField(fm) {
|
|
1917
|
-
if (!("updated" in fm))
|
|
1739
|
+
/** Execute one `promote` op (behavior-identical to the former inlined branch). */
|
|
1740
|
+
export async function handlePromoteOp(op, ctx) {
|
|
1741
|
+
const { config, stashDir, sourceRun, target, memoryByRef, warnings, pushSkipReason, promoted, promotedSourceRefs } = ctx;
|
|
1742
|
+
const entry = memoryByRef.get(op.ref);
|
|
1743
|
+
if (!entry) {
|
|
1744
|
+
warnings.push(`Promote: ${op.ref} not found in loaded memories — skipping.`);
|
|
1745
|
+
// Phantom ref: not in processed, so no skipReason (same rationale as
|
|
1746
|
+
// delete_ref_missing above).
|
|
1918
1747
|
return;
|
|
1919
|
-
|
|
1920
|
-
|
|
1748
|
+
}
|
|
1749
|
+
// Within-run source-ref dedup: skip if this source memory was already
|
|
1750
|
+
// promoted earlier in this run (safety belt — mergePlans already
|
|
1751
|
+
// deduplicates promote ops by source ref via Map, but this guard also
|
|
1752
|
+
// catches any future code paths that bypass mergePlans).
|
|
1753
|
+
if (promotedSourceRefs.has(op.ref)) {
|
|
1754
|
+
warnings.push(`Skipping promote: ${op.ref} already promoted in this run`);
|
|
1755
|
+
pushSkipReason("promote", op.ref, "promote_already_promoted_this_run");
|
|
1756
|
+
return;
|
|
1757
|
+
}
|
|
1758
|
+
let knowledgeRef = op.knowledgeRef;
|
|
1759
|
+
try {
|
|
1760
|
+
parseAssetRef(knowledgeRef);
|
|
1761
|
+
}
|
|
1762
|
+
catch {
|
|
1763
|
+
const slug = op.knowledgeRef
|
|
1764
|
+
.replace(/^knowledge:/, "")
|
|
1765
|
+
.replace(/[^a-z0-9-]/gi, "-")
|
|
1766
|
+
.toLowerCase();
|
|
1767
|
+
knowledgeRef = `knowledge:${slug}`;
|
|
1768
|
+
warnings.push(`Normalized invalid ref "${op.knowledgeRef}" → "${knowledgeRef}"`);
|
|
1769
|
+
}
|
|
1770
|
+
// Idempotency: check pending proposals by target ref
|
|
1771
|
+
const existingProposals = listProposals(stashDir, { ref: knowledgeRef });
|
|
1772
|
+
if (existingProposals.some((p) => p.status === "pending")) {
|
|
1773
|
+
warnings.push(`Skipping promote: pending proposal already exists for ${knowledgeRef}`);
|
|
1774
|
+
pushSkipReason("promote", op.ref, "promote_pending_proposal_exists");
|
|
1921
1775
|
return;
|
|
1922
|
-
|
|
1923
|
-
if
|
|
1924
|
-
|
|
1776
|
+
}
|
|
1777
|
+
// Idempotency: check if knowledge asset already exists
|
|
1778
|
+
const parsedKnowledgeRef = parseAssetRef(knowledgeRef);
|
|
1779
|
+
const destPath = path.join(target.source.path, "knowledge", `${parsedKnowledgeRef.name}.md`);
|
|
1780
|
+
if (fs.existsSync(destPath)) {
|
|
1781
|
+
warnings.push(`Skipping promote: ${knowledgeRef} already exists in source`);
|
|
1782
|
+
pushSkipReason("promote", op.ref, "promote_already_exists");
|
|
1783
|
+
return;
|
|
1784
|
+
}
|
|
1785
|
+
let memoryContent = "";
|
|
1786
|
+
try {
|
|
1787
|
+
memoryContent = fs.readFileSync(entry.filePath, "utf8");
|
|
1788
|
+
}
|
|
1789
|
+
catch (e) {
|
|
1790
|
+
warnings.push(`Promote: could not read ${op.ref}: ${String(e)}`);
|
|
1791
|
+
pushSkipReason("promote", op.ref, "promote_read_failed");
|
|
1792
|
+
return;
|
|
1793
|
+
}
|
|
1794
|
+
// Defensive sanitization: legacy memory files written by older
|
|
1795
|
+
// consolidate runs may still carry outer code fences or broken YAML.
|
|
1796
|
+
// Strip them here so we never propose a polluted asset.
|
|
1797
|
+
const promoteSanitized = sanitizeMergedContent(memoryContent);
|
|
1798
|
+
if (!promoteSanitized.ok) {
|
|
1799
|
+
warnings.push(`Promote: rejected ${op.ref} — source memory failed sanitization (${promoteSanitized.reason}).`);
|
|
1800
|
+
pushSkipReason("promote", op.ref, "promote_sanitization_failed");
|
|
1801
|
+
return;
|
|
1802
|
+
}
|
|
1803
|
+
memoryContent = promoteSanitized.result.content;
|
|
1804
|
+
// SOURCE_SUPERSEDED guard: refuse to promote a memory whose source
|
|
1805
|
+
// frontmatter carries `status: superseded`. Predicate at module top
|
|
1806
|
+
// (`hasSupersededStatus`) so tests can exercise it directly.
|
|
1807
|
+
if (hasSupersededStatus(promoteSanitized.result.frontmatter)) {
|
|
1808
|
+
warnings.push(`Promote: refused for ${op.ref} → ${knowledgeRef} — source memory has status:superseded; superseded memories are not promotable knowledge.`);
|
|
1809
|
+
pushSkipReason("promote", op.ref, "promote_superseded");
|
|
1810
|
+
return;
|
|
1811
|
+
}
|
|
1812
|
+
// Parse the source memory up-front so the body/frontmatter checks below
|
|
1813
|
+
// share the same parsed view.
|
|
1814
|
+
const parsedMemory = parseFrontmatter(memoryContent);
|
|
1815
|
+
// Reject sources whose body is too small to make useful knowledge.
|
|
1816
|
+
// Observed failure: memory files whose body is literally a tags string
|
|
1817
|
+
// ("discord,notification,send-notification") get promoted to knowledge
|
|
1818
|
+
// proposals that no reviewer would accept. Threshold is conservative —
|
|
1819
|
+
// 100 chars catches single-line tag dumps without rejecting genuinely
|
|
1820
|
+
// terse but valid notes.
|
|
1821
|
+
const PROMOTE_BODY_MIN_CHARS = 100;
|
|
1822
|
+
const sourceBody = parsedMemory.content.trim();
|
|
1823
|
+
if (sourceBody.length < PROMOTE_BODY_MIN_CHARS) {
|
|
1824
|
+
warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — source memory body is too small (${sourceBody.length} chars; need ≥${PROMOTE_BODY_MIN_CHARS}) to make useful knowledge.`);
|
|
1825
|
+
pushSkipReason("promote", op.ref, "promote_source_too_small");
|
|
1826
|
+
return;
|
|
1827
|
+
}
|
|
1828
|
+
// Cross-run + within-run content dedup: if an identical body already
|
|
1829
|
+
// exists in ANY pending consolidate proposal (regardless of target ref),
|
|
1830
|
+
// skip. This prevents duplicate proposals when:
|
|
1831
|
+
// (a) Multiple source memories have identical bodies but differ only
|
|
1832
|
+
// in noise frontmatter (`inferenceProcessed: true` twin alongside
|
|
1833
|
+
// the original; differing `updated:` timestamps; etc.) — the body
|
|
1834
|
+
// is the load-bearing content, so dedup must hash on body only.
|
|
1835
|
+
// (b) A prior run created a proposal for the same body under a
|
|
1836
|
+
// different knowledgeRef slug.
|
|
1837
|
+
// Use cacheHash (case-preserving stripped body) to match the canonical
|
|
1838
|
+
// hash domain used by the body-embedding cache and pending-proposal set.
|
|
1839
|
+
const bodyHash = cacheHash(sourceBody);
|
|
1840
|
+
const allPendingConsolidateProposals = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
|
|
1841
|
+
const contentDupProposal = allPendingConsolidateProposals.find((p) => {
|
|
1842
|
+
return cacheHash(p.payload.content) === bodyHash;
|
|
1843
|
+
});
|
|
1844
|
+
if (contentDupProposal) {
|
|
1845
|
+
warnings.push(`Skipping promote: identical body already pending as proposal ${contentDupProposal.id} (ref: ${contentDupProposal.ref}); skipping duplicate for ${op.ref} → ${knowledgeRef}`);
|
|
1846
|
+
pushSkipReason("promote", op.ref, "dedup_pending_proposal");
|
|
1925
1847
|
return;
|
|
1926
1848
|
}
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1849
|
+
try {
|
|
1850
|
+
// Use LLM-provided description; fall back to memory's own description
|
|
1851
|
+
// (post-sanitization frontmatter is authoritative).
|
|
1852
|
+
const description = (typeof op.description === "string" && op.description.trim()
|
|
1853
|
+
? op.description.trim()
|
|
1854
|
+
: parsedMemory.data?.description?.trim()) ?? "";
|
|
1855
|
+
// Validate the resolved frontmatter before emitting a proposal.
|
|
1856
|
+
// Required field: non-empty description. Reject obvious truncation
|
|
1857
|
+
// markers (description ends with `,`/`;`/`:`/`...`/hanging connector)
|
|
1858
|
+
// so the queue never sees half-formed metadata that the reviewer
|
|
1859
|
+
// would only reject.
|
|
1860
|
+
const fmCheck = validateProposalFrontmatter({ description });
|
|
1861
|
+
if (!fmCheck.ok) {
|
|
1862
|
+
warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — ${fmCheck.reason}.`);
|
|
1863
|
+
pushSkipReason("promote", op.ref, "promote_invalid_frontmatter");
|
|
1864
|
+
return;
|
|
1865
|
+
}
|
|
1866
|
+
// Merge `description` INTO the body's YAML frontmatter so it lands in
|
|
1867
|
+
// the on-disk asset when the proposal is accepted. The descriptionQuality
|
|
1868
|
+
// validator parses `payload.content` body (not the envelope
|
|
1869
|
+
// `payload.frontmatter`), and a memory's native frontmatter has
|
|
1870
|
+
// `captureMode`/`beliefState`/etc. but never `description` — without
|
|
1871
|
+
// this merge, 60+ pending proposals were blocked at accept-time with
|
|
1872
|
+
// MISSING_FRONTMATTER_DESCRIPTION even though the envelope had it.
|
|
1873
|
+
// (The body-frontmatter assumption baked into the 2026-05-20 comment
|
|
1874
|
+
// below was wrong: body fm and envelope fm only converge when the
|
|
1875
|
+
// writer explicitly merges them, which it now does.)
|
|
1876
|
+
const mergedBodyFm = {
|
|
1877
|
+
...(parsedMemory.data ?? {}),
|
|
1878
|
+
description,
|
|
1879
|
+
};
|
|
1880
|
+
const serializedMergedFm = serializeFrontmatter(mergedBodyFm);
|
|
1881
|
+
const proposalContent = assembleAssetFromString(serializedMergedFm, parsedMemory.content);
|
|
1882
|
+
// Pre-emit dedup against pending consolidate proposals from the
|
|
1883
|
+
// same improve run (slug-variant match). The cross-run content-hash
|
|
1884
|
+
// dedup inside `mergePlans` handles duplicates against existing
|
|
1885
|
+
// stash assets — see commit history for the deletion of the
|
|
1886
|
+
// unbounded embedding + cross-type slug branches.
|
|
1887
|
+
const dedup = await checkPreEmitDedup({
|
|
1888
|
+
candidateRef: knowledgeRef,
|
|
1889
|
+
candidateText: `${description}. ${memoryContent}`,
|
|
1890
|
+
stashDir,
|
|
1891
|
+
config,
|
|
1892
|
+
});
|
|
1893
|
+
if (dedup.duplicate) {
|
|
1894
|
+
warnings.push(`Promote: skipped ${op.ref} → ${knowledgeRef} — ${dedup.reason}.`);
|
|
1895
|
+
pushSkipReason("promote", op.ref, "promote_dedup_window");
|
|
1939
1896
|
return;
|
|
1940
1897
|
}
|
|
1941
|
-
|
|
1898
|
+
const proposalResult = createProposal(stashDir, {
|
|
1899
|
+
ref: knowledgeRef,
|
|
1900
|
+
source: "consolidate",
|
|
1901
|
+
sourceRun,
|
|
1902
|
+
payload: {
|
|
1903
|
+
content: proposalContent,
|
|
1904
|
+
frontmatter: { description },
|
|
1905
|
+
},
|
|
1906
|
+
...(typeof op.confidence === "number" ? { confidence: op.confidence } : {}),
|
|
1907
|
+
});
|
|
1908
|
+
if (isProposalSkipped(proposalResult)) {
|
|
1909
|
+
warnings.push(`Promote: skipped proposal for ${op.ref} (${proposalResult.reason}): ${proposalResult.message}`);
|
|
1910
|
+
pushSkipReason("promote", op.ref, `promote_proposal_${proposalResult.reason}`);
|
|
1911
|
+
}
|
|
1912
|
+
else {
|
|
1913
|
+
promoted.push(proposalResult.id);
|
|
1914
|
+
promotedSourceRefs.add(op.ref);
|
|
1915
|
+
markJournalCompleted(stashDir, op.ref);
|
|
1916
|
+
}
|
|
1917
|
+
}
|
|
1918
|
+
catch (e) {
|
|
1919
|
+
warnings.push(`Promote: createProposal failed for ${op.ref}: ${String(e)}`);
|
|
1920
|
+
pushSkipReason("promote", op.ref, "promote_create_failed");
|
|
1921
|
+
}
|
|
1922
|
+
}
|
|
1923
|
+
/** Execute one `contradict` op (behavior-identical to the former inlined branch). */
|
|
1924
|
+
export async function handleContradictOp(op, ctx) {
|
|
1925
|
+
const { stashDir, memoryByRef, warnings, pushSkipReason, counts } = ctx;
|
|
1926
|
+
// Confidence gate: surface-level topic overlap causes false positives
|
|
1927
|
+
// (investigation 2026-06-18). Require ≥0.92 confidence before writing
|
|
1928
|
+
// contradiction edges. Missing confidence field defaults to 1.0 for
|
|
1929
|
+
// backward compatibility with responses that predate this field.
|
|
1930
|
+
const opConfidence = typeof op.confidence === "number" ? op.confidence : 1.0;
|
|
1931
|
+
if (opConfidence < 0.92) {
|
|
1932
|
+
warnings.push(`Contradict: confidence ${opConfidence.toFixed(2)} below 0.92 threshold for ${op.ref} <-> ${op.contradictedByRef} — skipping.`);
|
|
1933
|
+
pushSkipReason("contradict", op.ref, "contradict_low_confidence");
|
|
1934
|
+
return;
|
|
1935
|
+
}
|
|
1936
|
+
// C-3 / #382: Write contradictedBy edges so resolveFamilyContradictions
|
|
1937
|
+
// (the SCC resolver in memory-improve.ts) has edges to work on.
|
|
1938
|
+
// Zep arXiv:2501.13956 §3 — unified belief-revision with contradiction edges.
|
|
1939
|
+
const entry = memoryByRef.get(op.ref);
|
|
1940
|
+
const contradictorEntry = memoryByRef.get(op.contradictedByRef);
|
|
1941
|
+
if (!entry) {
|
|
1942
|
+
warnings.push(`Contradict: ${op.ref} not found in loaded memories — skipping.`);
|
|
1943
|
+
// Phantom ref: not in processed, so no skipReason (same rationale as
|
|
1944
|
+
// delete_ref_missing).
|
|
1942
1945
|
return;
|
|
1943
1946
|
}
|
|
1944
|
-
if (
|
|
1945
|
-
|
|
1946
|
-
|
|
1947
|
+
if (!contradictorEntry) {
|
|
1948
|
+
warnings.push(`Contradict: ${op.contradictedByRef} not found — skipping.`);
|
|
1949
|
+
// op.ref IS in the batch (entry found above) so the skipReason is
|
|
1950
|
+
// correctly charged against a real processed memory.
|
|
1951
|
+
pushSkipReason("contradict", op.ref, "contradict_target_missing");
|
|
1947
1952
|
return;
|
|
1948
1953
|
}
|
|
1954
|
+
try {
|
|
1955
|
+
// Write the contradiction edge: op.ref is contradicted by op.contradictedByRef
|
|
1956
|
+
writeContradictEdge(entry.filePath, op.contradictedByRef);
|
|
1957
|
+
counts.contradicted++;
|
|
1958
|
+
markJournalCompleted(stashDir, op.ref);
|
|
1959
|
+
}
|
|
1960
|
+
catch (e) {
|
|
1961
|
+
warnings.push(`Contradict: failed to write edge for ${op.ref}: ${String(e)}`);
|
|
1962
|
+
pushSkipReason("contradict", op.ref, "contradict_write_failed");
|
|
1963
|
+
}
|
|
1949
1964
|
}
|
|
1965
|
+
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
1950
1966
|
/**
|
|
1951
1967
|
* Normalise a knowledge slug for variant-aware deduplication. Collapses:
|
|
1952
1968
|
* - date suffixes (`-may-2026`, `-2026-05-03`, `-2026`)
|
|
@@ -2019,11 +2035,13 @@ async function checkPreEmitDedup(opts) {
|
|
|
2019
2035
|
* doesn't match the pattern (assumed to already be an ISO timestamp).
|
|
2020
2036
|
*/
|
|
2021
2037
|
function parseSinceToIso(since) {
|
|
2022
|
-
|
|
2023
|
-
|
|
2038
|
+
// Canonical CLI unit grammar: `m` = minutes, `M` = months (see core/time.ts
|
|
2039
|
+
// DURATION_UNITS). Non-matching input is returned unchanged (assumed to
|
|
2040
|
+
// already be an ISO timestamp).
|
|
2041
|
+
const ms = parseDuration(since, DURATION_UNITS);
|
|
2042
|
+
if (ms === null)
|
|
2024
2043
|
return since;
|
|
2025
|
-
|
|
2026
|
-
return new Date(Date.now() - parseInt(m[1], 10) * multiplier).toISOString();
|
|
2044
|
+
return new Date(Date.now() - ms).toISOString();
|
|
2027
2045
|
}
|
|
2028
2046
|
export function narrowToIncrementalCandidates(memories, since, warnings, neighborsPerChanged = 5) {
|
|
2029
2047
|
const sinceIso = parseSinceToIso(since);
|
|
@@ -2129,7 +2147,7 @@ function loadMemoriesForSource(source, stashDir, warnings) {
|
|
|
2129
2147
|
}
|
|
2130
2148
|
return memories;
|
|
2131
2149
|
}
|
|
2132
|
-
async function generateMergedContent(config, primaryRef, primaryBody, secondaryRefs, memoryByRef) {
|
|
2150
|
+
async function generateMergedContent(config, primaryRef, primaryBody, secondaryRefs, memoryByRef, activeProfile) {
|
|
2133
2151
|
// Only handle single-secondary merges per design (one call per merge op)
|
|
2134
2152
|
const secRef = secondaryRefs[0];
|
|
2135
2153
|
const secEntry = memoryByRef.get(secRef);
|
|
@@ -2173,7 +2191,7 @@ async function generateMergedContent(config, primaryRef, primaryBody, secondaryR
|
|
|
2173
2191
|
.join("\n");
|
|
2174
2192
|
// Use the same per-process profile resolution as the chunk-plan call above
|
|
2175
2193
|
// so the merge generation step doesn't silently revert to the default LLM.
|
|
2176
|
-
const llmConfig = resolveConsolidateLlmConfig(config);
|
|
2194
|
+
const llmConfig = resolveConsolidateLlmConfig(config, activeProfile);
|
|
2177
2195
|
const result = await tryLlmFeature("memory_consolidation", config, async () => {
|
|
2178
2196
|
if (!llmConfig)
|
|
2179
2197
|
return { ok: false, error: "No LLM configured for consolidation" };
|
|
@@ -2251,8 +2269,8 @@ async function generateMergedContent(config, primaryRef, primaryBody, secondaryR
|
|
|
2251
2269
|
}
|
|
2252
2270
|
normalizeUpdatedField(repairedFmData);
|
|
2253
2271
|
const repairedYaml = serializeFrontmatter(repairedFmData);
|
|
2254
|
-
const bodyPart = mergedFm.content
|
|
2255
|
-
return { content:
|
|
2272
|
+
const bodyPart = typeof mergedFm.content === "string" ? mergedFm.content : "";
|
|
2273
|
+
return { content: assembleAssetFromString(repairedYaml, bodyPart) };
|
|
2256
2274
|
}
|
|
2257
2275
|
}
|
|
2258
2276
|
catch {
|