akm-cli 0.8.0-rc1 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{.github/CHANGELOG.md → CHANGELOG.md} +191 -3
- package/README.md +22 -6
- package/SECURITY.md +93 -0
- package/dist/cli/config-migrate.js +144 -0
- package/dist/cli/config-validate.js +39 -0
- package/dist/cli/confirm.js +73 -0
- package/dist/cli/parse-args.js +93 -3
- package/dist/cli/shared.js +129 -0
- package/dist/cli.js +2162 -1258
- package/dist/commands/add-cli.js +279 -0
- package/dist/commands/agent-dispatch.js +20 -12
- package/dist/commands/agent-support.js +11 -5
- package/dist/commands/completions.js +3 -0
- package/dist/commands/config-cli.js +129 -517
- package/dist/commands/consolidate.js +1533 -144
- package/dist/commands/curate.js +44 -3
- package/dist/commands/db-cli.js +23 -0
- package/dist/commands/distill-promotion-policy.js +5 -3
- package/dist/commands/distill.js +906 -100
- package/dist/commands/env.js +213 -0
- package/dist/commands/eval-cases.js +3 -0
- package/dist/commands/events.js +3 -0
- package/dist/commands/extract-cli.js +127 -0
- package/dist/commands/extract-prompt.js +204 -0
- package/dist/commands/extract.js +477 -0
- package/dist/commands/feedback-cli.js +331 -0
- package/dist/commands/graph.js +260 -5
- package/dist/commands/health.js +977 -51
- package/dist/commands/help/help-accept.md +6 -3
- package/dist/commands/help/help-improve.md +36 -8
- package/dist/commands/help/help-proposals.md +7 -4
- package/dist/commands/help/help-reject.md +5 -2
- package/dist/commands/history.js +51 -16
- package/dist/commands/improve-auto-accept.js +97 -0
- package/dist/commands/improve-cli.js +236 -0
- package/dist/commands/improve-profiles.js +184 -0
- package/dist/commands/improve-result-file.js +167 -0
- package/dist/commands/improve.js +1725 -332
- package/dist/commands/info.js +3 -0
- package/dist/commands/init.js +49 -1
- package/dist/commands/installed-stashes.js +6 -23
- package/dist/commands/knowledge.js +3 -0
- package/dist/commands/lint/agent-linter.js +3 -0
- package/dist/commands/lint/base-linter.js +233 -5
- package/dist/commands/lint/command-linter.js +3 -0
- package/dist/commands/lint/default-linter.js +3 -0
- package/dist/commands/lint/env-key-rules.js +154 -0
- package/dist/commands/lint/index.js +92 -3
- package/dist/commands/lint/knowledge-linter.js +3 -0
- package/dist/commands/lint/markdown-insertion.js +343 -0
- package/dist/commands/lint/memory-linter.js +3 -0
- package/dist/commands/lint/registry.js +3 -0
- package/dist/commands/lint/skill-linter.js +3 -0
- package/dist/commands/lint/task-linter.js +15 -12
- package/dist/commands/lint/types.js +3 -0
- package/dist/commands/lint/workflow-linter.js +3 -0
- package/dist/commands/lint.js +3 -0
- package/dist/commands/migration-help.js +5 -2
- package/dist/commands/proposal-drain-policies.js +128 -0
- package/dist/commands/proposal-drain.js +477 -0
- package/dist/commands/proposal.js +60 -6
- package/dist/commands/propose.js +24 -19
- package/dist/commands/reflect.js +1004 -94
- package/dist/commands/registry-cli.js +150 -0
- package/dist/commands/registry-search.js +3 -0
- package/dist/commands/remember-cli.js +257 -0
- package/dist/commands/remember.js +15 -6
- package/dist/commands/schema-repair.js +88 -15
- package/dist/commands/search.js +99 -14
- package/dist/commands/secret.js +173 -0
- package/dist/commands/self-update.js +3 -0
- package/dist/commands/show.js +32 -13
- package/dist/commands/source-add.js +7 -35
- package/dist/commands/source-clone.js +3 -0
- package/dist/commands/source-manage.js +3 -0
- package/dist/commands/tasks.js +161 -95
- package/dist/commands/url-checker.js +3 -0
- package/dist/core/action-contributors.js +3 -0
- package/dist/core/asset-ref.js +17 -2
- package/dist/core/asset-registry.js +9 -2
- package/dist/core/asset-serialize.js +88 -0
- package/dist/core/asset-spec.js +61 -5
- package/dist/core/common.js +93 -5
- package/dist/core/concurrent.js +3 -0
- package/dist/core/config-io.js +347 -0
- package/dist/core/config-migration.js +622 -0
- package/dist/core/config-schema.js +558 -0
- package/dist/core/config-sources.js +108 -0
- package/dist/core/config-types.js +4 -0
- package/dist/core/config-walker.js +337 -0
- package/dist/core/config.js +366 -1077
- package/dist/core/errors.js +42 -20
- package/dist/core/events.js +31 -25
- package/dist/core/file-lock.js +104 -0
- package/dist/core/frontmatter.js +75 -10
- package/dist/core/lesson-lint.js +3 -0
- package/dist/core/markdown.js +3 -0
- package/dist/core/memory-belief.js +62 -0
- package/dist/core/memory-contradiction-detect.js +274 -0
- package/dist/core/memory-improve.js +142 -14
- package/dist/core/parse.js +3 -0
- package/dist/core/paths.js +218 -50
- package/dist/core/proposal-quality-validators.js +380 -0
- package/dist/core/proposal-validators.js +11 -3
- package/dist/core/proposals.js +464 -5
- package/dist/core/state-db.js +349 -56
- package/dist/core/text-truncation.js +107 -0
- package/dist/core/time.js +3 -0
- package/dist/core/tty.js +59 -0
- package/dist/core/warn.js +7 -2
- package/dist/core/write-source.js +12 -0
- package/dist/indexer/db-backup.js +391 -0
- package/dist/indexer/db-search.js +136 -28
- package/dist/indexer/db.js +662 -166
- package/dist/indexer/ensure-index.js +3 -0
- package/dist/indexer/file-context.js +3 -0
- package/dist/indexer/graph-boost.js +162 -40
- package/dist/indexer/graph-db.js +241 -51
- package/dist/indexer/graph-dedup.js +3 -7
- package/dist/indexer/graph-extraction.js +242 -149
- package/dist/indexer/index-context.js +3 -9
- package/dist/indexer/indexer.js +84 -14
- package/dist/indexer/llm-cache.js +24 -19
- package/dist/indexer/manifest.js +3 -0
- package/dist/indexer/matchers.js +184 -11
- package/dist/indexer/memory-inference.js +94 -50
- package/dist/indexer/metadata-contributors.js +3 -0
- package/dist/indexer/metadata.js +114 -48
- package/dist/indexer/path-resolver.js +3 -0
- package/dist/indexer/project-context.js +192 -0
- package/dist/indexer/ranking-contributors.js +134 -7
- package/dist/indexer/ranking.js +8 -1
- package/dist/indexer/search-fields.js +5 -9
- package/dist/indexer/search-hit-enrichers.js +91 -2
- package/dist/indexer/search-source.js +20 -1
- package/dist/indexer/semantic-status.js +4 -1
- package/dist/indexer/staleness-detect.js +447 -0
- package/dist/indexer/usage-events.js +12 -9
- package/dist/indexer/walker.js +3 -0
- package/dist/integrations/agent/builders.js +135 -0
- package/dist/integrations/agent/config.js +121 -401
- package/dist/integrations/agent/detect.js +3 -0
- package/dist/integrations/agent/index.js +6 -14
- package/dist/integrations/agent/model-aliases.js +55 -0
- package/dist/integrations/agent/profiles.js +3 -0
- package/dist/integrations/agent/prompts.js +137 -8
- package/dist/integrations/agent/runner.js +208 -0
- package/dist/integrations/agent/sdk-runner.js +8 -2
- package/dist/integrations/agent/spawn.js +54 -14
- package/dist/integrations/github.js +3 -0
- package/dist/integrations/lockfile.js +22 -51
- package/dist/integrations/session-logs/index.js +4 -0
- package/dist/integrations/session-logs/inline-refs.js +35 -0
- package/dist/integrations/session-logs/pre-filter.js +152 -0
- package/dist/integrations/session-logs/providers/claude-code.js +226 -0
- package/dist/integrations/session-logs/providers/opencode.js +231 -25
- package/dist/integrations/session-logs/types.js +3 -0
- package/dist/llm/call-ai.js +14 -26
- package/dist/llm/client.js +16 -2
- package/dist/llm/embedder.js +20 -29
- package/dist/llm/embedders/cache.js +3 -7
- package/dist/llm/embedders/local.js +42 -1
- package/dist/llm/embedders/remote.js +20 -8
- package/dist/llm/embedders/types.js +3 -7
- package/dist/llm/feature-gate.js +92 -56
- package/dist/llm/graph-extract.js +401 -30
- package/dist/llm/index-passes.js +44 -29
- package/dist/llm/memory-infer.js +30 -2
- package/dist/llm/metadata-enhance.js +3 -7
- package/dist/llm/prompts/extract-session.md +80 -0
- package/dist/llm/prompts/graph-extract-user-prompt.md +24 -1
- package/dist/output/cli-hints-full.md +60 -32
- package/dist/output/cli-hints-short.md +10 -7
- package/dist/output/cli-hints.js +5 -2
- package/dist/output/context.js +60 -8
- package/dist/output/renderers.js +170 -194
- package/dist/output/shapes/curate.js +56 -0
- package/dist/output/shapes/distill.js +10 -0
- package/dist/output/shapes/env-list.js +19 -0
- package/dist/output/shapes/events.js +11 -0
- package/dist/output/shapes/helpers.js +424 -0
- package/dist/output/shapes/history.js +7 -0
- package/dist/output/shapes/passthrough.js +105 -0
- package/dist/output/shapes/proposal-accept.js +7 -0
- package/dist/output/shapes/proposal-diff.js +7 -0
- package/dist/output/shapes/proposal-list.js +7 -0
- package/dist/output/shapes/proposal-producer.js +11 -0
- package/dist/output/shapes/proposal-reject.js +7 -0
- package/dist/output/shapes/proposal-show.js +7 -0
- package/dist/output/shapes/registry-search.js +6 -0
- package/dist/output/shapes/registry.js +30 -0
- package/dist/output/shapes/search.js +6 -0
- package/dist/output/shapes/secret-list.js +19 -0
- package/dist/output/shapes/show.js +6 -0
- package/dist/output/shapes/vault-list.js +19 -0
- package/dist/output/shapes.js +51 -549
- package/dist/output/text/add.js +6 -0
- package/dist/output/text/clone.js +6 -0
- package/dist/output/text/config.js +6 -0
- package/dist/output/text/curate.js +6 -0
- package/dist/output/text/distill.js +7 -0
- package/dist/output/text/enable-disable.js +7 -0
- package/dist/output/text/events.js +10 -0
- package/dist/output/text/feedback.js +6 -0
- package/dist/output/text/helpers.js +1059 -0
- package/dist/output/text/history.js +7 -0
- package/dist/output/text/import.js +6 -0
- package/dist/output/text/index.js +6 -0
- package/dist/output/text/info.js +6 -0
- package/dist/output/text/init.js +6 -0
- package/dist/output/text/list.js +6 -0
- package/dist/output/text/proposal-producer.js +8 -0
- package/dist/output/text/proposal.js +12 -0
- package/dist/output/text/registry-commands.js +11 -0
- package/dist/output/text/registry.js +30 -0
- package/dist/output/text/remember.js +6 -0
- package/dist/output/text/remove.js +6 -0
- package/dist/output/text/save.js +6 -0
- package/dist/output/text/search.js +6 -0
- package/dist/output/text/show.js +6 -0
- package/dist/output/text/update.js +6 -0
- package/dist/output/text/upgrade.js +6 -0
- package/dist/output/text/vault.js +16 -0
- package/dist/output/text/wiki.js +15 -0
- package/dist/output/text/workflow.js +14 -0
- package/dist/output/text.js +44 -1329
- package/dist/registry/build-index.js +3 -0
- package/dist/registry/create-provider-registry.js +3 -0
- package/dist/registry/factory.js +4 -1
- package/dist/registry/origin-resolve.js +3 -0
- package/dist/registry/providers/index.js +3 -0
- package/dist/registry/providers/skills-sh.js +11 -2
- package/dist/registry/providers/static-index.js +10 -1
- package/dist/registry/providers/types.js +3 -24
- package/dist/registry/resolve.js +11 -16
- package/dist/registry/types.js +3 -0
- package/dist/scripts/migrate-storage.js +17767 -0
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +9031 -0
- package/dist/scripts/migrations/v16-to-v17.js +141 -0
- package/dist/setup/detect.js +3 -0
- package/dist/setup/ripgrep-install.js +3 -0
- package/dist/setup/ripgrep-resolve.js +3 -0
- package/dist/setup/setup.js +306 -67
- package/dist/setup/steps.js +3 -15
- package/dist/sources/include.js +3 -0
- package/dist/sources/provider-factory.js +3 -11
- package/dist/sources/provider.js +3 -20
- package/dist/sources/providers/filesystem.js +19 -23
- package/dist/sources/providers/git.js +171 -21
- package/dist/sources/providers/index.js +3 -0
- package/dist/sources/providers/install-types.js +3 -13
- package/dist/sources/providers/npm.js +3 -4
- package/dist/sources/providers/provider-utils.js +3 -0
- package/dist/sources/providers/sync-from-ref.js +3 -11
- package/dist/sources/providers/tar-utils.js +3 -0
- package/dist/sources/providers/website.js +18 -22
- package/dist/sources/resolve.js +3 -0
- package/dist/sources/types.js +3 -0
- package/dist/sources/website-ingest.js +3 -0
- package/dist/tasks/backends/cron.js +3 -0
- package/dist/tasks/backends/exec-utils.js +3 -0
- package/dist/tasks/backends/index.js +3 -11
- package/dist/tasks/backends/launchd.js +3 -0
- package/dist/tasks/backends/schtasks.js +3 -0
- package/dist/tasks/parser.js +51 -38
- package/dist/tasks/resolveAkmBin.js +3 -0
- package/dist/tasks/runner.js +35 -9
- package/dist/tasks/schedule.js +20 -1
- package/dist/tasks/schema.js +5 -3
- package/dist/tasks/validator.js +6 -3
- package/dist/version.js +3 -0
- package/dist/wiki/wiki-templates.js +3 -0
- package/dist/wiki/wiki.js +3 -0
- package/dist/workflows/authoring.js +3 -0
- package/dist/workflows/cli.js +3 -0
- package/dist/workflows/db.js +140 -10
- package/dist/workflows/document-cache.js +3 -10
- package/dist/workflows/parser.js +3 -0
- package/dist/workflows/renderer.js +3 -0
- package/dist/workflows/runs.js +18 -1
- package/dist/workflows/schema.js +3 -0
- package/dist/workflows/scope-key.js +3 -0
- package/dist/workflows/validator.js +5 -9
- package/docs/README.md +7 -2
- package/docs/data-and-telemetry.md +225 -0
- package/docs/migration/release-notes/0.7.5.md +2 -2
- package/docs/migration/release-notes/0.8.0.md +57 -5
- package/docs/migration/v0.7-to-v0.8.md +1378 -0
- package/package.json +28 -11
- package/.github/LICENSE +0 -374
- package/dist/commands/install-audit.js +0 -385
- package/dist/commands/vault.js +0 -307
- package/dist/indexer/match-contributors.js +0 -141
- package/dist/integrations/agent/pipeline.js +0 -39
- package/dist/integrations/agent/runners.js +0 -31
|
@@ -1,82 +1,196 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
1
5
|
import fs from "node:fs";
|
|
2
6
|
import path from "node:path";
|
|
3
7
|
import readline from "node:readline";
|
|
4
|
-
import { stringify as yamlStringify } from "yaml";
|
|
8
|
+
import { parse as yamlParse, stringify as yamlStringify } from "yaml";
|
|
5
9
|
import { parseAssetRef } from "../core/asset-ref";
|
|
10
|
+
import { assembleAssetFromString } from "../core/asset-serialize";
|
|
6
11
|
import { resolveStashDir, timestampForFilename } from "../core/common";
|
|
7
|
-
import { loadConfig } from "../core/config";
|
|
12
|
+
import { getDefaultLlmConfig, loadConfig } from "../core/config";
|
|
8
13
|
import { ConfigError } from "../core/errors";
|
|
14
|
+
import { appendEvent } from "../core/events";
|
|
9
15
|
import { parseFrontmatter } from "../core/frontmatter";
|
|
16
|
+
import { writeContradictEdge } from "../core/memory-belief";
|
|
10
17
|
import { parseEmbeddedJsonResponse } from "../core/parse";
|
|
11
|
-
import {
|
|
18
|
+
import { hasHotCaptureMode, hasSupersededStatus, MERGE_ABSOLUTE_FLOOR_CHARS, MERGE_SHRINK_RATIO_MIN, validateProposalFrontmatter, } from "../core/proposal-quality-validators";
|
|
19
|
+
import { createProposal, isProposalSkipped, listProposals } from "../core/proposals";
|
|
20
|
+
import { detectTruncatedDescription } from "../core/text-truncation";
|
|
21
|
+
// Re-export the moved helpers so existing test imports continue to resolve.
|
|
22
|
+
export { hasSupersededStatus, validateProposalFrontmatter };
|
|
12
23
|
import { warn } from "../core/warn";
|
|
13
24
|
import { deleteAssetFromSource, resolveWriteTarget, writeAssetToSource } from "../core/write-source";
|
|
14
|
-
import { closeDatabase, getAllEntries, openExistingDatabase } from "../indexer/db";
|
|
25
|
+
import { closeDatabase, findEntryIdByRef, getAllEntries, getEntryById, getNeighborsByEntryId, openExistingDatabase, } from "../indexer/db";
|
|
26
|
+
import { resolveImproveProcessRunnerFromProfile } from "../integrations/agent/runner";
|
|
15
27
|
import { chatCompletion } from "../llm/client";
|
|
28
|
+
import { cosineSimilarity, embedBatch } from "../llm/embedder";
|
|
16
29
|
import { isLlmFeatureEnabled, tryLlmFeature } from "../llm/feature-gate";
|
|
17
30
|
// ── Prompts ─────────────────────────────────────────────────────────────────
|
|
18
31
|
const CONSOLIDATE_SYSTEM_PROMPT = `You are the akm consolidate assistant analyzing memory assets.
|
|
19
32
|
|
|
20
33
|
Rules:
|
|
21
34
|
1. MERGE: Two or more memories are substantially duplicated or closely related → propose merging. Return the primary ref to keep and secondary refs to delete. Do NOT include mergedContent — the merge will be executed in a separate step.
|
|
22
|
-
2. DELETE: Memory is clearly outdated, contradicted, or redundant → propose deletion.
|
|
23
|
-
3. PROMOTE: Memory expresses a stable, reusable fact suitable as a \`knowledge:\` asset → propose promotion. Do NOT delete the source memory.
|
|
24
|
-
4.
|
|
35
|
+
2. DELETE: Memory is clearly outdated, contradicted, or redundant → propose deletion. NEVER propose delete for memories annotated \`(captureMode: hot)\` — they are user-explicit and only the user can retire them. The downstream guard will refuse these regardless, so proposing them just wastes tokens.
|
|
36
|
+
3. PROMOTE: Memory expresses a stable, reusable fact suitable as a \`knowledge:\` asset → propose promotion. Do NOT delete the source memory. NEVER propose promote / merge / contradict for memories annotated \`(already queued)\` — they have a pending proposal whose body matches; a duplicate will be deterministically dropped, so proposing them just wastes tokens.
|
|
37
|
+
4. CONTRADICT: Two memories make mutually exclusive factual claims about the same subject (e.g. "always use VPN" vs "VPN is optional") → mark the older or less authoritative one as contradicted. This writes a contradictedBy edge so the belief-resolution SCC algorithm can resolve the conflict. Do NOT delete contradicted memories — let the belief resolver decide.
|
|
38
|
+
5. KEEP: Memory is unique and current → omit from output.
|
|
25
39
|
|
|
26
40
|
Return ONLY JSON (no prose, no code fences):
|
|
27
41
|
{
|
|
28
42
|
"operations": [
|
|
29
|
-
{ "op": "merge", "primary": "memory:<name>", "secondaries": ["memory:<name>", ...], "mergeStrategy": "synthesize" },
|
|
30
|
-
{ "op": "delete", "ref": "memory:<name>", "reason": "<brief reason>" },
|
|
31
|
-
{ "op": "promote", "ref": "memory:<name>", "knowledgeRef": "knowledge:<suggested-slug>", "reason": "<brief reason>" }
|
|
43
|
+
{ "op": "merge", "primary": "memory:<name>", "secondaries": ["memory:<name>", ...], "mergeStrategy": "synthesize", "confidence": 0.95 },
|
|
44
|
+
{ "op": "delete", "ref": "memory:<name>", "reason": "<brief reason>", "confidence": 0.90 },
|
|
45
|
+
{ "op": "promote", "ref": "memory:<name>", "knowledgeRef": "knowledge:<suggested-slug>", "reason": "<brief reason>", "description": "<one sentence describing the new knowledge asset>", "confidence": 0.92 },
|
|
46
|
+
{ "op": "contradict", "ref": "memory:<name>", "contradictedByRef": "memory:<name>", "reason": "<brief reason>", "confidence": 0.88 }
|
|
32
47
|
],
|
|
33
48
|
"warnings": ["<optional concerns>"]
|
|
34
|
-
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
For every operation, emit a \`confidence\` field in [0, 1] expressing your certainty that the operation is correct and safe. Use 0.95+ only when evidence is unambiguous. Omit the field rather than guessing if you are uncertain.
|
|
52
|
+
|
|
53
|
+
When the merged content includes an \`updated\` frontmatter field, the value MUST be a real ISO date string (e.g. \`updated: 2026-05-20\`). NEVER emit \`updated: today\`, \`updated: {today}\`, \`updated: {today: null}\`, \`updated: now\`, or any other literal placeholder/template-variable. If you do not have a real source-of-truth date, OMIT the \`updated\` field entirely — the post-processor will not invent one for you.`;
|
|
54
|
+
/**
|
|
55
|
+
* JSON Schema for structured consolidate plans (PR 1 of the asset-writers
|
|
56
|
+
* decision — see knowledge:projects/akm/asset-writers-investigation/00-synthesis).
|
|
57
|
+
* Mirrors the {ops[], warnings?[]} shape currently described in
|
|
58
|
+
* CONSOLIDATE_SYSTEM_PROMPT. Providers with `supportsJsonSchema: true` enforce
|
|
59
|
+
* the shape upstream so the chunk-level "invalid plan from AI — skipping"
|
|
60
|
+
* branch in `runConsolidate` becomes unreachable on schema-honouring providers.
|
|
61
|
+
*
|
|
62
|
+
* The four operation variants (merge / delete / promote / contradict) are
|
|
63
|
+
* modeled as a oneOf so a structured-output provider can still tell them apart
|
|
64
|
+
* by the required `op` discriminator. `parseEmbeddedJsonResponse` keeps
|
|
65
|
+
* working as a fallback parser for providers that ignore the schema.
|
|
66
|
+
*/
|
|
67
|
+
export const CONSOLIDATE_PLAN_JSON_SCHEMA = {
|
|
68
|
+
type: "object",
|
|
69
|
+
required: ["operations"],
|
|
70
|
+
additionalProperties: false,
|
|
71
|
+
properties: {
|
|
72
|
+
operations: {
|
|
73
|
+
type: "array",
|
|
74
|
+
description: "Ordered list of consolidate operations the planner proposes.",
|
|
75
|
+
items: {
|
|
76
|
+
oneOf: [
|
|
77
|
+
{
|
|
78
|
+
type: "object",
|
|
79
|
+
required: ["op", "primary", "secondaries", "mergeStrategy"],
|
|
80
|
+
additionalProperties: false,
|
|
81
|
+
properties: {
|
|
82
|
+
op: { type: "string", enum: ["merge"] },
|
|
83
|
+
primary: { type: "string", minLength: 1 },
|
|
84
|
+
secondaries: {
|
|
85
|
+
type: "array",
|
|
86
|
+
minItems: 1,
|
|
87
|
+
items: { type: "string", minLength: 1 },
|
|
88
|
+
},
|
|
89
|
+
mergeStrategy: { type: "string", minLength: 1 },
|
|
90
|
+
confidence: { type: "number", minimum: 0, maximum: 1 },
|
|
91
|
+
},
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
type: "object",
|
|
95
|
+
required: ["op", "ref", "reason"],
|
|
96
|
+
additionalProperties: false,
|
|
97
|
+
properties: {
|
|
98
|
+
op: { type: "string", enum: ["delete"] },
|
|
99
|
+
ref: { type: "string", minLength: 1 },
|
|
100
|
+
reason: { type: "string", minLength: 1 },
|
|
101
|
+
confidence: { type: "number", minimum: 0, maximum: 1 },
|
|
102
|
+
},
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
type: "object",
|
|
106
|
+
required: ["op", "ref", "knowledgeRef", "reason"],
|
|
107
|
+
additionalProperties: false,
|
|
108
|
+
properties: {
|
|
109
|
+
op: { type: "string", enum: ["promote"] },
|
|
110
|
+
ref: { type: "string", minLength: 1 },
|
|
111
|
+
knowledgeRef: { type: "string", minLength: 1 },
|
|
112
|
+
reason: { type: "string", minLength: 1 },
|
|
113
|
+
description: { type: "string" },
|
|
114
|
+
confidence: { type: "number", minimum: 0, maximum: 1 },
|
|
115
|
+
},
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
type: "object",
|
|
119
|
+
required: ["op", "ref", "contradictedByRef", "reason"],
|
|
120
|
+
additionalProperties: false,
|
|
121
|
+
properties: {
|
|
122
|
+
op: { type: "string", enum: ["contradict"] },
|
|
123
|
+
ref: { type: "string", minLength: 1 },
|
|
124
|
+
contradictedByRef: { type: "string", minLength: 1 },
|
|
125
|
+
reason: { type: "string", minLength: 1 },
|
|
126
|
+
confidence: { type: "number", minimum: 0, maximum: 1 },
|
|
127
|
+
},
|
|
128
|
+
},
|
|
129
|
+
],
|
|
130
|
+
},
|
|
131
|
+
},
|
|
132
|
+
warnings: {
|
|
133
|
+
type: "array",
|
|
134
|
+
description: "Optional list of human-readable concerns the planner wants to surface.",
|
|
135
|
+
items: { type: "string" },
|
|
136
|
+
},
|
|
137
|
+
},
|
|
138
|
+
};
|
|
35
139
|
export function isConsolidationEligibleMemoryName(name) {
|
|
36
140
|
return !name.endsWith(".derived");
|
|
37
141
|
}
|
|
38
|
-
|
|
39
|
-
|
|
142
|
+
/**
|
|
143
|
+
* Returns true when the memory file has `captureMode: hot` in its frontmatter.
|
|
144
|
+
*
|
|
145
|
+
* Hot memories are USER-EXPLICIT (written via `akm remember` on the hot path).
|
|
146
|
+
* The consolidate LLM is forbidden from deleting or auto-merging them — the
|
|
147
|
+
* user wrote them on purpose and only the user can decide to retire them.
|
|
148
|
+
*
|
|
149
|
+
* Reads the file once per check; consolidate runs against ~10 memories per
|
|
150
|
+
* chunk so the IO cost is trivial. Returns false on any read/parse error
|
|
151
|
+
* (fail-safe: an unparseable file is treated as not-hot, but the broader
|
|
152
|
+
* consolidate flow already guards against unparseable memories elsewhere).
|
|
153
|
+
*
|
|
154
|
+
* Defends against four observed defect classes (see
|
|
155
|
+
* `memory:akm-improve-critical-review-2026-05-20`):
|
|
156
|
+
* - LLM marks a memory contradicted then deletes (dangling contradictedBy)
|
|
157
|
+
* - LLM merges two unrelated memories sharing a topic keyword
|
|
158
|
+
* - LLM judges a recent durable design memo as "redundant"
|
|
159
|
+
* - Cascade deletes (LLM uses ref:X as `contradictedBy` for ref:Y then deletes both)
|
|
160
|
+
*/
|
|
161
|
+
export function isHotCapturedMemory(filePath) {
|
|
40
162
|
try {
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
return true;
|
|
47
|
-
return path.resolve(e.stashDir) === path.resolve(sourceFilterPath);
|
|
48
|
-
})
|
|
49
|
-
.filter((e) => isConsolidationEligibleMemoryName(e.entry.name))
|
|
50
|
-
.map((e) => ({
|
|
51
|
-
name: e.entry.name,
|
|
52
|
-
filePath: e.filePath,
|
|
53
|
-
description: e.entry.description ?? "",
|
|
54
|
-
tags: e.entry.tags ?? [],
|
|
55
|
-
stashDir: e.stashDir,
|
|
56
|
-
}));
|
|
163
|
+
if (!fs.existsSync(filePath))
|
|
164
|
+
return false;
|
|
165
|
+
const content = fs.readFileSync(filePath, "utf8");
|
|
166
|
+
const parsed = parseFrontmatter(content);
|
|
167
|
+
return hasHotCaptureMode(parsed.data);
|
|
57
168
|
}
|
|
58
169
|
catch {
|
|
59
|
-
return
|
|
60
|
-
}
|
|
61
|
-
finally {
|
|
62
|
-
if (db)
|
|
63
|
-
closeDatabase(db);
|
|
170
|
+
return false;
|
|
64
171
|
}
|
|
65
172
|
}
|
|
66
|
-
function
|
|
67
|
-
if (!fs.existsSync(
|
|
68
|
-
return
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
173
|
+
export function consolidateGuardStatus(filePath) {
|
|
174
|
+
if (!fs.existsSync(filePath))
|
|
175
|
+
return "missing";
|
|
176
|
+
let content;
|
|
177
|
+
try {
|
|
178
|
+
content = fs.readFileSync(filePath, "utf8");
|
|
179
|
+
}
|
|
180
|
+
catch {
|
|
181
|
+
return "unparseable";
|
|
182
|
+
}
|
|
183
|
+
let parsed;
|
|
184
|
+
try {
|
|
185
|
+
parsed = parseFrontmatter(content);
|
|
186
|
+
}
|
|
187
|
+
catch {
|
|
188
|
+
return "unparseable";
|
|
78
189
|
}
|
|
79
|
-
|
|
190
|
+
const data = parsed.data;
|
|
191
|
+
if (!data || Object.keys(data).length === 0)
|
|
192
|
+
return "unparseable";
|
|
193
|
+
return hasHotCaptureMode(data) ? "hot" : "safe";
|
|
80
194
|
}
|
|
81
195
|
// ── Chunk sizing ─────────────────────────────────────────────────────────────
|
|
82
196
|
/**
|
|
@@ -94,21 +208,21 @@ const CHARS_PER_TOKEN = 3;
|
|
|
94
208
|
*/
|
|
95
209
|
const PROMPT_OVERHEAD_TOKENS = 2_000;
|
|
96
210
|
/**
|
|
97
|
-
* Default effective token budget used when
|
|
98
|
-
* set. This is intentionally conservative (4 096)
|
|
99
|
-
* the model's actual context window, because:
|
|
211
|
+
* Default effective token budget used when the default LLM profile's
|
|
212
|
+
* `contextLength` is not set. This is intentionally conservative (4 096)
|
|
213
|
+
* rather than being set to the model's actual context window, because:
|
|
100
214
|
*
|
|
101
|
-
* - When the agent path is used
|
|
215
|
+
* - When the agent path is used, the agent CLI (e.g. opencode)
|
|
102
216
|
* prepends its own large system prompt + conversation history before
|
|
103
217
|
* forwarding to the model. That overhead easily consumes 30K+ tokens on
|
|
104
218
|
* a model with a 16K context window, leaving very little room for
|
|
105
219
|
* chunk content.
|
|
106
|
-
* - When the HTTP path is used (
|
|
107
|
-
* user prompt are sent, so the budget can be set to the
|
|
108
|
-
* context length via
|
|
220
|
+
* - When the HTTP path is used (an LLM profile is selected), only the akm
|
|
221
|
+
* system prompt and user prompt are sent, so the budget can be set to the
|
|
222
|
+
* model's actual context length via profiles.llm[defaults.llm].contextLength.
|
|
109
223
|
*
|
|
110
|
-
* Set
|
|
111
|
-
* context window to allow larger chunks on the HTTP path.
|
|
224
|
+
* Set profiles.llm[defaults.llm].contextLength in your config file to the
|
|
225
|
+
* model's actual context window to allow larger chunks on the HTTP path.
|
|
112
226
|
*/
|
|
113
227
|
export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
|
|
114
228
|
/**
|
|
@@ -125,40 +239,181 @@ export const DEFAULT_CONTEXT_LENGTH_TOKENS = 4_096;
|
|
|
125
239
|
*
|
|
126
240
|
* @param contextLength - Model context window in tokens.
|
|
127
241
|
* @param bodyTruncation - Max chars per memory body included in the prompt.
|
|
242
|
+
* @param maxChunkSize - Optional override for the hardcoded cap of 50 (1–50).
|
|
128
243
|
*/
|
|
129
|
-
export function computeSafeChunkSize(contextLength, bodyTruncation) {
|
|
244
|
+
export function computeSafeChunkSize(contextLength, bodyTruncation, maxChunkSize) {
|
|
130
245
|
const usableTokens = Math.max(contextLength - PROMPT_OVERHEAD_TOKENS, 0);
|
|
131
246
|
const tokensPerMemory = Math.max(Math.ceil(bodyTruncation / CHARS_PER_TOKEN), 1);
|
|
132
247
|
const raw = Math.floor(usableTokens / tokensPerMemory);
|
|
133
|
-
return Math.max(1, Math.min(50, raw));
|
|
248
|
+
return Math.max(1, Math.min(maxChunkSize ?? 50, raw));
|
|
249
|
+
}
|
|
250
|
+
// ── Similarity clustering (C-1 / #380) ──────────────────────────────────────
|
|
251
|
+
/**
|
|
252
|
+
* Re-order memories so that similar ones are placed adjacent to each other
|
|
253
|
+
* before the memories are sliced into chunks. This ensures high-similarity
|
|
254
|
+
* memories land in the same LLM context window, allowing the consolidate
|
|
255
|
+
* model to detect and merge duplicates that would otherwise be split across
|
|
256
|
+
* chunks and survive indefinitely.
|
|
257
|
+
*
|
|
258
|
+
* Algorithm: greedy nearest-neighbour chain starting from the first memory.
|
|
259
|
+
* Each step selects the unused memory with the highest cosine similarity to
|
|
260
|
+
* the last-placed memory. O(n²) — acceptable for the expected N < 200.
|
|
261
|
+
*
|
|
262
|
+
* mem0 arXiv:2504.19413 — every candidate compared against whole store.
|
|
263
|
+
* A-MEM arXiv:2502.12110 — atomic notes linked by similarity.
|
|
264
|
+
*
|
|
265
|
+
* Returns the original order unchanged when:
|
|
266
|
+
* - The embedding config is not present.
|
|
267
|
+
* - Embedding requests fail (fail-open).
|
|
268
|
+
* - There are fewer than 3 memories (no benefit to reordering).
|
|
269
|
+
*/
|
|
270
|
+
async function clusterMemoriesBySimilarity(memories, config) {
|
|
271
|
+
if (memories.length < 3 || !config.embedding)
|
|
272
|
+
return memories;
|
|
273
|
+
const texts = memories.map((m) => {
|
|
274
|
+
const parts = [];
|
|
275
|
+
if (m.description)
|
|
276
|
+
parts.push(m.description);
|
|
277
|
+
if (m.tags.length > 0)
|
|
278
|
+
parts.push(m.tags.join(" "));
|
|
279
|
+
return parts.join(". ") || m.name;
|
|
280
|
+
});
|
|
281
|
+
let embeddings = null;
|
|
282
|
+
try {
|
|
283
|
+
embeddings = await embedBatch(texts, config.embedding);
|
|
284
|
+
}
|
|
285
|
+
catch {
|
|
286
|
+
// Fail open: embedding failures degrade gracefully to original order.
|
|
287
|
+
return memories;
|
|
288
|
+
}
|
|
289
|
+
if (!embeddings || embeddings.length !== memories.length)
|
|
290
|
+
return memories;
|
|
291
|
+
// Greedy nearest-neighbour chain.
|
|
292
|
+
const used = new Array(memories.length).fill(false);
|
|
293
|
+
const ordered = [];
|
|
294
|
+
let current = 0; // start from the first memory
|
|
295
|
+
ordered.push(memories[current]);
|
|
296
|
+
used[current] = true;
|
|
297
|
+
for (let step = 1; step < memories.length; step++) {
|
|
298
|
+
const currentEmb = embeddings[current];
|
|
299
|
+
let bestIdx = -1;
|
|
300
|
+
let bestSim = -Infinity;
|
|
301
|
+
for (let j = 0; j < memories.length; j++) {
|
|
302
|
+
if (used[j])
|
|
303
|
+
continue;
|
|
304
|
+
const sim = cosineSimilarity(currentEmb, embeddings[j]);
|
|
305
|
+
if (sim > bestSim) {
|
|
306
|
+
bestSim = sim;
|
|
307
|
+
bestIdx = j;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
if (bestIdx === -1)
|
|
311
|
+
break;
|
|
312
|
+
ordered.push(memories[bestIdx]);
|
|
313
|
+
used[bestIdx] = true;
|
|
314
|
+
current = bestIdx;
|
|
315
|
+
}
|
|
316
|
+
return ordered;
|
|
134
317
|
}
|
|
135
318
|
// ── Chunk helpers ────────────────────────────────────────────────────────────
|
|
136
|
-
|
|
319
|
+
/**
|
|
320
|
+
* Build the per-chunk user prompt fed to the consolidate LLM.
|
|
321
|
+
*
|
|
322
|
+
* Each memory is annotated with two flags that drive the system-prompt
|
|
323
|
+
* rules at lines 181-186:
|
|
324
|
+
* - `(captureMode: hot)` — user-explicit memory; system prompt rule 2
|
|
325
|
+
* forbids proposing delete. ~60 wasted LLM verdicts/4h on this user's
|
|
326
|
+
* stack before this annotation.
|
|
327
|
+
* - `(already queued)` — the memory's body hash matches a pending
|
|
328
|
+
* consolidate proposal; system prompt rule 3 forbids proposing
|
|
329
|
+
* promote/merge/contradict. ~107/4h before this annotation.
|
|
330
|
+
*
|
|
331
|
+
* Both annotations are visible to the LLM. `pendingProposalBodyHashes`
|
|
332
|
+
* is precomputed once per run by `loadPendingConsolidateProposalHashes`
|
|
333
|
+
* so the cost stays O(memories) inside the chunk loop.
|
|
334
|
+
*/
|
|
335
|
+
export function buildChunkPrompt(sourceName, memories, chunkIndex, totalChunks, bodyTruncation, pendingProposalBodyHashes = new Set()) {
|
|
137
336
|
const start = memories[0] ? `memory:${memories[0].name}` : "";
|
|
138
337
|
const end = memories[memories.length - 1] ? `memory:${memories[memories.length - 1].name}` : "";
|
|
338
|
+
const annotationsByIndex = [];
|
|
339
|
+
const hotRefs = [];
|
|
340
|
+
for (const m of memories) {
|
|
341
|
+
let body = "";
|
|
342
|
+
try {
|
|
343
|
+
body = fs.readFileSync(m.filePath, "utf8");
|
|
344
|
+
}
|
|
345
|
+
catch {
|
|
346
|
+
body = "(unreadable)";
|
|
347
|
+
}
|
|
348
|
+
const parsed = parseFrontmatter(body);
|
|
349
|
+
const isHot = parsed.data.captureMode === "hot";
|
|
350
|
+
const bodyHash = createHash("sha256").update(parsed.content.trim(), "utf8").digest("hex");
|
|
351
|
+
const isAlreadyQueued = pendingProposalBodyHashes.has(bodyHash);
|
|
352
|
+
annotationsByIndex.push({ isHot, isAlreadyQueued, body });
|
|
353
|
+
if (isHot)
|
|
354
|
+
hotRefs.push(`memory:${m.name}`);
|
|
355
|
+
}
|
|
139
356
|
const lines = [
|
|
140
357
|
`Source: ${sourceName}`,
|
|
141
358
|
`Chunk ${chunkIndex + 1} of ${totalChunks}, memories ${start}–${end}:`,
|
|
142
359
|
"",
|
|
143
360
|
];
|
|
361
|
+
// Top-of-prompt protection block for hot refs. Neutral phrasing — avoid
|
|
362
|
+
// op-words like "promote", "merge", "contradict" so the model doesn't
|
|
363
|
+
// accidentally treat the warning as a hint to use that op elsewhere
|
|
364
|
+
// (variant B leaked the word "contradict" into the control sample
|
|
365
|
+
// during the diagnostic).
|
|
366
|
+
if (hotRefs.length > 0) {
|
|
367
|
+
lines.push("⛔ DO NOT propose any `delete` operation for these refs — they are user-explicit (captureMode: hot) and the downstream guard refuses them regardless. Proposing delete for any of these only wastes tokens.");
|
|
368
|
+
for (const ref of hotRefs)
|
|
369
|
+
lines.push(` - ${ref}`);
|
|
370
|
+
lines.push("");
|
|
371
|
+
}
|
|
144
372
|
for (let i = 0; i < memories.length; i++) {
|
|
145
373
|
const m = memories[i];
|
|
146
|
-
|
|
374
|
+
const { isHot, isAlreadyQueued, body } = annotationsByIndex[i];
|
|
375
|
+
const annotations = [];
|
|
376
|
+
if (isHot)
|
|
377
|
+
annotations.push("captureMode: hot");
|
|
378
|
+
if (isAlreadyQueued)
|
|
379
|
+
annotations.push("already queued");
|
|
380
|
+
const annotationSuffix = annotations.length > 0 ? ` (${annotations.join("; ")})` : "";
|
|
381
|
+
lines.push(`[${i + 1}] memory:${m.name}${annotationSuffix}`);
|
|
147
382
|
lines.push(`Description: ${m.description || "(none)"}`);
|
|
148
383
|
lines.push(`Tags: ${m.tags.length > 0 ? m.tags.join(", ") : "(none)"}`);
|
|
149
384
|
lines.push("---");
|
|
150
|
-
let body = "";
|
|
151
|
-
try {
|
|
152
|
-
body = fs.readFileSync(m.filePath, "utf8");
|
|
153
|
-
}
|
|
154
|
-
catch {
|
|
155
|
-
body = "(unreadable)";
|
|
156
|
-
}
|
|
157
385
|
lines.push(body.slice(0, bodyTruncation));
|
|
158
386
|
lines.push("");
|
|
159
387
|
}
|
|
160
388
|
return lines.join("\n");
|
|
161
389
|
}
|
|
390
|
+
/**
|
|
391
|
+
* Precompute body-hashes of all currently-pending consolidate proposals so
|
|
392
|
+
* the per-chunk prompt can annotate memories whose body would just produce
|
|
393
|
+
* a deterministic `dedup_pending_proposal` skip. Hash domain matches the
|
|
394
|
+
* dedup site at ~line 1510 (sha256 over the post-frontmatter content,
|
|
395
|
+
* trimmed). Empty set on any read/parse error — fail-safe to "annotate
|
|
396
|
+
* nothing" so the LLM still proposes, just slightly more wastefully.
|
|
397
|
+
*/
|
|
398
|
+
export function loadPendingConsolidateProposalHashes(stashDir) {
|
|
399
|
+
const hashes = new Set();
|
|
400
|
+
try {
|
|
401
|
+
const pending = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
|
|
402
|
+
for (const p of pending) {
|
|
403
|
+
try {
|
|
404
|
+
const body = parseFrontmatter(p.payload.content).content.trim();
|
|
405
|
+
hashes.add(createHash("sha256").update(body, "utf8").digest("hex"));
|
|
406
|
+
}
|
|
407
|
+
catch {
|
|
408
|
+
// skip malformed payloads — they can't dedup anyway
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
catch {
|
|
413
|
+
// listProposals throws on missing stash dir during tests — empty set is safe
|
|
414
|
+
}
|
|
415
|
+
return hashes;
|
|
416
|
+
}
|
|
162
417
|
function isValidOp(op) {
|
|
163
418
|
if (typeof op !== "object" || op === null)
|
|
164
419
|
return false;
|
|
@@ -172,43 +427,133 @@ function isValidOp(op) {
|
|
|
172
427
|
if (o.op === "promote") {
|
|
173
428
|
return typeof o.ref === "string" && typeof o.knowledgeRef === "string";
|
|
174
429
|
}
|
|
430
|
+
if (o.op === "contradict") {
|
|
431
|
+
return typeof o.ref === "string" && typeof o.contradictedByRef === "string";
|
|
432
|
+
}
|
|
175
433
|
return false;
|
|
176
434
|
}
|
|
177
|
-
function mergePlans(chunks) {
|
|
435
|
+
export function mergePlans(chunks, knownRefs) {
|
|
178
436
|
const mergeOps = new Map();
|
|
179
437
|
const deleteOps = new Map();
|
|
180
438
|
const promoteOps = new Map();
|
|
439
|
+
// C-3 / #382: contradict ops keyed by `ref|contradictedByRef` to deduplicate.
|
|
440
|
+
const contradictOps = new Map();
|
|
181
441
|
const warnings = [];
|
|
182
442
|
for (const chunk of chunks) {
|
|
183
443
|
for (const op of chunk) {
|
|
184
444
|
if (op.op === "merge") {
|
|
445
|
+
// Drop ops whose primary the LLM hallucinated (not in the loaded memory
|
|
446
|
+
// pool). Without this guard, a hallucinated primary flows all the way to
|
|
447
|
+
// Phase B where !memoryByRef.has(primary) fires and charges every real
|
|
448
|
+
// secondary with merge_primary_missing — masking LLM hallucinations as
|
|
449
|
+
// filter regressions in health metrics.
|
|
450
|
+
if (knownRefs && !knownRefs.has(op.primary)) {
|
|
451
|
+
warnings.push(`mergePlans: primary ${op.primary} not in loaded memory pool (LLM hallucination) — dropping op before execution.`);
|
|
452
|
+
// Use a dedicated skip reason so dashboards can distinguish
|
|
453
|
+
// hallucinated primaries from stale-DB regressions.
|
|
454
|
+
// Secondaries are real refs; they are NOT charged here — they remain
|
|
455
|
+
// available for other ops to claim.
|
|
456
|
+
continue;
|
|
457
|
+
}
|
|
458
|
+
// Filter hallucinated secondaries while preserving real ones.
|
|
459
|
+
let mergeOp = op;
|
|
460
|
+
if (knownRefs) {
|
|
461
|
+
const filteredSecondaries = op.secondaries.filter((sec) => {
|
|
462
|
+
if (!knownRefs.has(sec)) {
|
|
463
|
+
warnings.push(`mergePlans: secondary ${sec} not in loaded memory pool (LLM hallucination) — dropping from op.`);
|
|
464
|
+
return false;
|
|
465
|
+
}
|
|
466
|
+
return true;
|
|
467
|
+
});
|
|
468
|
+
if (filteredSecondaries.length !== op.secondaries.length) {
|
|
469
|
+
mergeOp = { ...op, secondaries: filteredSecondaries };
|
|
470
|
+
}
|
|
471
|
+
}
|
|
185
472
|
// merge wins over delete
|
|
186
|
-
if (deleteOps.has(
|
|
187
|
-
deleteOps.delete(
|
|
473
|
+
if (deleteOps.has(mergeOp.primary)) {
|
|
474
|
+
deleteOps.delete(mergeOp.primary);
|
|
188
475
|
}
|
|
189
|
-
for (const sec of
|
|
476
|
+
for (const sec of mergeOp.secondaries) {
|
|
190
477
|
if (deleteOps.has(sec))
|
|
191
478
|
deleteOps.delete(sec);
|
|
192
479
|
}
|
|
193
|
-
mergeOps.set(
|
|
480
|
+
mergeOps.set(mergeOp.primary, mergeOp);
|
|
194
481
|
}
|
|
195
482
|
else if (op.op === "delete") {
|
|
196
|
-
|
|
483
|
+
// merge and promote both win over delete. A promote is non-destructive
|
|
484
|
+
// (creates a proposal) but the source memory is counted in `promoted`;
|
|
485
|
+
// if a delete also fires, the ref lands in both `promoted` and
|
|
486
|
+
// `skipReasons`, breaking the invariant by +1.
|
|
487
|
+
if (!mergeOps.has(op.ref) && !promoteOps.has(op.ref)) {
|
|
197
488
|
deleteOps.set(op.ref, op);
|
|
198
489
|
}
|
|
199
490
|
}
|
|
200
491
|
else if (op.op === "promote") {
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
492
|
+
// C-2 / #381: when both a promote and a merge target the same ref,
|
|
493
|
+
// queue the promote FIRST rather than discarding it. The promote op
|
|
494
|
+
// routes through createProposal (the human-gated proposal queue), so
|
|
495
|
+
// it is non-destructive. The merge follows after the proposal is
|
|
496
|
+
// created. This preserves the human reviewer's ability to inspect the
|
|
497
|
+
// promotion before the source memory is merged/deleted.
|
|
498
|
+
// AGM K*8 — retain the maximally informative consistent subset.
|
|
499
|
+
promoteOps.set(op.ref, op);
|
|
500
|
+
}
|
|
501
|
+
else if (op.op === "contradict") {
|
|
502
|
+
// Deduplicate by ref+contradictedByRef pair.
|
|
503
|
+
const key = `${op.ref}|${op.contradictedByRef}`;
|
|
504
|
+
if (!contradictOps.has(key)) {
|
|
505
|
+
contradictOps.set(key, op);
|
|
207
506
|
}
|
|
208
507
|
}
|
|
209
508
|
}
|
|
210
509
|
}
|
|
211
|
-
|
|
510
|
+
// Second pass: enforce merge-wins-over-delete and deduplicate secondaries.
|
|
511
|
+
//
|
|
512
|
+
// 1. Delete/secondary ordering bug: the per-chunk loop removes delete ops
|
|
513
|
+
// for secondaries that were already in deleteOps, but misses the case
|
|
514
|
+
// where the delete chunk came first. A full sweep here fixes both orders.
|
|
515
|
+
//
|
|
516
|
+
// 2. Cross-merge secondary dedup: if ref A is a secondary in two merge ops,
|
|
517
|
+
// only the first (insertion-order) retains it. Without this, a successful
|
|
518
|
+
// merge credits A to mergedSecondaries and a later merge's emitMerge-
|
|
519
|
+
// FailureSkips also charges A to skipReasons — double-counting A while
|
|
520
|
+
// processed has it only once.
|
|
521
|
+
//
|
|
522
|
+
// 3. Primary-as-secondary dedup: if ref A is a primary in one merge op and
|
|
523
|
+
// a secondary in another, remove A from the secondary list. Both merges
|
|
524
|
+
// would otherwise claim A (merged++ for A, then mergedSecondaries++ for A)
|
|
525
|
+
// breaking the invariant the same way.
|
|
526
|
+
// Also remove delete ops for any ref claimed by a promote op (handles the
|
|
527
|
+
// case where the delete chunk appeared before the promote chunk).
|
|
528
|
+
for (const ref of promoteOps.keys()) {
|
|
529
|
+
deleteOps.delete(ref);
|
|
530
|
+
}
|
|
531
|
+
const claimedSecondaries = new Set();
|
|
532
|
+
for (const mergeOp of mergeOps.values()) {
|
|
533
|
+
deleteOps.delete(mergeOp.primary);
|
|
534
|
+
mergeOp.secondaries = mergeOp.secondaries.filter((sec) => {
|
|
535
|
+
if (mergeOps.has(sec)) {
|
|
536
|
+
warnings.push(`Merge: secondary ${sec} is also a merge primary — removing from secondary list to avoid double-count.`);
|
|
537
|
+
return false;
|
|
538
|
+
}
|
|
539
|
+
if (claimedSecondaries.has(sec)) {
|
|
540
|
+
warnings.push(`Merge: secondary ${sec} appears in multiple merge ops — retaining in first op only.`);
|
|
541
|
+
return false;
|
|
542
|
+
}
|
|
543
|
+
claimedSecondaries.add(sec);
|
|
544
|
+
deleteOps.delete(sec);
|
|
545
|
+
return true;
|
|
546
|
+
});
|
|
547
|
+
}
|
|
548
|
+
// C-2 / #381: promote ops are ordered BEFORE merge ops so that the
|
|
549
|
+
// human-gated proposal queue entry is created before any destructive merge.
|
|
550
|
+
// Phase B processes ops in array order, so promote executes first.
|
|
551
|
+
const ops = [
|
|
552
|
+
...promoteOps.values(),
|
|
553
|
+
...mergeOps.values(),
|
|
554
|
+
...deleteOps.values(),
|
|
555
|
+
...contradictOps.values(),
|
|
556
|
+
];
|
|
212
557
|
return { ops, warnings };
|
|
213
558
|
}
|
|
214
559
|
function getJournalPath(stashDir) {
|
|
@@ -359,8 +704,7 @@ function archiveMemory(filePath, stashDir, ref, reason, opIndex, supersededBy, w
|
|
|
359
704
|
...(supersededBy ? { superseded_by: supersededBy } : {}),
|
|
360
705
|
superseded_reason: reason,
|
|
361
706
|
};
|
|
362
|
-
|
|
363
|
-
content = `---\n${fmStr}\n---\n${parsed.content}`;
|
|
707
|
+
content = assembleAssetFromString(yamlStringify(newFm).trimEnd(), parsed.content);
|
|
364
708
|
}
|
|
365
709
|
catch {
|
|
366
710
|
if (warnings)
|
|
@@ -377,9 +721,44 @@ function archiveMemory(filePath, stashDir, ref, reason, opIndex, supersededBy, w
|
|
|
377
721
|
warnings.push(`archiveMemory: write failed for ${ref}: ${String(e)}`);
|
|
378
722
|
}
|
|
379
723
|
}
|
|
724
|
+
// ── LLM resolution ──────────────────────────────────────────────────────────
|
|
725
|
+
/**
|
|
726
|
+
* Resolve the LLM connection for the consolidate pass.
|
|
727
|
+
*
|
|
728
|
+
* Priority order (mirrors extract / reflect / distill — see
|
|
729
|
+
* `src/commands/extract.ts:421-438` and the canonical
|
|
730
|
+
* `resolveImproveProcessRunnerFromProfile` pattern):
|
|
731
|
+
*
|
|
732
|
+
* 1. `profiles.improve.default.processes.consolidate.profile` (or `mode`)
|
|
733
|
+
* via {@link resolveImproveProcessRunnerFromProfile}. Lets the user pin
|
|
734
|
+
* a dedicated model (e.g. `ministral-3b`) for consolidation instead of
|
|
735
|
+
* whatever `defaults.llm` happens to be.
|
|
736
|
+
* 2. `getDefaultLlmConfig(config)` — the baseline default LLM profile.
|
|
737
|
+
*
|
|
738
|
+
* Regression guard (2026-05-26): before this resolver, `akmConsolidate`
|
|
739
|
+
* called `getDefaultLlmConfig` directly and silently ignored a configured
|
|
740
|
+
* `processes.consolidate.profile`, sending every chunk to the default LLM
|
|
741
|
+
* (often a long-context model loaded with a smaller runtime `n_ctx`, causing
|
|
742
|
+
* silent 400s from LM Studio). The investigation lives at
|
|
743
|
+
* `/tmp/akm-health-investigations/consolidation-no-op.md`.
|
|
744
|
+
*/
|
|
745
|
+
function resolveConsolidateLlmConfig(config) {
|
|
746
|
+
const consolidateProcess = config.profiles?.improve?.default?.processes?.consolidate;
|
|
747
|
+
const runnerSpec = resolveImproveProcessRunnerFromProfile(consolidateProcess, config);
|
|
748
|
+
if (runnerSpec && runnerSpec.kind === "llm") {
|
|
749
|
+
return runnerSpec.connection;
|
|
750
|
+
}
|
|
751
|
+
// Non-LLM runner modes (agent/sdk) don't apply to consolidate's HTTP path;
|
|
752
|
+
// fall back to the default LLM profile rather than disabling the pass.
|
|
753
|
+
return getDefaultLlmConfig(config);
|
|
754
|
+
}
|
|
380
755
|
// ── Main entry point ─────────────────────────────────────────────────────────
|
|
381
756
|
export async function akmConsolidate(opts = {}) {
|
|
382
757
|
const startMs = Date.now();
|
|
758
|
+
// Derive a stable PROV-DM token for this run. Callers (e.g. akmImprove)
|
|
759
|
+
// should pass opts.sourceRun to tie proposals back to the parent run;
|
|
760
|
+
// standalone `akm consolidate` gets a self-contained token.
|
|
761
|
+
const sourceRun = opts.sourceRun ?? `consolidate-${startMs}`;
|
|
383
762
|
const config = opts.config ?? loadConfig();
|
|
384
763
|
const stashDir = opts.stashDir ?? resolveStashDir();
|
|
385
764
|
if (!isLlmFeatureEnabled(config, "memory_consolidation")) {
|
|
@@ -394,13 +773,24 @@ export async function akmConsolidate(opts = {}) {
|
|
|
394
773
|
merged: 0,
|
|
395
774
|
deleted: 0,
|
|
396
775
|
promoted: [],
|
|
776
|
+
contradicted: 0,
|
|
397
777
|
warnings: [],
|
|
398
778
|
durationMs: Date.now() - startMs,
|
|
399
779
|
};
|
|
400
780
|
}
|
|
401
781
|
const warnings = [];
|
|
402
782
|
checkForIncompleteJournal(stashDir, opts.recoveryMode ?? "abort", warnings);
|
|
403
|
-
|
|
783
|
+
let memories = loadMemoriesForSource(opts.target, stashDir, warnings);
|
|
784
|
+
// Pre-flight: filter out stale DB entries whose files no longer exist on
|
|
785
|
+
// disk. Without this, memories deleted by a prior run (but not yet
|
|
786
|
+
// reindexed) appear in chunk prompts, causing the LLM to generate plans
|
|
787
|
+
// against ghost refs and wasting tokens. Filtering here ensures the chunk
|
|
788
|
+
// pool and memoryByRef are authoritative against the actual filesystem state.
|
|
789
|
+
const staleCount = memories.filter((m) => !fs.existsSync(m.filePath)).length;
|
|
790
|
+
if (staleCount > 0) {
|
|
791
|
+
warnings.push(`Pre-flight: filtered ${staleCount} stale DB entr${staleCount === 1 ? "y" : "ies"} (file absent on disk) from memory pool before chunking.`);
|
|
792
|
+
}
|
|
793
|
+
memories = memories.filter((m) => fs.existsSync(m.filePath));
|
|
404
794
|
if (memories.length === 0) {
|
|
405
795
|
return {
|
|
406
796
|
schemaVersion: 1,
|
|
@@ -413,55 +803,181 @@ export async function akmConsolidate(opts = {}) {
|
|
|
413
803
|
merged: 0,
|
|
414
804
|
deleted: 0,
|
|
415
805
|
promoted: [],
|
|
806
|
+
contradicted: 0,
|
|
416
807
|
warnings,
|
|
417
808
|
durationMs: Date.now() - startMs,
|
|
418
809
|
};
|
|
419
810
|
}
|
|
811
|
+
if (opts.incrementalSince) {
|
|
812
|
+
memories = narrowToIncrementalCandidates(memories, opts.incrementalSince, warnings);
|
|
813
|
+
if (memories.length === 0) {
|
|
814
|
+
return {
|
|
815
|
+
schemaVersion: 1,
|
|
816
|
+
ok: true,
|
|
817
|
+
shape: "consolidate-result",
|
|
818
|
+
dryRun: opts.dryRun ?? false,
|
|
819
|
+
previewOnly: false,
|
|
820
|
+
target: opts.target ?? stashDir,
|
|
821
|
+
processed: 0,
|
|
822
|
+
merged: 0,
|
|
823
|
+
deleted: 0,
|
|
824
|
+
promoted: [],
|
|
825
|
+
contradicted: 0,
|
|
826
|
+
warnings,
|
|
827
|
+
durationMs: Date.now() - startMs,
|
|
828
|
+
};
|
|
829
|
+
}
|
|
830
|
+
}
|
|
420
831
|
// Consolidation always uses the HTTP LLM client directly — never the agent
|
|
421
832
|
// CLI. The agent CLI is for interactive agent sessions (reflect, propose);
|
|
422
833
|
// structured JSON generation works better and faster via HTTP.
|
|
423
|
-
|
|
834
|
+
//
|
|
835
|
+
// Honor `profiles.improve.default.processes.consolidate.profile` first; fall
|
|
836
|
+
// back to the default LLM. See {@link resolveConsolidateLlmConfig}.
|
|
837
|
+
const llmConfig = resolveConsolidateLlmConfig(config);
|
|
838
|
+
const isHttpPath = !!llmConfig;
|
|
424
839
|
// Chunk sizing: derive a safe chunk size from the configured model context
|
|
425
|
-
// window
|
|
426
|
-
//
|
|
427
|
-
//
|
|
428
|
-
//
|
|
840
|
+
// window so that the full prompt (system prompt + chunk user prompt) never
|
|
841
|
+
// exceeds the model's n_ctx limit. When no context length is configured we
|
|
842
|
+
// fall back to DEFAULT_CONTEXT_LENGTH_TOKENS (8 000) which is conservative
|
|
843
|
+
// enough for most 8K–16K local models.
|
|
429
844
|
//
|
|
430
845
|
// bodyTruncation caps the body excerpt included per memory in the prompt.
|
|
431
846
|
// Reducing it further than 500 chars degrades consolidation quality, so we
|
|
432
847
|
// keep it fixed and let computeSafeChunkSize vary the number of memories
|
|
433
848
|
// per chunk instead.
|
|
434
849
|
const bodyTruncation = 500;
|
|
435
|
-
const modelContextLength =
|
|
436
|
-
const chunkSize = computeSafeChunkSize(modelContextLength, bodyTruncation);
|
|
850
|
+
const modelContextLength = llmConfig?.contextLength ?? DEFAULT_CONTEXT_LENGTH_TOKENS;
|
|
851
|
+
const chunkSize = computeSafeChunkSize(modelContextLength, bodyTruncation, opts.maxChunkSize);
|
|
437
852
|
// -- Phase A: plan generation -----------------------------------------------
|
|
438
853
|
const sourceName = opts.target ?? stashDir;
|
|
854
|
+
// C-1 / #380: Pre-cluster memories by embedding similarity before chunking.
|
|
855
|
+
// This ensures that semantically similar memories land in the same LLM
|
|
856
|
+
// context window, allowing the model to detect and merge duplicates that
|
|
857
|
+
// would otherwise be split across chunks and survive indefinitely.
|
|
858
|
+
// mem0 arXiv:2504.19413, A-MEM arXiv:2502.12110.
|
|
859
|
+
// Fails open: if embeddings are unavailable or fail, original order is used.
|
|
860
|
+
const clusteredMemories = await clusterMemoriesBySimilarity(memories, config);
|
|
439
861
|
const chunks = [];
|
|
440
|
-
for (let i = 0; i <
|
|
441
|
-
chunks.push(
|
|
862
|
+
for (let i = 0; i < clusteredMemories.length; i += chunkSize) {
|
|
863
|
+
chunks.push(clusteredMemories.slice(i, i + chunkSize));
|
|
442
864
|
}
|
|
443
|
-
|
|
865
|
+
// 2026-05-27 prompt-context fix: precompute body-hashes of pending
|
|
866
|
+
// consolidate proposals once, so the per-chunk prompt can annotate
|
|
867
|
+
// memories whose body would just produce a deterministic
|
|
868
|
+
// `dedup_pending_proposal` skip. Cuts ~110 wasted LLM proposals per
|
|
869
|
+
// 4h on this user's stack. See
|
|
870
|
+
// /tmp/akm-health-investigations/tuning-reasons-investigation.md §Q3.
|
|
871
|
+
const pendingProposalBodyHashes = loadPendingConsolidateProposalHashes(stashDir);
|
|
872
|
+
warn(`[consolidate] ${memories.length} memories / ${chunks.length} chunk(s) / chunk_size=${chunkSize}` +
|
|
873
|
+
` / pending-proposal hashes: ${pendingProposalBodyHashes.size}`);
|
|
444
874
|
const chunkOpsArrays = [];
|
|
445
|
-
|
|
875
|
+
// Structured skip-reason histogram (2026-05-26): every deterministic
|
|
876
|
+
// post-LLM op rejection site below also calls `pushSkipReason` so the
|
|
877
|
+
// health rollup can aggregate without regex-parsing English warning
|
|
878
|
+
// strings. See `/tmp/akm-health-investigations/tuning-reasons-investigation.md` §Q2.
|
|
879
|
+
const skipReasons = [];
|
|
880
|
+
// Tracks refs already emitted to skipReasons. A ref can only occupy one
|
|
881
|
+
// accounting bucket; subsequent skip ops for the same ref are recorded as
|
|
882
|
+
// warnings but must not push a second skipReasons entry (that would inflate
|
|
883
|
+
// Σ(skipReasons) and break the invariant by +1 per duplicate).
|
|
884
|
+
const skipReasonEmittedRefs = new Set();
|
|
885
|
+
const pushSkipReason = (op, ref, reason) => {
|
|
886
|
+
// 2026-05-27 cross-chunk double-count fix: if `ref` already contributed
|
|
887
|
+
// to judgedNoAction in its own chunk (a different chunk proposed an op
|
|
888
|
+
// for it that is now being rejected here), promote it from the
|
|
889
|
+
// judgedNoAction bucket into the more specific skipReason bucket.
|
|
890
|
+
// Preserves the invariant: processed == actioned + judgedNoAction +
|
|
891
|
+
// Σ(skipReasons) + failedChunkMemories.
|
|
892
|
+
if (judgedNoActionRefs.delete(ref))
|
|
893
|
+
judgedNoAction--;
|
|
894
|
+
if (skipReasonEmittedRefs.has(ref)) {
|
|
895
|
+
// Already counted once. Record the extra skip for observability but
|
|
896
|
+
// don't push to skipReasons — that would break the accounting invariant.
|
|
897
|
+
warnings.push(`Skip: ${ref} already in skipReasons (${reason} via ${op}); not re-counted.`);
|
|
898
|
+
return;
|
|
899
|
+
}
|
|
900
|
+
skipReasonEmittedRefs.add(ref);
|
|
901
|
+
skipReasons.push({ op, ref, reason });
|
|
902
|
+
};
|
|
903
|
+
// judgedNoAction tracks memories the LLM saw inside a chunk but proposed
|
|
904
|
+
// no op for. Computed per chunk as `chunk.length − unique(targetRefs in ops)`.
|
|
905
|
+
let judgedNoAction = 0;
|
|
906
|
+
// 2026-05-27 cross-chunk double-count fix: refs that contributed to
|
|
907
|
+
// judgedNoAction in their own chunk. When a different chunk's op references
|
|
908
|
+
// one of these as a secondary and that op later fails, the ref would land
|
|
909
|
+
// in BOTH judgedNoAction and skipReasons (delta +1 per occurrence). Track
|
|
910
|
+
// the set so the merge-failure path can decrement and re-bucket.
|
|
911
|
+
const judgedNoActionRefs = new Set();
|
|
912
|
+
// 2026-05-26 accounting-leak fix: memories that belong to a chunk whose
|
|
913
|
+
// LLM call failed before any per-chunk noAction calculation runs. They
|
|
914
|
+
// would otherwise vanish from the envelope's accounting (no judgedNoAction
|
|
915
|
+
// bump, no skipReasons entry, no actioned counter).
|
|
916
|
+
let failedChunkMemories = 0;
|
|
917
|
+
// 2026-05-26 accounting-leak fix: per-secondary tally so successful merges
|
|
918
|
+
// account for `1 + secondaries.length` memories instead of 1.
|
|
919
|
+
let mergedSecondaries = 0;
|
|
920
|
+
// C-6 / #392: Replace two-consecutive-failures abort with failure-rate threshold.
|
|
921
|
+
// Consecutive-count policies are brittle against transient LM Studio reloads:
|
|
922
|
+
// two transient failures abort the run even though the next chunk would succeed.
|
|
923
|
+
// Rate-based abort (≥50% failure over ≥4 chunks) is more robust.
|
|
924
|
+
// Tanenbaum, Distributed Systems §8 — rate-based policies with minimum sample sizes.
|
|
925
|
+
let totalChunksProcessed = 0;
|
|
926
|
+
let totalChunksFailed = 0;
|
|
927
|
+
const ABORT_MIN_CHUNKS = 4;
|
|
928
|
+
const ABORT_FAILURE_RATE = 0.5;
|
|
446
929
|
for (let chunkIdx = 0; chunkIdx < chunks.length; chunkIdx++) {
|
|
447
|
-
// Abort
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
930
|
+
// Abort if failure rate >= 50% over at least 4 processed chunks.
|
|
931
|
+
if (totalChunksProcessed >= ABORT_MIN_CHUNKS) {
|
|
932
|
+
const failureRate = totalChunksFailed / totalChunksProcessed;
|
|
933
|
+
if (failureRate >= ABORT_FAILURE_RATE) {
|
|
934
|
+
const skipped = chunks.length - chunkIdx;
|
|
935
|
+
const abortMsg = `Consolidation aborted — failure rate ${(failureRate * 100).toFixed(0)}% over ${totalChunksProcessed} chunks (>= ${ABORT_FAILURE_RATE * 100}% threshold). LLM may be unavailable. ${skipped} chunk(s) skipped.`;
|
|
936
|
+
warn(abortMsg);
|
|
937
|
+
warnings.push(abortMsg);
|
|
938
|
+
// Account for memories in chunks we never attempted: they are
|
|
939
|
+
// neither judgedNoAction (no plan parsed) nor skipReason (no op
|
|
940
|
+
// rejected). Without this, the accounting invariant fails by
|
|
941
|
+
// `Σ(unattempted_chunk.length)` whenever the abort fires.
|
|
942
|
+
for (let i = chunkIdx; i < chunks.length; i++) {
|
|
943
|
+
failedChunkMemories += chunks[i].length;
|
|
944
|
+
}
|
|
945
|
+
break;
|
|
946
|
+
}
|
|
453
947
|
}
|
|
454
948
|
const chunk = chunks[chunkIdx];
|
|
949
|
+
// All-hot chunk early-exit. The per-prompt hot-list block (see
|
|
950
|
+
// buildChunkPrompt) only *discourages* delete proposals on a mixed chunk;
|
|
951
|
+
// when EVERY memory in the chunk is captureMode: hot, the only ops the LLM
|
|
952
|
+
// could ever propose are deletes — all of which the downstream guard
|
|
953
|
+
// refuses unconditionally. Calling the model is therefore pure token waste.
|
|
954
|
+
// Skip the request entirely and bucket every memory as judgedNoAction (we
|
|
955
|
+
// judged "no action" without spending an LLM call), preserving the
|
|
956
|
+
// accounting invariant `processed == actioned + judgedNoAction +
|
|
957
|
+
// Σ(skipReasons) + failedChunkMemories`. Not counted toward the
|
|
958
|
+
// LLM-failure-rate abort policy — no request was attempted.
|
|
959
|
+
if (chunk.length > 0 && chunk.every((m) => isHotCapturedMemory(m.filePath))) {
|
|
960
|
+
for (const m of chunk)
|
|
961
|
+
judgedNoActionRefs.add(`memory:${m.name}`);
|
|
962
|
+
judgedNoAction += chunk.length;
|
|
963
|
+
warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length}: all ${chunk.length} memories are captureMode: hot — skipping LLM (judged no-action).`);
|
|
964
|
+
continue;
|
|
965
|
+
}
|
|
455
966
|
warn(`[consolidate] chunk ${chunkIdx + 1}/${chunks.length} (${chunk.length} memories) …`);
|
|
456
|
-
const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation);
|
|
967
|
+
const userPrompt = buildChunkPrompt(sourceName, chunk, chunkIdx, chunks.length, bodyTruncation, pendingProposalBodyHashes);
|
|
457
968
|
const raw = await tryLlmFeature("memory_consolidation", config, async () => {
|
|
458
|
-
if (!
|
|
969
|
+
if (!llmConfig)
|
|
459
970
|
return { ok: false, error: "No LLM configured for consolidation" };
|
|
460
971
|
try {
|
|
461
|
-
|
|
972
|
+
// responseSchema lift (PR 1, asset-writers-investigation §5): pass
|
|
973
|
+
// the consolidate plan schema so providers with
|
|
974
|
+
// `supportsJsonSchema: true` enforce shape upstream. Providers that
|
|
975
|
+
// ignore the option fall through to the existing
|
|
976
|
+
// `parseEmbeddedJsonResponse` path on the response side.
|
|
977
|
+
const content = await chatCompletion(llmConfig, [
|
|
462
978
|
{ role: "system", content: CONSOLIDATE_SYSTEM_PROMPT },
|
|
463
979
|
{ role: "user", content: userPrompt },
|
|
464
|
-
]);
|
|
980
|
+
], { responseSchema: CONSOLIDATE_PLAN_JSON_SCHEMA, enableThinking: false });
|
|
465
981
|
return { ok: true, content };
|
|
466
982
|
}
|
|
467
983
|
catch (e) {
|
|
@@ -469,8 +985,15 @@ export async function akmConsolidate(opts = {}) {
|
|
|
469
985
|
}
|
|
470
986
|
}, { ok: false, error: `chunk ${chunkIdx + 1} failed` });
|
|
471
987
|
if (!raw.ok) {
|
|
988
|
+
warn(raw.error ?? `chunk ${chunkIdx + 1} failed`);
|
|
472
989
|
warnings.push(raw.error ?? `chunk ${chunkIdx + 1} failed`);
|
|
473
|
-
|
|
990
|
+
totalChunksProcessed++;
|
|
991
|
+
totalChunksFailed++;
|
|
992
|
+
// Account for the chunk's memories under the failed-chunk bucket.
|
|
993
|
+
// judgedNoAction does NOT run on this path (it's after the success
|
|
994
|
+
// guards) so without this the accounting invariant breaks on every
|
|
995
|
+
// chunk-level transport/parse failure.
|
|
996
|
+
failedChunkMemories += chunk.length;
|
|
474
997
|
continue;
|
|
475
998
|
}
|
|
476
999
|
if (process.env.AKM_DEBUG_LLM) {
|
|
@@ -482,11 +1005,14 @@ export async function akmConsolidate(opts = {}) {
|
|
|
482
1005
|
const hint = raw.content !== undefined && raw.content.trim() === ""
|
|
483
1006
|
? " (empty response — if using a thinking model, disable thinking mode)"
|
|
484
1007
|
: "";
|
|
1008
|
+
warn(`Chunk ${chunkIdx + 1}: invalid plan from AI — skipping.${hint}`);
|
|
485
1009
|
warnings.push(`Chunk ${chunkIdx + 1}: invalid plan from AI — skipping.${hint}`);
|
|
486
|
-
|
|
1010
|
+
totalChunksProcessed++;
|
|
1011
|
+
totalChunksFailed++;
|
|
1012
|
+
failedChunkMemories += chunk.length;
|
|
487
1013
|
continue;
|
|
488
1014
|
}
|
|
489
|
-
|
|
1015
|
+
totalChunksProcessed++; // success
|
|
490
1016
|
const ops = [];
|
|
491
1017
|
for (const op of parsed.operations) {
|
|
492
1018
|
if (isValidOp(op)) {
|
|
@@ -502,9 +1028,37 @@ export async function akmConsolidate(opts = {}) {
|
|
|
502
1028
|
warnings.push(w);
|
|
503
1029
|
}
|
|
504
1030
|
}
|
|
1031
|
+
// Per-chunk judgedNoAction: count memories the LLM saw but proposed no
|
|
1032
|
+
// op for. Membership is by `memory:<name>` ref against the targets of
|
|
1033
|
+
// each op (primary + secondaries for merge; ref otherwise). 2026-05-26:
|
|
1034
|
+
// pre-fix this was a 78/119 (66%) silent drop in the cron run — no
|
|
1035
|
+
// warning, event, or counter. See tuning investigation §Q2.
|
|
1036
|
+
const targetRefs = new Set();
|
|
1037
|
+
for (const op of ops) {
|
|
1038
|
+
if (op.op === "merge") {
|
|
1039
|
+
targetRefs.add(op.primary);
|
|
1040
|
+
for (const s of op.secondaries)
|
|
1041
|
+
targetRefs.add(s);
|
|
1042
|
+
}
|
|
1043
|
+
else {
|
|
1044
|
+
targetRefs.add(op.ref);
|
|
1045
|
+
}
|
|
1046
|
+
}
|
|
1047
|
+
let chunkNoAction = 0;
|
|
1048
|
+
for (const m of chunk) {
|
|
1049
|
+
const memRef = `memory:${m.name}`;
|
|
1050
|
+
if (!targetRefs.has(memRef)) {
|
|
1051
|
+
chunkNoAction++;
|
|
1052
|
+
judgedNoActionRefs.add(memRef);
|
|
1053
|
+
}
|
|
1054
|
+
}
|
|
1055
|
+
judgedNoAction += chunkNoAction;
|
|
505
1056
|
chunkOpsArrays.push(ops);
|
|
506
1057
|
}
|
|
507
|
-
|
|
1058
|
+
// Build the known-refs set from the already-filtered memory pool so
|
|
1059
|
+
// mergePlans() can reject LLM-hallucinated primary refs before execution.
|
|
1060
|
+
const knownRefs = new Set(memories.map((m) => `memory:${m.name}`));
|
|
1061
|
+
const { ops: allOps, warnings: mergeWarnings } = mergePlans(chunkOpsArrays, knownRefs);
|
|
508
1062
|
warnings.push(...mergeWarnings);
|
|
509
1063
|
// -- Dry-run: show AI plan without executing any writes --------------------
|
|
510
1064
|
if (opts.dryRun) {
|
|
@@ -519,6 +1073,13 @@ export async function akmConsolidate(opts = {}) {
|
|
|
519
1073
|
merged: 0,
|
|
520
1074
|
deleted: 0,
|
|
521
1075
|
promoted: [],
|
|
1076
|
+
contradicted: 0,
|
|
1077
|
+
failedChunks: totalChunksFailed,
|
|
1078
|
+
totalChunks: chunks.length,
|
|
1079
|
+
judgedNoAction,
|
|
1080
|
+
skipReasons,
|
|
1081
|
+
mergedSecondaries,
|
|
1082
|
+
failedChunkMemories,
|
|
522
1083
|
planned: allOps,
|
|
523
1084
|
warnings,
|
|
524
1085
|
durationMs: Date.now() - startMs,
|
|
@@ -528,9 +1089,20 @@ export async function akmConsolidate(opts = {}) {
|
|
|
528
1089
|
// -- HTTP path: warn about quality and confirm unless auto-accepted --------
|
|
529
1090
|
if (isHttpPath) {
|
|
530
1091
|
warnings.push("Running on HTTP path — plan generated from truncated memory excerpts; quality may vary.");
|
|
531
|
-
|
|
1092
|
+
// Per-proposal confidence gating is handled by the caller (improve.ts)
|
|
1093
|
+
// via runAutoAcceptGate after this function returns. The gate reads
|
|
1094
|
+
// proposal.confidence (forwarded from op.confidence above) and applies
|
|
1095
|
+
// a minimumThreshold floor of 95 for consolidate's destructive ops.
|
|
1096
|
+
// Here we only gate the interactive-confirm path for manual/HTTP invocations.
|
|
1097
|
+
if (opts.autoAccept === undefined && allOps.length > 0) {
|
|
532
1098
|
const n = allOps.length;
|
|
533
|
-
|
|
1099
|
+
// Non-interactive contexts (CI / test runners / piped stdin) must not
|
|
1100
|
+
// block on an unanswerable prompt. Default to a non-destructive "no"
|
|
1101
|
+
// so callers in those contexts get the same "aborted, preview only"
|
|
1102
|
+
// shape they'd get from explicit user dismissal. AKM_NON_INTERACTIVE
|
|
1103
|
+
// lets callers force this path even when stdin happens to be a TTY.
|
|
1104
|
+
const nonInteractive = process.stdin.isTTY === false || process.env.AKM_NON_INTERACTIVE === "1";
|
|
1105
|
+
const answer = nonInteractive ? false : await promptConfirm(`Apply ${n} operations? [y/N] `);
|
|
534
1106
|
if (!answer) {
|
|
535
1107
|
return {
|
|
536
1108
|
schemaVersion: 1,
|
|
@@ -543,8 +1115,15 @@ export async function akmConsolidate(opts = {}) {
|
|
|
543
1115
|
merged: 0,
|
|
544
1116
|
deleted: 0,
|
|
545
1117
|
promoted: [],
|
|
1118
|
+
contradicted: 0,
|
|
1119
|
+
failedChunks: totalChunksFailed,
|
|
1120
|
+
totalChunks: chunks.length,
|
|
1121
|
+
judgedNoAction,
|
|
1122
|
+
skipReasons,
|
|
1123
|
+
mergedSecondaries,
|
|
1124
|
+
failedChunkMemories,
|
|
546
1125
|
planned: allOps,
|
|
547
|
-
warnings: [...warnings, "Aborted by user."],
|
|
1126
|
+
warnings: [...warnings, nonInteractive ? "Non-interactive context: skipped apply." : "Aborted by user."],
|
|
548
1127
|
durationMs: Date.now() - startMs,
|
|
549
1128
|
};
|
|
550
1129
|
}
|
|
@@ -559,6 +1138,12 @@ export async function akmConsolidate(opts = {}) {
|
|
|
559
1138
|
let merged = 0;
|
|
560
1139
|
let deleted = 0;
|
|
561
1140
|
const promoted = [];
|
|
1141
|
+
let contradicted = 0; // C-3 / #382: count of contradiction edges written
|
|
1142
|
+
// Within-run dedup: track source refs for which a promote proposal was
|
|
1143
|
+
// already created this run. The LLM can return multiple promote ops for
|
|
1144
|
+
// different source memories that happen to have identical content (all are
|
|
1145
|
+
// duplicate memories), so we also need a content-hash guard below.
|
|
1146
|
+
const promotedSourceRefs = new Set();
|
|
562
1147
|
// Build a lookup map: ref → MemoryEntry
|
|
563
1148
|
const memoryByRef = new Map();
|
|
564
1149
|
for (const m of memories) {
|
|
@@ -566,11 +1151,40 @@ export async function akmConsolidate(opts = {}) {
|
|
|
566
1151
|
}
|
|
567
1152
|
for (let opIndex = 0; opIndex < allOps.length; opIndex++) {
|
|
568
1153
|
const op = allOps[opIndex];
|
|
569
|
-
|
|
1154
|
+
const opDisplayRef = op.op === "merge" ? op.primary : op.op === "contradict" ? `${op.ref} ↔ ${op.contradictedByRef}` : op.ref;
|
|
1155
|
+
warn(`[consolidate] ${opIndex + 1}/${allOps.length} ${op.op} ${opDisplayRef}`);
|
|
570
1156
|
if (op.op === "merge") {
|
|
1157
|
+
// Accounting helper: emit a per-participant skipReason for failed
|
|
1158
|
+
// merges so primary + every loaded-memory secondary land in the
|
|
1159
|
+
// structured skip histogram. Pre-2026-05-26 only the primary was
|
|
1160
|
+
// counted (1 skipReason per failed merge), leaving N secondaries
|
|
1161
|
+
// unaccounted for in the `processed == actioned + noAction + Σskips`
|
|
1162
|
+
// invariant — the source of the 4–11 silent leaks per run.
|
|
1163
|
+
const emitMergeFailureSkips = (reason) => {
|
|
1164
|
+
if (memoryByRef.has(op.primary))
|
|
1165
|
+
pushSkipReason("merge", op.primary, reason);
|
|
1166
|
+
for (const secRef of op.secondaries) {
|
|
1167
|
+
if (memoryByRef.has(secRef))
|
|
1168
|
+
pushSkipReason("merge", secRef, reason);
|
|
1169
|
+
}
|
|
1170
|
+
};
|
|
571
1171
|
const primaryEntry = memoryByRef.get(op.primary);
|
|
572
1172
|
if (!primaryEntry) {
|
|
573
|
-
|
|
1173
|
+
// This fires when a prior op in the same run consumed this ref as a
|
|
1174
|
+
// secondary and Fix-A pruned it from memoryByRef. It should NOT fire
|
|
1175
|
+
// for hallucinated primaries (those are dropped by mergePlans() before
|
|
1176
|
+
// reaching here). If this counter is non-zero, suspect an intra-run
|
|
1177
|
+
// cross-chunk race, not a filter regression.
|
|
1178
|
+
warnings.push(`Merge: primary ${op.primary} not found in loaded memories (pruned by prior op this run) — skipping.`);
|
|
1179
|
+
emitMergeFailureSkips("merge_primary_missing");
|
|
1180
|
+
continue;
|
|
1181
|
+
}
|
|
1182
|
+
// Defense-in-depth: even if the entry is in memoryByRef (pre-flight ran
|
|
1183
|
+
// before this run's own ops), the file may have been deleted by a
|
|
1184
|
+
// concurrent process or an edge case the pre-flight filter missed.
|
|
1185
|
+
if (!fs.existsSync(primaryEntry.filePath)) {
|
|
1186
|
+
warnings.push(`Merge: primary ${op.primary} file gone at execution time (stale entry) — skipping.`);
|
|
1187
|
+
emitMergeFailureSkips("merge_primary_file_gone");
|
|
574
1188
|
continue;
|
|
575
1189
|
}
|
|
576
1190
|
// Phase B: generate merged content
|
|
@@ -579,29 +1193,106 @@ export async function akmConsolidate(opts = {}) {
|
|
|
579
1193
|
const secEntry = memoryByRef.get(secRef);
|
|
580
1194
|
if (!secEntry) {
|
|
581
1195
|
warnings.push(`Merge: secondary ${secRef} not found — skipping merge op.`);
|
|
1196
|
+
// No accounting impact: a missing secondary is a phantom ref and
|
|
1197
|
+
// never contributed to any chunk's targetRefs reduction. We still
|
|
1198
|
+
// continue the loop to gather the remaining valid secondaries.
|
|
582
1199
|
continue;
|
|
583
1200
|
}
|
|
584
1201
|
secondaryBodies.push(secRef);
|
|
585
1202
|
}
|
|
586
|
-
if (secondaryBodies.length === 0)
|
|
1203
|
+
if (secondaryBodies.length === 0) {
|
|
1204
|
+
warnings.push(`Merge: ${op.primary} has no valid secondaries — skipping.`);
|
|
1205
|
+
emitMergeFailureSkips("merge_no_valid_secondaries");
|
|
587
1206
|
continue;
|
|
1207
|
+
}
|
|
1208
|
+
// Pre-flight hot guard — skip the LLM call entirely if any participant
|
|
1209
|
+
// is hot or unparseable. Without this, mixed chunks still send hot merges
|
|
1210
|
+
// to the planner which proposes them; generateMergedContent() is then
|
|
1211
|
+
// called, produces output without `description`, and the skip is
|
|
1212
|
+
// misattributed to merge_missing_description instead of the real cause.
|
|
1213
|
+
const preflightParticipants = [op.primary, ...op.secondaries];
|
|
1214
|
+
const preflightBlocked = preflightParticipants.flatMap((ref) => {
|
|
1215
|
+
const e = memoryByRef.get(ref);
|
|
1216
|
+
if (!e)
|
|
1217
|
+
return [];
|
|
1218
|
+
const verdict = consolidateGuardStatus(e.filePath);
|
|
1219
|
+
if (verdict === "hot" || verdict === "unparseable")
|
|
1220
|
+
return [{ ref, verdict }];
|
|
1221
|
+
return [];
|
|
1222
|
+
});
|
|
1223
|
+
if (preflightBlocked.length > 0) {
|
|
1224
|
+
const detail = preflightBlocked.map((p) => `${p.ref} (${p.verdict})`).join(", ");
|
|
1225
|
+
warnings.push(`Merge: refused for ${op.primary} — ${preflightBlocked.length} participant(s) blocked by hot/unparseable frontmatter guard (pre-flight): ${detail}`);
|
|
1226
|
+
emitMergeFailureSkips("merge_participant_blocked");
|
|
1227
|
+
continue;
|
|
1228
|
+
}
|
|
588
1229
|
let primaryBody = "";
|
|
589
1230
|
try {
|
|
590
1231
|
primaryBody = fs.readFileSync(primaryEntry.filePath, "utf8");
|
|
591
1232
|
}
|
|
592
1233
|
catch {
|
|
593
1234
|
warnings.push(`Merge: could not read primary ${op.primary} — skipping.`);
|
|
1235
|
+
emitMergeFailureSkips("merge_read_failed");
|
|
594
1236
|
continue;
|
|
595
1237
|
}
|
|
596
|
-
const
|
|
597
|
-
if (
|
|
1238
|
+
const mergeResult = await generateMergedContent(config, op.primary, primaryBody, op.secondaries, memoryByRef);
|
|
1239
|
+
if ("error" in mergeResult) {
|
|
1240
|
+
warnings.push(`Merge: ${mergeResult.error} for ${mergeResult.detail}.`);
|
|
1241
|
+
emitMergeFailureSkips(mergeResult.error);
|
|
598
1242
|
continue;
|
|
599
|
-
|
|
1243
|
+
}
|
|
1244
|
+
const mergedContent = mergeResult.content;
|
|
1245
|
+
// Validate frontmatter of merged content — must have a `---` block
|
|
1246
|
+
// with at minimum a `description` field. We parse via the hand-rolled
|
|
1247
|
+
// parser (cheap) AND require non-empty description. This guards against
|
|
1248
|
+
// the historical defect where merged memories were written back with
|
|
1249
|
+
// empty `description` and later polluted the promote path.
|
|
1250
|
+
let parsedMerged;
|
|
600
1251
|
try {
|
|
601
|
-
parseFrontmatter(mergedContent);
|
|
1252
|
+
parsedMerged = parseFrontmatter(mergedContent);
|
|
602
1253
|
}
|
|
603
1254
|
catch {
|
|
604
1255
|
warnings.push(`Merge: merged content for ${op.primary} has invalid frontmatter — skipping.`);
|
|
1256
|
+
emitMergeFailureSkips("merge_invalid_frontmatter");
|
|
1257
|
+
continue;
|
|
1258
|
+
}
|
|
1259
|
+
if (parsedMerged.frontmatter === null) {
|
|
1260
|
+
warnings.push(`Merge: merged content for ${op.primary} has no frontmatter block — skipping.`);
|
|
1261
|
+
emitMergeFailureSkips("merge_invalid_frontmatter");
|
|
1262
|
+
continue;
|
|
1263
|
+
}
|
|
1264
|
+
const mergedDesc = parsedMerged.data.description;
|
|
1265
|
+
if (typeof mergedDesc !== "string" || mergedDesc.trim().length === 0) {
|
|
1266
|
+
warnings.push(`Merge: merged content for ${op.primary} missing description — skipping.`);
|
|
1267
|
+
emitMergeFailureSkips("merge_missing_description");
|
|
1268
|
+
continue;
|
|
1269
|
+
}
|
|
1270
|
+
const truncReason = detectTruncatedDescription(mergedDesc);
|
|
1271
|
+
if (truncReason) {
|
|
1272
|
+
warnings.push(`Merge: merged content for ${op.primary} has truncated description (${truncReason}) — skipping.`);
|
|
1273
|
+
emitMergeFailureSkips("merge_truncated_description");
|
|
1274
|
+
continue;
|
|
1275
|
+
}
|
|
1276
|
+
// captureMode:hot guard — refuse the merge if ANY participating memory
|
|
1277
|
+
// (primary or secondary) was user-captured or has unparseable frontmatter
|
|
1278
|
+
// (could have hidden a hot flag). Hot memories are user-explicit and
|
|
1279
|
+
// must not be deleted/overwritten by the consolidate LLM. 14 user
|
|
1280
|
+
// memories were silent-deleted by consolidate before this guard landed;
|
|
1281
|
+
// recovery required copying from .akm/archive/ by hand.
|
|
1282
|
+
const mergeParticipants = [op.primary, ...op.secondaries];
|
|
1283
|
+
const blockedParticipants = mergeParticipants.flatMap((ref) => {
|
|
1284
|
+
const e = memoryByRef.get(ref);
|
|
1285
|
+
if (!e)
|
|
1286
|
+
return [];
|
|
1287
|
+
const verdict = consolidateGuardStatus(e.filePath);
|
|
1288
|
+
if (verdict === "hot" || verdict === "unparseable")
|
|
1289
|
+
return [{ ref, verdict }];
|
|
1290
|
+
return [];
|
|
1291
|
+
});
|
|
1292
|
+
if (blockedParticipants.length > 0) {
|
|
1293
|
+
const detail = blockedParticipants.map((p) => `${p.ref} (${p.verdict})`).join(", ");
|
|
1294
|
+
warnings.push(`Merge: refused for ${op.primary} — ${blockedParticipants.length} participant(s) blocked by hot/unparseable frontmatter guard: ${detail}`);
|
|
1295
|
+
emitMergeFailureSkips("merge_participant_blocked");
|
|
605
1296
|
continue;
|
|
606
1297
|
}
|
|
607
1298
|
// Backup secondaries before deleting
|
|
@@ -618,6 +1309,7 @@ export async function akmConsolidate(opts = {}) {
|
|
|
618
1309
|
}
|
|
619
1310
|
catch (e) {
|
|
620
1311
|
warnings.push(`Merge: write failed for ${op.primary}: ${String(e)}`);
|
|
1312
|
+
emitMergeFailureSkips("merge_write_failed");
|
|
621
1313
|
continue;
|
|
622
1314
|
}
|
|
623
1315
|
// Archive and delete secondaries (P1-B: soft-invalidation)
|
|
@@ -639,11 +1331,45 @@ export async function akmConsolidate(opts = {}) {
|
|
|
639
1331
|
}
|
|
640
1332
|
markJournalCompleted(stashDir, op.primary);
|
|
641
1333
|
merged++;
|
|
1334
|
+
// 2026-05-26 accounting-leak fix: `merged` is op-level, but each
|
|
1335
|
+
// successful merge actions `1 + secondaries.length` memories. Without
|
|
1336
|
+
// this counter the accounting invariant breaks by `secondaries.length`
|
|
1337
|
+
// per successful merge (chunk loop excluded all secondaries from
|
|
1338
|
+
// judgedNoAction via targetRefs, but only the primary is credited to
|
|
1339
|
+
// `merged`). Count only loaded-memory secondaries; phantom secondary
|
|
1340
|
+
// refs never affected any chunk's targetRefs in the first place.
|
|
1341
|
+
for (const secRef of op.secondaries) {
|
|
1342
|
+
if (memoryByRef.has(secRef))
|
|
1343
|
+
mergedSecondaries++;
|
|
1344
|
+
}
|
|
1345
|
+
// Prune consumed refs from memoryByRef so later ops in this run cannot
|
|
1346
|
+
// reference an absorbed secondary as a merge primary and proceed with a
|
|
1347
|
+
// stale entry. Primary is rewritten (not deleted), so we only remove
|
|
1348
|
+
// secondaries; the primary ref remains valid under its new content.
|
|
1349
|
+
for (const secRef of op.secondaries) {
|
|
1350
|
+
memoryByRef.delete(secRef);
|
|
1351
|
+
}
|
|
642
1352
|
}
|
|
643
1353
|
else if (op.op === "delete") {
|
|
644
1354
|
const entry = memoryByRef.get(op.ref);
|
|
645
1355
|
if (!entry) {
|
|
646
1356
|
warnings.push(`Delete: ${op.ref} not found in loaded memories — skipping.`);
|
|
1357
|
+
// Phantom ref: not in the batch so not in processed. Pushing to
|
|
1358
|
+
// skipReasons would inflate Σ(skipReasons) without a matching processed
|
|
1359
|
+
// entry, breaking the accounting invariant. Visibility is preserved via
|
|
1360
|
+
// the warnings array above.
|
|
1361
|
+
continue;
|
|
1362
|
+
}
|
|
1363
|
+
// captureMode:hot guard — refuse to delete user-captured memories OR
|
|
1364
|
+
// memories whose frontmatter is unparseable (could have hidden the hot
|
|
1365
|
+
// flag). The consolidate LLM was deleting hot-captured user memos as
|
|
1366
|
+
// "redundant" — 14 such deletes were silently archived between
|
|
1367
|
+
// 2026-05-19 and 2026-05-20 before this guard. Hot memories are
|
|
1368
|
+
// user-explicit and may only be deleted by the user.
|
|
1369
|
+
const guard = consolidateGuardStatus(entry.filePath);
|
|
1370
|
+
if (guard === "hot" || guard === "unparseable") {
|
|
1371
|
+
warnings.push(`Delete: refused for ${op.ref} — ${guard === "hot" ? "captureMode:hot (user-explicit; never auto-delete)" : "frontmatter unparseable (cannot verify hot flag absent)"}. Reason from LLM: "${op.reason ?? "n/a"}"`);
|
|
1372
|
+
pushSkipReason("delete", op.ref, "captureMode_hot_refused");
|
|
647
1373
|
continue;
|
|
648
1374
|
}
|
|
649
1375
|
if (fs.existsSync(entry.filePath)) {
|
|
@@ -656,15 +1382,40 @@ export async function akmConsolidate(opts = {}) {
|
|
|
656
1382
|
await deleteAssetFromSource(target.source, target.config, parsedRef);
|
|
657
1383
|
markJournalCompleted(stashDir, op.ref);
|
|
658
1384
|
deleted++;
|
|
1385
|
+
// Prune from memoryByRef so later ops in this run cannot reference a
|
|
1386
|
+
// deleted memory as a merge primary or secondary.
|
|
1387
|
+
memoryByRef.delete(op.ref);
|
|
659
1388
|
}
|
|
660
1389
|
catch (e) {
|
|
661
|
-
|
|
1390
|
+
// Distinguish "file already absent" from genuine failures. A prior run
|
|
1391
|
+
// may have deleted the file but the DB was not yet re-indexed, so the
|
|
1392
|
+
// ref still appeared in memoryByRef. The delete goal is already met.
|
|
1393
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
1394
|
+
if (msg.includes("not found in source")) {
|
|
1395
|
+
warnings.push(`Delete: ${op.ref} — file already absent (stale DB entry); skipping.`);
|
|
1396
|
+
pushSkipReason("delete", op.ref, "delete_already_gone");
|
|
1397
|
+
}
|
|
1398
|
+
else {
|
|
1399
|
+
warnings.push(`Delete: failed for ${op.ref}: ${String(e)}`);
|
|
1400
|
+
pushSkipReason("delete", op.ref, "delete_failed");
|
|
1401
|
+
}
|
|
662
1402
|
}
|
|
663
1403
|
}
|
|
664
1404
|
else if (op.op === "promote") {
|
|
665
1405
|
const entry = memoryByRef.get(op.ref);
|
|
666
1406
|
if (!entry) {
|
|
667
1407
|
warnings.push(`Promote: ${op.ref} not found in loaded memories — skipping.`);
|
|
1408
|
+
// Phantom ref: not in processed, so no skipReason (same rationale as
|
|
1409
|
+
// delete_ref_missing above).
|
|
1410
|
+
continue;
|
|
1411
|
+
}
|
|
1412
|
+
// Within-run source-ref dedup: skip if this source memory was already
|
|
1413
|
+
// promoted earlier in this run (safety belt — mergePlans already
|
|
1414
|
+
// deduplicates promote ops by source ref via Map, but this guard also
|
|
1415
|
+
// catches any future code paths that bypass mergePlans).
|
|
1416
|
+
if (promotedSourceRefs.has(op.ref)) {
|
|
1417
|
+
warnings.push(`Skipping promote: ${op.ref} already promoted in this run`);
|
|
1418
|
+
pushSkipReason("promote", op.ref, "promote_already_promoted_this_run");
|
|
668
1419
|
continue;
|
|
669
1420
|
}
|
|
670
1421
|
let knowledgeRef = op.knowledgeRef;
|
|
@@ -679,10 +1430,11 @@ export async function akmConsolidate(opts = {}) {
|
|
|
679
1430
|
knowledgeRef = `knowledge:${slug}`;
|
|
680
1431
|
warnings.push(`Normalized invalid ref "${op.knowledgeRef}" → "${knowledgeRef}"`);
|
|
681
1432
|
}
|
|
682
|
-
// Idempotency: check pending proposals
|
|
1433
|
+
// Idempotency: check pending proposals by target ref
|
|
683
1434
|
const existingProposals = listProposals(stashDir, { ref: knowledgeRef });
|
|
684
1435
|
if (existingProposals.some((p) => p.status === "pending")) {
|
|
685
1436
|
warnings.push(`Skipping promote: pending proposal already exists for ${knowledgeRef}`);
|
|
1437
|
+
pushSkipReason("promote", op.ref, "promote_pending_proposal_exists");
|
|
686
1438
|
continue;
|
|
687
1439
|
}
|
|
688
1440
|
// Idempotency: check if knowledge asset already exists
|
|
@@ -690,6 +1442,7 @@ export async function akmConsolidate(opts = {}) {
|
|
|
690
1442
|
const destPath = path.join(target.source.path, "knowledge", `${parsedKnowledgeRef.name}.md`);
|
|
691
1443
|
if (fs.existsSync(destPath)) {
|
|
692
1444
|
warnings.push(`Skipping promote: ${knowledgeRef} already exists in source`);
|
|
1445
|
+
pushSkipReason("promote", op.ref, "promote_already_exists");
|
|
693
1446
|
continue;
|
|
694
1447
|
}
|
|
695
1448
|
let memoryContent = "";
|
|
@@ -698,24 +1451,173 @@ export async function akmConsolidate(opts = {}) {
|
|
|
698
1451
|
}
|
|
699
1452
|
catch (e) {
|
|
700
1453
|
warnings.push(`Promote: could not read ${op.ref}: ${String(e)}`);
|
|
1454
|
+
pushSkipReason("promote", op.ref, "promote_read_failed");
|
|
1455
|
+
continue;
|
|
1456
|
+
}
|
|
1457
|
+
// Defensive sanitization: legacy memory files written by older
|
|
1458
|
+
// consolidate runs may still carry outer code fences or broken YAML.
|
|
1459
|
+
// Strip them here so we never propose a polluted asset.
|
|
1460
|
+
const promoteSanitized = sanitizeMergedContent(memoryContent);
|
|
1461
|
+
if (!promoteSanitized.ok) {
|
|
1462
|
+
warnings.push(`Promote: rejected ${op.ref} — source memory failed sanitization (${promoteSanitized.reason}).`);
|
|
1463
|
+
pushSkipReason("promote", op.ref, "promote_sanitization_failed");
|
|
1464
|
+
continue;
|
|
1465
|
+
}
|
|
1466
|
+
memoryContent = promoteSanitized.result.content;
|
|
1467
|
+
// SOURCE_SUPERSEDED guard: refuse to promote a memory whose source
|
|
1468
|
+
// frontmatter carries `status: superseded`. Predicate at module top
|
|
1469
|
+
// (`hasSupersededStatus`) so tests can exercise it directly.
|
|
1470
|
+
if (hasSupersededStatus(promoteSanitized.result.frontmatter)) {
|
|
1471
|
+
warnings.push(`Promote: refused for ${op.ref} → ${knowledgeRef} — source memory has status:superseded; superseded memories are not promotable knowledge.`);
|
|
1472
|
+
pushSkipReason("promote", op.ref, "promote_superseded");
|
|
1473
|
+
continue;
|
|
1474
|
+
}
|
|
1475
|
+
// Parse the source memory up-front so the body/frontmatter checks below
|
|
1476
|
+
// share the same parsed view.
|
|
1477
|
+
const parsedMemory = parseFrontmatter(memoryContent);
|
|
1478
|
+
// Reject sources whose body is too small to make useful knowledge.
|
|
1479
|
+
// Observed failure: memory files whose body is literally a tags string
|
|
1480
|
+
// ("discord,notification,send-notification") get promoted to knowledge
|
|
1481
|
+
// proposals that no reviewer would accept. Threshold is conservative —
|
|
1482
|
+
// 100 chars catches single-line tag dumps without rejecting genuinely
|
|
1483
|
+
// terse but valid notes.
|
|
1484
|
+
const PROMOTE_BODY_MIN_CHARS = 100;
|
|
1485
|
+
const sourceBody = parsedMemory.content.trim();
|
|
1486
|
+
if (sourceBody.length < PROMOTE_BODY_MIN_CHARS) {
|
|
1487
|
+
warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — source memory body is too small (${sourceBody.length} chars; need ≥${PROMOTE_BODY_MIN_CHARS}) to make useful knowledge.`);
|
|
1488
|
+
pushSkipReason("promote", op.ref, "promote_source_too_small");
|
|
1489
|
+
continue;
|
|
1490
|
+
}
|
|
1491
|
+
// Cross-run + within-run content dedup: if an identical body already
|
|
1492
|
+
// exists in ANY pending consolidate proposal (regardless of target ref),
|
|
1493
|
+
// skip. This prevents duplicate proposals when:
|
|
1494
|
+
// (a) Multiple source memories have identical bodies but differ only
|
|
1495
|
+
// in noise frontmatter (`inferenceProcessed: true` twin alongside
|
|
1496
|
+
// the original; differing `updated:` timestamps; etc.) — the body
|
|
1497
|
+
// is the load-bearing content, so dedup must hash on body only.
|
|
1498
|
+
// (b) A prior run created a proposal for the same body under a
|
|
1499
|
+
// different knowledgeRef slug.
|
|
1500
|
+
const bodyHash = createHash("sha256").update(sourceBody, "utf8").digest("hex");
|
|
1501
|
+
const allPendingConsolidateProposals = listProposals(stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
|
|
1502
|
+
const contentDupProposal = allPendingConsolidateProposals.find((p) => {
|
|
1503
|
+
const otherBody = parseFrontmatter(p.payload.content).content.trim();
|
|
1504
|
+
return createHash("sha256").update(otherBody, "utf8").digest("hex") === bodyHash;
|
|
1505
|
+
});
|
|
1506
|
+
if (contentDupProposal) {
|
|
1507
|
+
warnings.push(`Skipping promote: identical body already pending as proposal ${contentDupProposal.id} (ref: ${contentDupProposal.ref}); skipping duplicate for ${op.ref} → ${knowledgeRef}`);
|
|
1508
|
+
pushSkipReason("promote", op.ref, "dedup_pending_proposal");
|
|
701
1509
|
continue;
|
|
702
1510
|
}
|
|
703
1511
|
try {
|
|
704
|
-
|
|
1512
|
+
// Use LLM-provided description; fall back to memory's own description
|
|
1513
|
+
// (post-sanitization frontmatter is authoritative).
|
|
1514
|
+
const description = (typeof op.description === "string" && op.description.trim()
|
|
1515
|
+
? op.description.trim()
|
|
1516
|
+
: parsedMemory.data?.description?.trim()) ?? "";
|
|
1517
|
+
// Validate the resolved frontmatter before emitting a proposal.
|
|
1518
|
+
// Required field: non-empty description. Reject obvious truncation
|
|
1519
|
+
// markers (description ends with `,`/`;`/`:`/`...`/hanging connector)
|
|
1520
|
+
// so the queue never sees half-formed metadata that the reviewer
|
|
1521
|
+
// would only reject.
|
|
1522
|
+
const fmCheck = validateProposalFrontmatter({ description });
|
|
1523
|
+
if (!fmCheck.ok) {
|
|
1524
|
+
warnings.push(`Promote: rejected ${op.ref} → ${knowledgeRef} — ${fmCheck.reason}.`);
|
|
1525
|
+
pushSkipReason("promote", op.ref, "promote_invalid_frontmatter");
|
|
1526
|
+
continue;
|
|
1527
|
+
}
|
|
1528
|
+
// Merge `description` INTO the body's YAML frontmatter so it lands in
|
|
1529
|
+
// the on-disk asset when the proposal is accepted. The descriptionQuality
|
|
1530
|
+
// validator parses `payload.content` body (not the envelope
|
|
1531
|
+
// `payload.frontmatter`), and a memory's native frontmatter has
|
|
1532
|
+
// `captureMode`/`beliefState`/etc. but never `description` — without
|
|
1533
|
+
// this merge, 60+ pending proposals were blocked at accept-time with
|
|
1534
|
+
// MISSING_FRONTMATTER_DESCRIPTION even though the envelope had it.
|
|
1535
|
+
// (The body-frontmatter assumption baked into the 2026-05-20 comment
|
|
1536
|
+
// below was wrong: body fm and envelope fm only converge when the
|
|
1537
|
+
// writer explicitly merges them, which it now does.)
|
|
1538
|
+
const mergedBodyFm = {
|
|
1539
|
+
...(parsedMemory.data ?? {}),
|
|
1540
|
+
description,
|
|
1541
|
+
};
|
|
1542
|
+
const serializedMergedFm = yamlStringify(mergedBodyFm).trimEnd();
|
|
1543
|
+
const proposalContent = assembleAssetFromString(serializedMergedFm, parsedMemory.content);
|
|
1544
|
+
// Pre-emit dedup against pending consolidate proposals from the
|
|
1545
|
+
// same improve run (slug-variant match). The cross-run content-hash
|
|
1546
|
+
// dedup inside `mergePlans` handles duplicates against existing
|
|
1547
|
+
// stash assets — see commit history for the deletion of the
|
|
1548
|
+
// unbounded embedding + cross-type slug branches.
|
|
1549
|
+
const dedup = await checkPreEmitDedup({
|
|
1550
|
+
candidateRef: knowledgeRef,
|
|
1551
|
+
candidateText: `${description}. ${memoryContent}`,
|
|
1552
|
+
stashDir,
|
|
1553
|
+
config,
|
|
1554
|
+
});
|
|
1555
|
+
if (dedup.duplicate) {
|
|
1556
|
+
warnings.push(`Promote: skipped ${op.ref} → ${knowledgeRef} — ${dedup.reason}.`);
|
|
1557
|
+
pushSkipReason("promote", op.ref, "promote_dedup_window");
|
|
1558
|
+
continue;
|
|
1559
|
+
}
|
|
1560
|
+
const proposalResult = createProposal(stashDir, {
|
|
705
1561
|
ref: knowledgeRef,
|
|
706
1562
|
source: "consolidate",
|
|
707
|
-
|
|
1563
|
+
sourceRun,
|
|
1564
|
+
payload: {
|
|
1565
|
+
content: proposalContent,
|
|
1566
|
+
frontmatter: { description },
|
|
1567
|
+
},
|
|
1568
|
+
...(typeof op.confidence === "number" ? { confidence: op.confidence } : {}),
|
|
708
1569
|
});
|
|
709
|
-
|
|
710
|
-
|
|
1570
|
+
if (isProposalSkipped(proposalResult)) {
|
|
1571
|
+
warnings.push(`Promote: skipped proposal for ${op.ref} (${proposalResult.reason}): ${proposalResult.message}`);
|
|
1572
|
+
pushSkipReason("promote", op.ref, `promote_proposal_${proposalResult.reason}`);
|
|
1573
|
+
}
|
|
1574
|
+
else {
|
|
1575
|
+
promoted.push(proposalResult.id);
|
|
1576
|
+
promotedSourceRefs.add(op.ref);
|
|
1577
|
+
markJournalCompleted(stashDir, op.ref);
|
|
1578
|
+
}
|
|
711
1579
|
}
|
|
712
1580
|
catch (e) {
|
|
713
1581
|
warnings.push(`Promote: createProposal failed for ${op.ref}: ${String(e)}`);
|
|
1582
|
+
pushSkipReason("promote", op.ref, "promote_create_failed");
|
|
1583
|
+
}
|
|
1584
|
+
}
|
|
1585
|
+
else if (op.op === "contradict") {
|
|
1586
|
+
// C-3 / #382: Write contradictedBy edges so resolveFamilyContradictions
|
|
1587
|
+
// (the SCC resolver in memory-improve.ts) has edges to work on.
|
|
1588
|
+
// Zep arXiv:2501.13956 §3 — unified belief-revision with contradiction edges.
|
|
1589
|
+
const entry = memoryByRef.get(op.ref);
|
|
1590
|
+
const contradictorEntry = memoryByRef.get(op.contradictedByRef);
|
|
1591
|
+
if (!entry) {
|
|
1592
|
+
warnings.push(`Contradict: ${op.ref} not found in loaded memories — skipping.`);
|
|
1593
|
+
// Phantom ref: not in processed, so no skipReason (same rationale as
|
|
1594
|
+
// delete_ref_missing).
|
|
1595
|
+
continue;
|
|
1596
|
+
}
|
|
1597
|
+
if (!contradictorEntry) {
|
|
1598
|
+
warnings.push(`Contradict: ${op.contradictedByRef} not found — skipping.`);
|
|
1599
|
+
// op.ref IS in the batch (entry found above) so the skipReason is
|
|
1600
|
+
// correctly charged against a real processed memory.
|
|
1601
|
+
pushSkipReason("contradict", op.ref, "contradict_target_missing");
|
|
1602
|
+
continue;
|
|
1603
|
+
}
|
|
1604
|
+
try {
|
|
1605
|
+
// Write the contradiction edge: op.ref is contradicted by op.contradictedByRef
|
|
1606
|
+
writeContradictEdge(entry.filePath, op.contradictedByRef);
|
|
1607
|
+
contradicted++;
|
|
1608
|
+
markJournalCompleted(stashDir, op.ref);
|
|
1609
|
+
}
|
|
1610
|
+
catch (e) {
|
|
1611
|
+
warnings.push(`Contradict: failed to write edge for ${op.ref}: ${String(e)}`);
|
|
1612
|
+
pushSkipReason("contradict", op.ref, "contradict_write_failed");
|
|
714
1613
|
}
|
|
715
1614
|
}
|
|
716
1615
|
}
|
|
717
1616
|
cleanupJournal(stashDir, timestamp);
|
|
718
|
-
// TTL cleanup: remove archive entries older than archiveRetentionDays (default 90)
|
|
1617
|
+
// TTL cleanup: remove archive entries older than archiveRetentionDays (default 90).
|
|
1618
|
+
// C-5 / #391: emit an `archive_cleanup` event before each deletion so the
|
|
1619
|
+
// audit trail records what was lost. Outbox pattern (EIP, Hohpe-Woolf) —
|
|
1620
|
+
// any event that is recorded must be queryable; silent deletes are an anti-pattern.
|
|
719
1621
|
const archiveDir = path.join(stashDir, ".akm", "archive");
|
|
720
1622
|
if (fs.existsSync(archiveDir)) {
|
|
721
1623
|
const retentionMs = (config.archiveRetentionDays ?? 90) * 86_400_000;
|
|
@@ -723,8 +1625,20 @@ export async function akmConsolidate(opts = {}) {
|
|
|
723
1625
|
for (const fname of fs.readdirSync(archiveDir)) {
|
|
724
1626
|
const fp = path.join(archiveDir, fname);
|
|
725
1627
|
try {
|
|
726
|
-
|
|
1628
|
+
const stat = fs.statSync(fp);
|
|
1629
|
+
if (stat.mtimeMs < cutoff) {
|
|
1630
|
+
// Emit event before deletion so the record survives the purge.
|
|
1631
|
+
appendEvent({
|
|
1632
|
+
eventType: "archive_cleanup",
|
|
1633
|
+
metadata: {
|
|
1634
|
+
file: fname,
|
|
1635
|
+
filePath: fp,
|
|
1636
|
+
ageMs: Date.now() - stat.mtimeMs,
|
|
1637
|
+
retentionMs,
|
|
1638
|
+
},
|
|
1639
|
+
});
|
|
727
1640
|
fs.unlinkSync(fp);
|
|
1641
|
+
}
|
|
728
1642
|
}
|
|
729
1643
|
catch {
|
|
730
1644
|
/* ignore race conditions */
|
|
@@ -742,57 +1656,465 @@ export async function akmConsolidate(opts = {}) {
|
|
|
742
1656
|
merged,
|
|
743
1657
|
deleted,
|
|
744
1658
|
promoted,
|
|
1659
|
+
contradicted,
|
|
1660
|
+
failedChunks: totalChunksFailed,
|
|
1661
|
+
totalChunks: chunks.length,
|
|
1662
|
+
judgedNoAction,
|
|
1663
|
+
skipReasons,
|
|
1664
|
+
mergedSecondaries,
|
|
1665
|
+
failedChunkMemories,
|
|
745
1666
|
warnings,
|
|
746
1667
|
durationMs: Date.now() - startMs,
|
|
747
1668
|
};
|
|
748
1669
|
}
|
|
749
1670
|
// ── Helpers ─────────────────────────────────────────────────────────────────
|
|
1671
|
+
// ── LLM-output sanitization ─────────────────────────────────────────────────
|
|
1672
|
+
//
|
|
1673
|
+
// Three classes of LLM defect have been observed across hundreds of
|
|
1674
|
+
// consolidate proposals (see audit notes in this branch):
|
|
1675
|
+
//
|
|
1676
|
+
// 1. Code-fence leakage: the entire merged asset is wrapped in
|
|
1677
|
+
// ```markdown … ``` (or ```yaml … ```) despite the prompt forbidding
|
|
1678
|
+
// fences. The post-processor used to pass this through verbatim, so the
|
|
1679
|
+
// first character of the asset content became a backtick rather than
|
|
1680
|
+
// `---`, defeating the frontmatter parser.
|
|
1681
|
+
// 2. YAML quote-escaping bugs: descriptions like `'"Specialty intro...:`
|
|
1682
|
+
// with unbalanced quotes that break the YAML reader. The post-processor
|
|
1683
|
+
// historically passed the LLM's raw scalar straight into a manually
|
|
1684
|
+
// assembled `description: <raw>` line.
|
|
1685
|
+
// 3. Truncated descriptions hitting token cutoffs — the model's max_tokens
|
|
1686
|
+
// runs out mid-sentence, leaving things like
|
|
1687
|
+
// `description: "Tables in narrow column containers need max-width:100% +"`
|
|
1688
|
+
// with no closing context.
|
|
1689
|
+
//
|
|
1690
|
+
// `sanitizeMergedContent` and `validateProposalFrontmatter` defend against
|
|
1691
|
+
// all three at the point where LLM output is consumed.
|
|
1692
|
+
/**
|
|
1693
|
+
* Attempt to recover a frontmatter block that is missing its closing `---`.
|
|
1694
|
+
*
|
|
1695
|
+
* Scans lines after the opening `---` for the first blank line or the first
|
|
1696
|
+
* line that cannot be a YAML scalar (i.e. not a key-value, indented
|
|
1697
|
+
* continuation, comment, or list item). Injects `---` before that line so
|
|
1698
|
+
* the normal parser can proceed.
|
|
1699
|
+
*
|
|
1700
|
+
* Returns the patched string on success, or `null` if the structure is too
|
|
1701
|
+
* ambiguous to recover safely (e.g. no opening `---`, or no body content
|
|
1702
|
+
* found after the frontmatter key-value lines).
|
|
1703
|
+
*/
|
|
1704
|
+
function recoverMalformedFrontmatter(raw) {
|
|
1705
|
+
if (!raw.startsWith("---"))
|
|
1706
|
+
return null;
|
|
1707
|
+
const lines = raw.split(/\r?\n/);
|
|
1708
|
+
// Skip the opening `---` line (index 0).
|
|
1709
|
+
let insertAt = -1;
|
|
1710
|
+
for (let i = 1; i < lines.length; i++) {
|
|
1711
|
+
const line = lines[i];
|
|
1712
|
+
// A blank line marks the end of the frontmatter block in many YAML variants.
|
|
1713
|
+
if (line.trim() === "") {
|
|
1714
|
+
insertAt = i;
|
|
1715
|
+
break;
|
|
1716
|
+
}
|
|
1717
|
+
// A line that is clearly body content: doesn't look like a YAML key, an
|
|
1718
|
+
// indented continuation, a comment, or a sequence item.
|
|
1719
|
+
const isYaml = /^\w[\w-]*\s*:/.test(line) || // key: value
|
|
1720
|
+
/^\s+\S/.test(line) || // indented continuation / nested
|
|
1721
|
+
/^\s*#/.test(line) || // YAML comment
|
|
1722
|
+
/^\s*-\s/.test(line); // sequence item
|
|
1723
|
+
if (!isYaml) {
|
|
1724
|
+
insertAt = i;
|
|
1725
|
+
break;
|
|
1726
|
+
}
|
|
1727
|
+
}
|
|
1728
|
+
if (insertAt < 0)
|
|
1729
|
+
return null;
|
|
1730
|
+
const result = [...lines.slice(0, insertAt), "---", ...lines.slice(insertAt)].join("\n");
|
|
1731
|
+
return result;
|
|
1732
|
+
}
|
|
1733
|
+
/**
|
|
1734
|
+
* Outer-fence stripper specific to consolidate. Unlike the shared
|
|
1735
|
+
* `stripMarkdownFences` helper (which only handles markdown fences), this
|
|
1736
|
+
* variant additionally recognises `yaml` and bare-language fences and refuses
|
|
1737
|
+
* to strip an unbalanced fence — i.e. a leading ``` with no trailing ``` is
|
|
1738
|
+
* treated as a malformed response, not partially sanitized.
|
|
1739
|
+
*
|
|
1740
|
+
* Returns `null` when only one half of a fence pair is present (caller
|
|
1741
|
+
* should reject the response entirely).
|
|
1742
|
+
*/
|
|
1743
|
+
export function stripOuterCodeFence(raw) {
|
|
1744
|
+
const trimmed = raw.trim();
|
|
1745
|
+
const leading = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n/i);
|
|
1746
|
+
const trailing = trimmed.match(/\r?\n```\s*$/);
|
|
1747
|
+
if (!leading && !trailing)
|
|
1748
|
+
return { content: trimmed, stripped: false };
|
|
1749
|
+
if (!leading || !trailing)
|
|
1750
|
+
return null; // unbalanced — refuse
|
|
1751
|
+
const inner = trimmed.slice(leading[0].length, trimmed.length - trailing[0].length).trim();
|
|
1752
|
+
return { content: inner, stripped: true };
|
|
1753
|
+
}
|
|
1754
|
+
export function sanitizeMergedContent(raw) {
|
|
1755
|
+
// Step 1: Strip outer code fence.
|
|
1756
|
+
// Recovery path: if only the leading fence is present, strip it and continue
|
|
1757
|
+
// provided the inner content starts with `---`. Trailing-only fences are NOT
|
|
1758
|
+
// recovered — a trailing ``` is more likely a body code block than a forgotten
|
|
1759
|
+
// wrapper, so recovering would silently corrupt the body.
|
|
1760
|
+
let body;
|
|
1761
|
+
{
|
|
1762
|
+
const fenceResult = stripOuterCodeFence(raw);
|
|
1763
|
+
if (fenceResult) {
|
|
1764
|
+
body = fenceResult.content;
|
|
1765
|
+
}
|
|
1766
|
+
else {
|
|
1767
|
+
const trimmed = raw.trim();
|
|
1768
|
+
const leadingMatch = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n([\s\S]*)$/i);
|
|
1769
|
+
const inner = leadingMatch ? leadingMatch[1].trim() : null;
|
|
1770
|
+
if (!inner?.startsWith("---")) {
|
|
1771
|
+
return { ok: false, reason: "UNBALANCED_CODE_FENCE" };
|
|
1772
|
+
}
|
|
1773
|
+
body = inner;
|
|
1774
|
+
}
|
|
1775
|
+
}
|
|
1776
|
+
// Strip <think> blocks (some local models still emit them despite system prompts).
|
|
1777
|
+
body = body.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
|
|
1778
|
+
// Step 2: Verify frontmatter sentinel.
|
|
1779
|
+
// Recovery path: LLM sometimes emits 1-2 lines of preamble (e.g. "Here is the
|
|
1780
|
+
// merged content:") before the `---`. Accept if `---` appears within 300 chars.
|
|
1781
|
+
// Beyond that it's more likely a body section divider, not a frontmatter start.
|
|
1782
|
+
if (!body.startsWith("---")) {
|
|
1783
|
+
const nlIdx = body.indexOf("\n---");
|
|
1784
|
+
if (nlIdx >= 0 && nlIdx < 300) {
|
|
1785
|
+
body = body.slice(nlIdx + 1);
|
|
1786
|
+
}
|
|
1787
|
+
else {
|
|
1788
|
+
return { ok: false, reason: "MISSING_FRONTMATTER_SENTINEL" };
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
// Extract frontmatter block.
|
|
1792
|
+
// Recovery path: LLM sometimes omits the closing `---` delimiter. Detect this
|
|
1793
|
+
// by scanning lines after the opening `---` for the first blank line or the
|
|
1794
|
+
// first line that isn't a YAML key-value pair, then inject `---` there.
|
|
1795
|
+
let match = body.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
|
|
1796
|
+
if (!match) {
|
|
1797
|
+
const recovered = recoverMalformedFrontmatter(body);
|
|
1798
|
+
if (recovered) {
|
|
1799
|
+
match = recovered.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
|
|
1800
|
+
}
|
|
1801
|
+
if (!match) {
|
|
1802
|
+
return { ok: false, reason: "MALFORMED_FRONTMATTER_BLOCK" };
|
|
1803
|
+
}
|
|
1804
|
+
}
|
|
1805
|
+
// Re-parse via the yaml library so any quote-escaping mistakes either get
|
|
1806
|
+
// normalised or surface as a parse error we can reject.
|
|
1807
|
+
// Recovery: if the strict yaml library fails, fall back to the lenient
|
|
1808
|
+
// hand-rolled parseFrontmatter parser, which tolerates common LLM YAML
|
|
1809
|
+
// quirks (unescaped special chars, bare scalars, etc.). If it recovers
|
|
1810
|
+
// at least one key, proceed — yamlStringify below will re-serialize
|
|
1811
|
+
// cleanly. Only reject if both parsers fail to extract any data.
|
|
1812
|
+
let parsedFm;
|
|
1813
|
+
try {
|
|
1814
|
+
parsedFm = yamlParse(match[1]);
|
|
1815
|
+
}
|
|
1816
|
+
catch (e) {
|
|
1817
|
+
const fallback = parseFrontmatter(`---\n${match[1]}\n---\n${match[2]}`);
|
|
1818
|
+
if (fallback.frontmatter !== null && Object.keys(fallback.data).length > 0) {
|
|
1819
|
+
parsedFm = fallback.data;
|
|
1820
|
+
}
|
|
1821
|
+
else {
|
|
1822
|
+
return { ok: false, reason: `INVALID_YAML: ${e instanceof Error ? e.message : String(e)}` };
|
|
1823
|
+
}
|
|
1824
|
+
}
|
|
1825
|
+
if (parsedFm === null || typeof parsedFm !== "object" || Array.isArray(parsedFm)) {
|
|
1826
|
+
return { ok: false, reason: "FRONTMATTER_NOT_OBJECT" };
|
|
1827
|
+
}
|
|
1828
|
+
const fm = parsedFm;
|
|
1829
|
+
// Normalise placeholder leaks like `updated: today`, `updated: {today: null}`,
|
|
1830
|
+
// `updated: now`, etc. The consolidate prompt instructs the LLM not to emit
|
|
1831
|
+
// these, but small models still do. Replace any such leak with today's ISO
|
|
1832
|
+
// date OR drop the field if we can't safely normalise it.
|
|
1833
|
+
normalizeUpdatedField(fm);
|
|
1834
|
+
// Re-serialise via yaml.stringify to fix any quoting quirks.
|
|
1835
|
+
let serialized;
|
|
1836
|
+
try {
|
|
1837
|
+
serialized = yamlStringify(fm).trimEnd();
|
|
1838
|
+
}
|
|
1839
|
+
catch (e) {
|
|
1840
|
+
return { ok: false, reason: `YAML_STRINGIFY_FAILED: ${e instanceof Error ? e.message : String(e)}` };
|
|
1841
|
+
}
|
|
1842
|
+
const cleaned = assembleAssetFromString(serialized, match[2]);
|
|
1843
|
+
return { ok: true, result: { content: cleaned, frontmatter: fm } };
|
|
1844
|
+
}
|
|
1845
|
+
/**
|
|
1846
|
+
* Mutate `fm.updated` in place to normalise placeholder leaks emitted by the
|
|
1847
|
+
* LLM. The consolidate prompt forbids these, but small models still produce
|
|
1848
|
+
* literal `today` / `{today: null}` / `now` values.
|
|
1849
|
+
*
|
|
1850
|
+
* Rules:
|
|
1851
|
+
* - A real ISO-style date string (YYYY-MM-DD, optionally with time) stays as-is.
|
|
1852
|
+
* - A Date object (some YAML parsers materialise dates) is converted to its
|
|
1853
|
+
* ISO yyyy-mm-dd form.
|
|
1854
|
+
* - A placeholder string ("today", "now", "{today}", "${today}", template
|
|
1855
|
+
* variables) is replaced with today's ISO date.
|
|
1856
|
+
* - A map/object (e.g. `{today: null}`) is replaced with today's ISO date.
|
|
1857
|
+
* - `null`, empty string, missing → left alone (no field added; reviewers
|
|
1858
|
+
* should not silently gain metadata they didn't write).
|
|
1859
|
+
*
|
|
1860
|
+
* Exported for unit testing.
|
|
1861
|
+
*/
|
|
1862
|
+
export function normalizeUpdatedField(fm) {
|
|
1863
|
+
if (!("updated" in fm))
|
|
1864
|
+
return;
|
|
1865
|
+
const v = fm.updated;
|
|
1866
|
+
if (v === null || v === undefined || v === "")
|
|
1867
|
+
return;
|
|
1868
|
+
const todayIso = new Date().toISOString().slice(0, 10);
|
|
1869
|
+
if (v instanceof Date) {
|
|
1870
|
+
fm.updated = v.toISOString().slice(0, 10);
|
|
1871
|
+
return;
|
|
1872
|
+
}
|
|
1873
|
+
if (typeof v === "string") {
|
|
1874
|
+
const trimmed = v.trim().toLowerCase();
|
|
1875
|
+
if (/^\d{4}-\d{2}-\d{2}/.test(v.trim()))
|
|
1876
|
+
return; // already a real date
|
|
1877
|
+
if (trimmed === "today" ||
|
|
1878
|
+
trimmed === "now" ||
|
|
1879
|
+
trimmed === "{today}" ||
|
|
1880
|
+
// biome-ignore lint/suspicious/noTemplateCurlyInString: matches the literal user-typed placeholder text "${today}" so we can normalize it to today's ISO date
|
|
1881
|
+
trimmed === "${today}" ||
|
|
1882
|
+
trimmed === "{{today}}" ||
|
|
1883
|
+
/^\{?\s*today\s*\}?$/.test(trimmed)) {
|
|
1884
|
+
fm.updated = todayIso;
|
|
1885
|
+
return;
|
|
1886
|
+
}
|
|
1887
|
+
// Unknown string format — leave alone so it's visible in the diff.
|
|
1888
|
+
return;
|
|
1889
|
+
}
|
|
1890
|
+
if (typeof v === "object") {
|
|
1891
|
+
// Maps like `{today: null}`, `{now: null}` — clearly a template leak.
|
|
1892
|
+
fm.updated = todayIso;
|
|
1893
|
+
return;
|
|
1894
|
+
}
|
|
1895
|
+
}
|
|
1896
|
+
/**
|
|
1897
|
+
* Normalise a knowledge slug for variant-aware deduplication. Collapses:
|
|
1898
|
+
* - date suffixes (`-may-2026`, `-2026-05-03`, `-2026`)
|
|
1899
|
+
* - numeric counter suffixes (`-2`, `-3`)
|
|
1900
|
+
* - trailing -patterns / -2026-05-03 styles
|
|
1901
|
+
* - word reorderings via alphabetical sort of the remaining tokens.
|
|
1902
|
+
*
|
|
1903
|
+
* Two slugs that normalise to the same string are considered the same asset
|
|
1904
|
+
* for dedup purposes even if they don't share an exact ref.
|
|
1905
|
+
*/
|
|
1906
|
+
export function normalizeSlugForDedup(ref) {
|
|
1907
|
+
const slug = ref.replace(/^[^:]+:/, "");
|
|
1908
|
+
const monthRe = /(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)/i;
|
|
1909
|
+
const tokens = slug
|
|
1910
|
+
.toLowerCase()
|
|
1911
|
+
.split("-")
|
|
1912
|
+
.filter((tok) => tok.length > 0)
|
|
1913
|
+
// Strip purely-numeric tokens (years, dates, counter suffixes like -2 / -3).
|
|
1914
|
+
// Numbers carry no semantic information for our dedup purposes — every
|
|
1915
|
+
// observed defective slug variant differs only in dates or counters.
|
|
1916
|
+
.filter((tok) => !/^\d+$/.test(tok))
|
|
1917
|
+
.filter((tok) => !monthRe.test(tok));
|
|
1918
|
+
// Sort to absorb word reorderings.
|
|
1919
|
+
tokens.sort();
|
|
1920
|
+
return tokens.join("-");
|
|
1921
|
+
}
|
|
1922
|
+
/**
|
|
1923
|
+
* Pre-emit dedup check: compare the candidate ref against pending consolidate
|
|
1924
|
+
* proposals only. Returns a reason string if a slug-variant match is found,
|
|
1925
|
+
* else null.
|
|
1926
|
+
*
|
|
1927
|
+
* Historical context (REMOVED 2026-05-20): this function previously also ran
|
|
1928
|
+
* (a) a normalised-slug match against existing knowledge AND memory entries
|
|
1929
|
+
* in the DB, and
|
|
1930
|
+
* (b) an embedding cosine-similarity check (>= 0.85) against ALL knowledge
|
|
1931
|
+
* and non-derived memory entries.
|
|
1932
|
+
* Both branches had ZERO observed fires across 30 sampled runs in the
|
|
1933
|
+
* post-fix window. The 29 actual dedup catches all came from the SEPARATE
|
|
1934
|
+
* content-hash dedup inside `mergePlans` (the older SHA-256 helper). The
|
|
1935
|
+
* embedding branch in particular had unbounded cost per promote (embedded
|
|
1936
|
+
* every knowledge + non-derived memory entry, every time) with no observed
|
|
1937
|
+
* benefit. Empirical signal → deleted.
|
|
1938
|
+
*
|
|
1939
|
+
* What remains: a check against pending consolidate proposals in the SAME
|
|
1940
|
+
* improve run. This catches duplicates queued back-to-back within a single
|
|
1941
|
+
* improve invocation — a different concern from the cross-run content-hash
|
|
1942
|
+
* dedup, and cheap (no embeddings, no DB query).
|
|
1943
|
+
*/
|
|
1944
|
+
export async function checkPreEmitDedup(opts) {
|
|
1945
|
+
const normCandidate = normalizeSlugForDedup(opts.candidateRef);
|
|
1946
|
+
// Pending consolidate proposals (slug match) — within the same improve run.
|
|
1947
|
+
const pendingConsolidate = listProposals(opts.stashDir, { status: "pending" }).filter((p) => p.source === "consolidate");
|
|
1948
|
+
for (const p of pendingConsolidate) {
|
|
1949
|
+
if (normalizeSlugForDedup(p.ref) === normCandidate) {
|
|
1950
|
+
return { duplicate: true, reason: `slug-variant of pending proposal ${p.id} (${p.ref})` };
|
|
1951
|
+
}
|
|
1952
|
+
}
|
|
1953
|
+
return { duplicate: false };
|
|
1954
|
+
}
|
|
1955
|
+
/**
|
|
1956
|
+
* Incremental candidate set: {changed} ∪ {top-k persisted-vector neighbours of
|
|
1957
|
+
* each changed memory}, intersected with the loaded pool. Returns [] when
|
|
1958
|
+
* nothing changed (caller emits a no-op envelope), the full pool when
|
|
1959
|
+
* everything changed or the index can't answer (fail-open to preserve merge
|
|
1960
|
+
* correctness). `since` is an ISO timestamp.
|
|
1961
|
+
*/
|
|
1962
|
+
export function narrowToIncrementalCandidates(memories, since, warnings) {
|
|
1963
|
+
const isChanged = (m) => {
|
|
1964
|
+
try {
|
|
1965
|
+
return fs.statSync(m.filePath).mtime.toISOString() > since;
|
|
1966
|
+
}
|
|
1967
|
+
catch {
|
|
1968
|
+
return true; // never silently drop a memory we cannot stat
|
|
1969
|
+
}
|
|
1970
|
+
};
|
|
1971
|
+
const changed = memories.filter(isChanged);
|
|
1972
|
+
if (changed.length === 0)
|
|
1973
|
+
return [];
|
|
1974
|
+
if (changed.length === memories.length)
|
|
1975
|
+
return memories;
|
|
1976
|
+
const NEIGHBORS_PER_CHANGED = 5;
|
|
1977
|
+
const byName = new Map(memories.map((m) => [m.name, m]));
|
|
1978
|
+
const keep = new Set(changed.map((m) => m.name));
|
|
1979
|
+
let db;
|
|
1980
|
+
try {
|
|
1981
|
+
db = openExistingDatabase();
|
|
1982
|
+
for (const m of changed) {
|
|
1983
|
+
const id = findEntryIdByRef(db, `memory:${m.name}`);
|
|
1984
|
+
if (id === undefined)
|
|
1985
|
+
continue;
|
|
1986
|
+
for (const hit of getNeighborsByEntryId(db, id, NEIGHBORS_PER_CHANGED + 1)) {
|
|
1987
|
+
if (hit.id === id)
|
|
1988
|
+
continue;
|
|
1989
|
+
const entry = getEntryById(db, hit.id);
|
|
1990
|
+
if (!entry)
|
|
1991
|
+
continue;
|
|
1992
|
+
const name = entry.entry.name;
|
|
1993
|
+
if (byName.has(name))
|
|
1994
|
+
keep.add(name); // only neighbours present in the loaded pool
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
}
|
|
1998
|
+
catch {
|
|
1999
|
+
warnings.push("Incremental consolidation: index unavailable — processing full pool.");
|
|
2000
|
+
return memories;
|
|
2001
|
+
}
|
|
2002
|
+
finally {
|
|
2003
|
+
if (db)
|
|
2004
|
+
closeDatabase(db);
|
|
2005
|
+
}
|
|
2006
|
+
const candidates = memories.filter((m) => keep.has(m.name));
|
|
2007
|
+
warnings.push(`Incremental consolidation: ${changed.length} changed + neighbours → ${candidates.length}/${memories.length} memories considered (since ${since}).`);
|
|
2008
|
+
return candidates;
|
|
2009
|
+
}
|
|
750
2010
|
function loadMemoriesForSource(source, stashDir, warnings) {
|
|
751
|
-
|
|
2011
|
+
// Load from DB first
|
|
2012
|
+
let memories = [];
|
|
2013
|
+
let db;
|
|
2014
|
+
try {
|
|
2015
|
+
db = openExistingDatabase();
|
|
2016
|
+
const entries = getAllEntries(db, "memory");
|
|
2017
|
+
memories = entries
|
|
2018
|
+
.filter((e) => {
|
|
2019
|
+
if (!source)
|
|
2020
|
+
return true;
|
|
2021
|
+
return path.resolve(e.stashDir) === path.resolve(source);
|
|
2022
|
+
})
|
|
2023
|
+
.filter((e) => isConsolidationEligibleMemoryName(e.entry.name))
|
|
2024
|
+
// Skip stale DB entries whose file was deleted by a prior run but not yet
|
|
2025
|
+
// re-indexed. Without this guard the deleted file's ref appears in chunks
|
|
2026
|
+
// sent to the LLM, which then proposes a second delete → delete_failed
|
|
2027
|
+
// because the file is already gone. Re-indexing runs on a cron cadence so
|
|
2028
|
+
// several successful deletes can accumulate before the DB catches up.
|
|
2029
|
+
.filter((e) => fs.existsSync(e.filePath))
|
|
2030
|
+
.map((e) => ({
|
|
2031
|
+
name: e.entry.name,
|
|
2032
|
+
filePath: e.filePath,
|
|
2033
|
+
description: e.entry.description ?? "",
|
|
2034
|
+
tags: e.entry.tags ?? [],
|
|
2035
|
+
stashDir: e.stashDir,
|
|
2036
|
+
}));
|
|
2037
|
+
}
|
|
2038
|
+
catch {
|
|
2039
|
+
memories = [];
|
|
2040
|
+
}
|
|
2041
|
+
finally {
|
|
2042
|
+
if (db)
|
|
2043
|
+
closeDatabase(db);
|
|
2044
|
+
}
|
|
752
2045
|
if (memories.length === 0) {
|
|
753
2046
|
// DB fallback: walk filesystem
|
|
754
2047
|
const memoriesDir = path.join(source ?? stashDir, "memories");
|
|
755
|
-
|
|
2048
|
+
const fsStashDir = source ?? stashDir;
|
|
2049
|
+
if (fs.existsSync(memoriesDir)) {
|
|
2050
|
+
for (const fname of fs.readdirSync(memoriesDir)) {
|
|
2051
|
+
if (!fname.endsWith(".md"))
|
|
2052
|
+
continue;
|
|
2053
|
+
const filePath = path.join(memoriesDir, fname);
|
|
2054
|
+
const name = fname.replace(/\.md$/, "");
|
|
2055
|
+
if (!isConsolidationEligibleMemoryName(name))
|
|
2056
|
+
continue;
|
|
2057
|
+
memories.push({ name, filePath, description: "", tags: [], stashDir: fsStashDir });
|
|
2058
|
+
}
|
|
2059
|
+
}
|
|
756
2060
|
if (memories.length > 0) {
|
|
757
2061
|
warnings.push("DB not found or empty — loaded memories directly from filesystem.");
|
|
758
2062
|
}
|
|
759
2063
|
}
|
|
760
2064
|
return memories;
|
|
761
2065
|
}
|
|
762
|
-
function
|
|
763
|
-
// If it looks like an absolute path, use directly
|
|
764
|
-
if (path.isAbsolute(sourceName))
|
|
765
|
-
return sourceName;
|
|
766
|
-
return sourceName;
|
|
767
|
-
}
|
|
768
|
-
async function generateMergedContent(config, primaryRef, primaryBody, secondaryRefs, memoryByRef, warnings) {
|
|
2066
|
+
async function generateMergedContent(config, primaryRef, primaryBody, secondaryRefs, memoryByRef) {
|
|
769
2067
|
// Only handle single-secondary merges per design (one call per merge op)
|
|
770
2068
|
const secRef = secondaryRefs[0];
|
|
771
2069
|
const secEntry = memoryByRef.get(secRef);
|
|
772
2070
|
if (!secEntry)
|
|
773
|
-
return
|
|
2071
|
+
return { error: "merge_read_failed", detail: `secondary ${secRef} not in memoryByRef` };
|
|
774
2072
|
let secBody = "";
|
|
775
2073
|
try {
|
|
776
2074
|
secBody = fs.readFileSync(secEntry.filePath, "utf8");
|
|
777
2075
|
}
|
|
778
2076
|
catch {
|
|
779
|
-
|
|
780
|
-
return null;
|
|
2077
|
+
return { error: "merge_read_failed", detail: `could not read secondary ${secRef}` };
|
|
781
2078
|
}
|
|
2079
|
+
const primaryFmKeys = Object.keys(parseFrontmatter(primaryBody).data);
|
|
2080
|
+
const secFmKeys = Object.keys(parseFrontmatter(secBody).data);
|
|
2081
|
+
const requiredFmKeys = [...new Set([...primaryFmKeys, ...secFmKeys])];
|
|
782
2082
|
const prompt = [
|
|
783
2083
|
"Merge these two memory assets into one. Output ONLY the merged markdown (with YAML frontmatter). Do not explain, do not use code fences.",
|
|
784
2084
|
"",
|
|
2085
|
+
"## OUTPUT FORMAT (MANDATORY)",
|
|
2086
|
+
"Return raw markdown content beginning DIRECTLY with the `---` frontmatter delimiter.",
|
|
2087
|
+
"DO NOT wrap your entire response in a code fence.",
|
|
2088
|
+
"",
|
|
2089
|
+
'GOOD: "---\\ndescription: ...\\n---\\nBody content."',
|
|
2090
|
+
'BAD: "```markdown\\n---\\ndescription: ...\\n---\\nBody content.\\n```"',
|
|
2091
|
+
'BAD: "```yaml\\n---\\ndescription: ...\\n---\\nBody content.\\n```"',
|
|
2092
|
+
"",
|
|
2093
|
+
"## FRONTMATTER RULES (MANDATORY)",
|
|
2094
|
+
"- The `updated:` field, if present, MUST be a real ISO date (e.g. `updated: 2026-05-20`). NEVER emit `updated: today`, `updated: now`, or `updated: {today: null}`. If you don't have a real date, OMIT the field — the post-processor will not invent one.",
|
|
2095
|
+
"- REQUIRED: The merged frontmatter MUST include a `description` field with a concise one-sentence summary of the merged asset's content. If neither source has a `description` field, synthesize one from the content.",
|
|
2096
|
+
requiredFmKeys.length > 0
|
|
2097
|
+
? `- CRITICAL: The merged frontmatter MUST include ALL of these keys from both source memories: ${requiredFmKeys.join(", ")}. Do NOT drop any of them.`
|
|
2098
|
+
: null,
|
|
2099
|
+
"",
|
|
785
2100
|
`=== Primary memory (${primaryRef}) ===`,
|
|
786
2101
|
primaryBody,
|
|
787
2102
|
"",
|
|
788
2103
|
`=== Secondary memory (${secRef}) ===`,
|
|
789
2104
|
secBody,
|
|
790
|
-
]
|
|
2105
|
+
]
|
|
2106
|
+
.filter((line) => line !== null)
|
|
2107
|
+
.join("\n");
|
|
2108
|
+
// Use the same per-process profile resolution as the chunk-plan call above
|
|
2109
|
+
// so the merge generation step doesn't silently revert to the default LLM.
|
|
2110
|
+
const llmConfig = resolveConsolidateLlmConfig(config);
|
|
791
2111
|
const result = await tryLlmFeature("memory_consolidation", config, async () => {
|
|
792
|
-
if (!
|
|
2112
|
+
if (!llmConfig)
|
|
793
2113
|
return { ok: false, error: "No LLM configured for consolidation" };
|
|
794
2114
|
try {
|
|
795
|
-
const content = await chatCompletion(
|
|
2115
|
+
const content = await chatCompletion(llmConfig, [{ role: "user", content: prompt }], {
|
|
2116
|
+
enableThinking: false,
|
|
2117
|
+
});
|
|
796
2118
|
return { ok: true, content };
|
|
797
2119
|
}
|
|
798
2120
|
catch (e) {
|
|
@@ -800,10 +2122,77 @@ async function generateMergedContent(config, primaryRef, primaryBody, secondaryR
|
|
|
800
2122
|
}
|
|
801
2123
|
}, { ok: false, error: `merge content generation failed for ${primaryRef}` });
|
|
802
2124
|
if (!result.ok) {
|
|
803
|
-
|
|
804
|
-
|
|
2125
|
+
return {
|
|
2126
|
+
error: "merge_transport_failed",
|
|
2127
|
+
detail: result.error ?? `merge content generation failed for ${primaryRef}`,
|
|
2128
|
+
};
|
|
2129
|
+
}
|
|
2130
|
+
// Sanitize LLM output: strip outer code fences (defends against the
|
|
2131
|
+
// ```markdown … ``` leak observed in production), re-serialise frontmatter
|
|
2132
|
+
// through the yaml lib (fixes quote-escaping mistakes), and reject empty
|
|
2133
|
+
// or fence-only responses.
|
|
2134
|
+
const sanitized = sanitizeMergedContent(result.content ?? "");
|
|
2135
|
+
if (!sanitized.ok) {
|
|
2136
|
+
const reason = sanitized.reason;
|
|
2137
|
+
const isFenceError = reason === "UNBALANCED_CODE_FENCE" ||
|
|
2138
|
+
reason === "MISSING_FRONTMATTER_SENTINEL" ||
|
|
2139
|
+
reason === "MALFORMED_FRONTMATTER_BLOCK" ||
|
|
2140
|
+
reason === "FRONTMATTER_NOT_OBJECT";
|
|
2141
|
+
const mergeReason = isFenceError ? "merge_fence_rejected" : "merge_yaml_invalid";
|
|
2142
|
+
return { error: mergeReason, detail: `${primaryRef} — ${reason}` };
|
|
2143
|
+
}
|
|
2144
|
+
const mergedRaw = sanitized.result.content;
|
|
2145
|
+
// C-4 / #383: Content-preservation lint (mem0 §3.2, arXiv:2504.19413).
|
|
2146
|
+
// Guards against LLM-generated merged content that silently drops information
|
|
2147
|
+
// from the source assets. Two checks:
|
|
2148
|
+
// 1. Body size: merged body must be >= 50% of the larger source body.
|
|
2149
|
+
// 2. Frontmatter superset: merged frontmatter must contain all keys present
|
|
2150
|
+
// in both source frontmatters.
|
|
2151
|
+
// Failures return a discriminated error so the call site can emit a specific
|
|
2152
|
+
// skip-reason key in the histogram.
|
|
2153
|
+
try {
|
|
2154
|
+
const primaryFm = parseFrontmatter(primaryBody);
|
|
2155
|
+
const secFm = parseFrontmatter(secBody);
|
|
2156
|
+
const mergedFm = parseFrontmatter(mergedRaw);
|
|
2157
|
+
// Check body size — blended floor: max(ratio × largerLen, absoluteFloor).
|
|
2158
|
+
// Deduplication is expected, so the ratio is lower than the reflect gate
|
|
2159
|
+
// (0.3 vs 0.5). The absolute floor protects very short memory pairs where
|
|
2160
|
+
// the ratio alone would produce a near-zero threshold.
|
|
2161
|
+
const primaryBodyLen = (primaryFm.content ?? "").trim().length;
|
|
2162
|
+
const secBodyLen = (secFm.content ?? "").trim().length;
|
|
2163
|
+
const mergedBodyLen = (mergedFm.content ?? "").trim().length;
|
|
2164
|
+
const largerBodyLen = Math.max(primaryBodyLen, secBodyLen);
|
|
2165
|
+
const mergeFloor = Math.max(MERGE_SHRINK_RATIO_MIN * largerBodyLen, MERGE_ABSOLUTE_FLOOR_CHARS);
|
|
2166
|
+
if (largerBodyLen > 0 && mergedBodyLen < mergeFloor) {
|
|
2167
|
+
return {
|
|
2168
|
+
error: "merge_content_too_short",
|
|
2169
|
+
detail: `${primaryRef} — merged body (${mergedBodyLen} chars) is less than floor (${Math.round(mergeFloor)} chars; max(${MERGE_SHRINK_RATIO_MIN}×${largerBodyLen}, ${MERGE_ABSOLUTE_FLOOR_CHARS}))`,
|
|
2170
|
+
};
|
|
2171
|
+
}
|
|
2172
|
+
// Check frontmatter superset — attempt repair before rejecting.
|
|
2173
|
+
const primaryKeys = Object.keys(primaryFm.data ?? {});
|
|
2174
|
+
const secKeys = Object.keys(secFm.data ?? {});
|
|
2175
|
+
const mergedKeys = new Set(Object.keys(mergedFm.data ?? {}));
|
|
2176
|
+
const missingKeys = [...new Set([...primaryKeys, ...secKeys])].filter((k) => !mergedKeys.has(k));
|
|
2177
|
+
if (missingKeys.length > 0) {
|
|
2178
|
+
// Inject missing keys from source FMs. Primary value wins on conflict.
|
|
2179
|
+
const repairedFmData = { ...mergedFm.data };
|
|
2180
|
+
for (const key of missingKeys) {
|
|
2181
|
+
repairedFmData[key] =
|
|
2182
|
+
key in primaryFm.data
|
|
2183
|
+
? primaryFm.data[key]
|
|
2184
|
+
: secFm.data[key];
|
|
2185
|
+
}
|
|
2186
|
+
normalizeUpdatedField(repairedFmData);
|
|
2187
|
+
const repairedYaml = yamlStringify(repairedFmData).trimEnd();
|
|
2188
|
+
const bodyPart = mergedFm.content ?? "";
|
|
2189
|
+
return { content: `---\n${repairedYaml}\n---\n${bodyPart}` };
|
|
2190
|
+
}
|
|
2191
|
+
}
|
|
2192
|
+
catch {
|
|
2193
|
+
// parseFrontmatter failures are non-fatal — allow the merge to proceed.
|
|
805
2194
|
}
|
|
806
|
-
return
|
|
2195
|
+
return { content: mergedRaw };
|
|
807
2196
|
}
|
|
808
2197
|
async function promptConfirm(message) {
|
|
809
2198
|
process.stdout.write(message);
|