akm-cli 0.9.0-rc.2 → 0.9.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -12
- package/README.md +7 -8
- package/SECURITY.md +5 -3
- package/dist/akm +44 -33
- package/dist/akm-migrate-storage +44 -33
- package/dist/assets/backends/schtasks-template.xml +2 -1
- package/dist/assets/improve-strategies/catchup.json +3 -1
- package/dist/assets/improve-strategies/consolidate.json +3 -1
- package/dist/assets/improve-strategies/frequent.json +3 -1
- package/dist/assets/improve-strategies/graph-refresh.json +3 -1
- package/dist/assets/improve-strategies/memory-focus.json +4 -1
- package/dist/assets/improve-strategies/proactive-maintenance.json +1 -0
- package/dist/assets/improve-strategies/quick.json +3 -1
- package/dist/assets/improve-strategies/recombine-only.json +2 -0
- package/dist/assets/improve-strategies/reflect-distill.json +1 -0
- package/dist/assets/improve-strategies/synthesize.json +2 -0
- package/dist/assets/tasks/core/backup.yml +2 -2
- package/dist/cli/config-migrate.js +554 -26
- package/dist/cli.js +28 -11
- package/dist/commands/backup-cli.js +6 -4
- package/dist/commands/config-cli.js +10 -3
- package/dist/commands/feedback-cli.js +24 -7
- package/dist/commands/health/checks.js +94 -15
- package/dist/commands/health/html-report.js +23 -2
- package/dist/commands/health/improve-metrics.js +45 -6
- package/dist/commands/health/md-report.js +10 -1
- package/dist/commands/health/surfaces.js +2 -1
- package/dist/commands/health/windows.js +1 -1
- package/dist/commands/health.js +1 -1
- package/dist/commands/improve/anti-collapse.js +3 -3
- package/dist/commands/improve/collapse-detector.js +5 -5
- package/dist/commands/improve/consolidate.js +123 -66
- package/dist/commands/improve/distill/promote-memory.js +8 -6
- package/dist/commands/improve/distill/quality-gate.js +1 -1
- package/dist/commands/improve/distill-guards.js +1 -1
- package/dist/commands/improve/distill-promotion-policy.js +32 -26
- package/dist/commands/improve/distill.js +52 -36
- package/dist/commands/improve/eligibility.js +9 -8
- package/dist/commands/improve/extract-prompt.js +2 -2
- package/dist/commands/improve/extract.js +43 -11
- package/dist/commands/improve/improve-auto-accept.js +14 -28
- package/dist/commands/improve/improve-cli.js +4 -2
- package/dist/commands/improve/improve-strategies.js +2 -1
- package/dist/commands/improve/improve.js +49 -8
- package/dist/commands/improve/loop-stages.js +11 -6
- package/dist/commands/improve/preparation.js +51 -17
- package/dist/commands/improve/procedural.js +10 -6
- package/dist/commands/improve/recombine.js +31 -7
- package/dist/commands/improve/reflect.js +24 -14
- package/dist/commands/improve/salience.js +5 -1
- package/dist/commands/improve/source-identity.js +43 -0
- package/dist/commands/migrate-cli.js +36 -0
- package/dist/commands/mv-cli.js +647 -258
- package/dist/commands/proposal/drain.js +16 -4
- package/dist/commands/proposal/proposal-cli.js +3 -2
- package/dist/commands/proposal/proposal.js +16 -55
- package/dist/commands/proposal/repository.js +664 -58
- package/dist/commands/read/curate.js +4 -2
- package/dist/commands/read/knowledge.js +4 -1
- package/dist/commands/read/search.js +6 -2
- package/dist/commands/read/show.js +8 -7
- package/dist/commands/sources/self-update.js +156 -112
- package/dist/commands/sources/sources-cli.js +7 -2
- package/dist/commands/tasks/default-tasks.js +5 -5
- package/dist/commands/tasks/tasks-cli.js +7 -3
- package/dist/commands/tasks/tasks.js +261 -68
- package/dist/core/common.js +36 -3
- package/dist/core/config/config-io.js +2 -2
- package/dist/core/config/config-schema.js +15 -3
- package/dist/core/config/config.js +14 -13
- package/dist/core/file-lock.js +2 -2
- package/dist/core/improve-result.js +87 -7
- package/dist/core/migration-backup.js +816 -171
- package/dist/core/migration-operation.js +44 -0
- package/dist/core/state/migrations.js +4 -7
- package/dist/core/state-db.js +23 -7
- package/dist/core/write-source.js +86 -18
- package/dist/indexer/db/db.js +109 -53
- package/dist/indexer/index-writer-lock.js +38 -37
- package/dist/indexer/index-written-assets.js +73 -56
- package/dist/indexer/indexer.js +5 -1
- package/dist/indexer/search/search-source.js +2 -2
- package/dist/indexer/usage/usage-events.js +8 -2
- package/dist/integrations/agent/engine-resolution.js +14 -7
- package/dist/scripts/migrate-storage.js +887 -992
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +494 -574
- package/dist/setup/detected-engines.js +7 -13
- package/dist/setup/engine-config.js +14 -4
- package/dist/setup/setup.js +1 -1
- package/dist/setup/steps/connection.js +1 -1
- package/dist/setup/steps/tasks.js +1 -1
- package/dist/sources/providers/git-stash.js +58 -21
- package/dist/sources/providers/git.js +1 -1
- package/dist/storage/engines/sqlite-migrations.js +138 -3
- package/dist/storage/repositories/events-repository.js +24 -0
- package/dist/storage/repositories/proposals-repository.js +14 -0
- package/dist/storage/repositories/task-history-repository.js +30 -3
- package/dist/tasks/backends/cron.js +71 -38
- package/dist/tasks/backends/exec-utils.js +55 -3
- package/dist/tasks/backends/launchd.js +241 -50
- package/dist/tasks/backends/schtasks.js +434 -59
- package/dist/tasks/command-executable.js +93 -0
- package/dist/tasks/embedded.js +2 -0
- package/dist/tasks/parser.js +142 -6
- package/dist/tasks/resolve-akm-bin.js +19 -4
- package/dist/tasks/runner.js +258 -93
- package/dist/tasks/schedule.js +108 -19
- package/dist/tasks/scheduler-invocation.js +86 -0
- package/dist/tasks/task-id.js +37 -0
- package/dist/workflows/db.js +24 -13
- package/dist/workflows/validator.js +24 -2
- package/docs/README.md +103 -0
- package/docs/migration/release-notes/0.9.0.md +28 -9
- package/docs/migration/v0.8-to-v0.9.md +186 -26
- package/package.json +2 -3
|
@@ -146,7 +146,7 @@ export function buildWindowMetrics(db, stateDbPath, since, until, now = () => Da
|
|
|
146
146
|
});
|
|
147
147
|
const { withLogs: taskRowsWithLogs, backed: existingLogRows } = partitionLogBackedRows(taskRows, logsDb);
|
|
148
148
|
const failedTaskRows = taskRows.filter((row) => row.status === "failed");
|
|
149
|
-
const activeRows = taskRows.filter((row) => row.status === "active");
|
|
149
|
+
const activeRows = taskRows.filter((row) => row.status === "active" && row.completed_at === null);
|
|
150
150
|
const stuckActiveRuns = activeRows.filter((row) => now() - new Date(row.started_at).getTime() > ACTIVE_RUN_WARN_MS).length;
|
|
151
151
|
const promptRows = taskRows.filter((row) => row.target_kind === "prompt");
|
|
152
152
|
const promptFailures = promptRows.filter((row) => {
|
package/dist/commands/health.js
CHANGED
|
@@ -94,7 +94,7 @@ export function akmHealth(options = {}) {
|
|
|
94
94
|
const taskRows = queryTaskHistory(db, { since });
|
|
95
95
|
const { withLogs: taskRowsWithLogs, backed: existingLogRows } = partitionLogBackedRows(taskRows, logsDb);
|
|
96
96
|
const failedTaskRows = taskRows.filter((row) => row.status === "failed");
|
|
97
|
-
const activeRows = taskRows.filter((row) => row.status === "active");
|
|
97
|
+
const activeRows = taskRows.filter((row) => row.status === "active" && row.completed_at === null);
|
|
98
98
|
const stuckActiveRuns = activeRows.filter((row) => now() - new Date(row.started_at).getTime() > ACTIVE_RUN_WARN_MS).length;
|
|
99
99
|
const promptRows = taskRows.filter((row) => row.target_kind === "prompt");
|
|
100
100
|
const promptFailures = promptRows.filter((row) => {
|
|
@@ -72,8 +72,8 @@ function distinctTokens(text) {
|
|
|
72
72
|
}
|
|
73
73
|
/**
|
|
74
74
|
* A merge must strictly increase information (R5 §4.2):
|
|
75
|
-
* 1. Provenance: the merged asset's `
|
|
76
|
-
*
|
|
75
|
+
* 1. Provenance: the merged asset's `xrefs` must be a superset of the union of
|
|
76
|
+
* all participants' `xrefs` plus the participant refs
|
|
77
77
|
* themselves — provenance never shrinks through a merge.
|
|
78
78
|
* 2. Specificity: distinctTokens(mergedBody) ≥ minSpecificityRetention ×
|
|
79
79
|
* |union(distinctTokens(participant bodies))| — a merge that only
|
|
@@ -111,7 +111,7 @@ export function checkMergeInformationFloor(mergedBody, mergedSourceRefs, partici
|
|
|
111
111
|
const specificityOk = specificityRetention >= minRetention;
|
|
112
112
|
const reasons = [];
|
|
113
113
|
if (!provenanceOk) {
|
|
114
|
-
reasons.push(`provenance shrank: merged
|
|
114
|
+
reasons.push(`provenance shrank: merged xrefs missing ${missing.length} ref(s) (e.g. ${missing[0]})`);
|
|
115
115
|
}
|
|
116
116
|
if (!specificityOk) {
|
|
117
117
|
reasons.push(`specificity retention ${specificityRetention.toFixed(2)} < ${minRetention} (merge genericized/shortened)`);
|
|
@@ -175,10 +175,10 @@ export function normHash(text) {
|
|
|
175
175
|
return hash.toString(16);
|
|
176
176
|
}
|
|
177
177
|
/**
|
|
178
|
-
* Score one canary against the live index, merge-following via
|
|
179
|
-
*
|
|
180
|
-
*
|
|
181
|
-
*
|
|
178
|
+
* Score one canary against the live index, merge-following via canonical
|
|
179
|
+
* `xrefs`. Legacy `source_refs` remains readable for assets written before the
|
|
180
|
+
* xref cutover. A hit is the anchor ref itself OR any returned entry whose
|
|
181
|
+
* provenance contains the anchor.
|
|
182
182
|
* Returns the 0-based rank of the first hit, or -1.
|
|
183
183
|
*/
|
|
184
184
|
function scoreCanary(indexDb, canary, k) {
|
|
@@ -188,7 +188,7 @@ function scoreCanary(indexDb, canary, k) {
|
|
|
188
188
|
const ref = makeAssetRef(r.entry.type, r.entry.name);
|
|
189
189
|
if (ref === canary.anchor_ref)
|
|
190
190
|
return i;
|
|
191
|
-
if (r.entry.sourceRefs?.includes(canary.anchor_ref))
|
|
191
|
+
if (r.entry.xrefs?.includes(canary.anchor_ref) || r.entry.sourceRefs?.includes(canary.anchor_ref))
|
|
192
192
|
return i;
|
|
193
193
|
}
|
|
194
194
|
return -1;
|
|
@@ -6,15 +6,15 @@ import fs from "node:fs";
|
|
|
6
6
|
import path from "node:path";
|
|
7
7
|
import readline from "node:readline";
|
|
8
8
|
import consolidateSystemPrompt from "../../assets/prompts/consolidate-system.md" with { type: "text" };
|
|
9
|
-
import { parseAssetRef } from "../../core/asset/asset-ref.js";
|
|
9
|
+
import { parseAssetRef, refToString } from "../../core/asset/asset-ref.js";
|
|
10
10
|
import { assembleAssetFromString, serializeFrontmatter } from "../../core/asset/asset-serialize.js";
|
|
11
11
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
12
|
-
import {
|
|
12
|
+
import { timestampForFilename } from "../../core/common.js";
|
|
13
13
|
import { getDefaultLlmConfig, getImproveProcessConfig, loadConfig } from "../../core/config/config.js";
|
|
14
14
|
import { ConfigError } from "../../core/errors.js";
|
|
15
15
|
// Note: appendEvent import removed (WS-3a: archive TTL machinery retired)
|
|
16
16
|
import { parseEmbeddedJsonResponse } from "../../core/parse.js";
|
|
17
|
-
import {
|
|
17
|
+
import { resolveStandardsContext } from "../../core/standards/resolve-standards-context.js";
|
|
18
18
|
import { detectTruncatedDescription } from "../../core/text-truncation.js";
|
|
19
19
|
import { DURATION_UNITS, parseDuration } from "../../core/time.js";
|
|
20
20
|
import { createProposal, isProposalSkipped, listProposals } from "../proposal/repository.js";
|
|
@@ -36,6 +36,7 @@ import { tryLlmFeature } from "../../llm/feature-gate.js";
|
|
|
36
36
|
import { getConsolidationJudgedMap, upsertConsolidationJudged, } from "../../storage/repositories/consolidation-repository.js";
|
|
37
37
|
import { getBodyEmbeddings, upsertBodyEmbeddings } from "../../storage/repositories/embeddings-repository.js";
|
|
38
38
|
import { resolveImproveStrategy, resolveProcessEnabled } from "./improve-strategies.js";
|
|
39
|
+
import { durableImproveRef } from "./source-identity.js";
|
|
39
40
|
// Chunk sizing + per-chunk prompt assembly live in ./consolidate/chunking.
|
|
40
41
|
// Imported for internal use by the orchestrator and re-exported for importers.
|
|
41
42
|
import { buildChunkPrompt, computeSafeChunkSize, DEFAULT_CONTEXT_LENGTH_TOKENS } from "./consolidate/chunking.js";
|
|
@@ -415,24 +416,36 @@ function backupFile(filePath, backupDir, name) {
|
|
|
415
416
|
}
|
|
416
417
|
// ── WS-3b: Generation frontmatter injection ───────────────────────────────────
|
|
417
418
|
/**
|
|
418
|
-
* Inject `generation` and `
|
|
419
|
+
* Inject `generation` and canonical `xrefs` into merged content.
|
|
419
420
|
* generation = max(sourceGenerations) + 1.
|
|
420
|
-
*
|
|
421
|
-
*
|
|
421
|
+
* xrefs = UNION of the provided provenance refs (participants + their cited
|
|
422
|
+
* sources) with anything already present in xrefs or legacy source_refs —
|
|
422
423
|
* R5 §4.2: the old set-if-absent behavior dropped second-generation
|
|
423
424
|
* provenance whenever the LLM emitted its own (partial) source_refs.
|
|
424
425
|
* Fails open — returns original content if frontmatter can't be parsed.
|
|
425
426
|
*/
|
|
426
|
-
function injectGenerationFrontmatter(mergedContent, sourceGenerations, provenanceRefs) {
|
|
427
|
+
export function injectGenerationFrontmatter(mergedContent, sourceGenerations, provenanceRefs) {
|
|
427
428
|
try {
|
|
428
429
|
const parsed = parseFrontmatter(mergedContent);
|
|
429
430
|
const existingFm = parsed.data;
|
|
430
|
-
const existingRefs =
|
|
431
|
+
const existingRefs = [
|
|
432
|
+
...(Array.isArray(existingFm.xrefs) ? existingFm.xrefs.map(String) : []),
|
|
433
|
+
...(Array.isArray(existingFm.source_refs) ? existingFm.source_refs.map(String) : []),
|
|
434
|
+
];
|
|
435
|
+
const canonicalRefs = [...existingRefs, ...provenanceRefs].flatMap((ref) => {
|
|
436
|
+
try {
|
|
437
|
+
return [refToString(parseAssetRef(ref))];
|
|
438
|
+
}
|
|
439
|
+
catch {
|
|
440
|
+
return [];
|
|
441
|
+
}
|
|
442
|
+
});
|
|
431
443
|
const updatedFm = {
|
|
432
444
|
...existingFm,
|
|
433
445
|
generation: computeMergedGeneration(sourceGenerations),
|
|
434
|
-
|
|
446
|
+
xrefs: [...new Set(canonicalRefs)],
|
|
435
447
|
};
|
|
448
|
+
delete updatedFm.source_refs;
|
|
436
449
|
return assembleAssetFromString(serializeFrontmatter(updatedFm), parsed.content);
|
|
437
450
|
}
|
|
438
451
|
catch {
|
|
@@ -568,6 +581,30 @@ export function makeConsolidateResult(overrides) {
|
|
|
568
581
|
};
|
|
569
582
|
}
|
|
570
583
|
// ── Main entry point ─────────────────────────────────────────────────────────
|
|
584
|
+
function resolveConsolidationWriteTarget(opts, config) {
|
|
585
|
+
if (opts.writeTarget) {
|
|
586
|
+
return {
|
|
587
|
+
...opts.writeTarget,
|
|
588
|
+
source: { ...opts.writeTarget.source, path: path.resolve(opts.writeTarget.source.path) },
|
|
589
|
+
};
|
|
590
|
+
}
|
|
591
|
+
if (opts.target && !path.isAbsolute(opts.target)) {
|
|
592
|
+
const target = resolveWriteTarget(config, opts.target);
|
|
593
|
+
return { ...target, source: { ...target.source, path: path.resolve(target.source.path) } };
|
|
594
|
+
}
|
|
595
|
+
// Programmatic callers historically supplied an absolute stash path. Normalize
|
|
596
|
+
// it immediately into the same unambiguous write-target shape used by named sources.
|
|
597
|
+
const explicitRoot = opts.target ?? opts.stashDir;
|
|
598
|
+
if (explicitRoot) {
|
|
599
|
+
const root = path.resolve(explicitRoot);
|
|
600
|
+
return {
|
|
601
|
+
source: { kind: "filesystem", name: "stash", path: root },
|
|
602
|
+
config: { type: "filesystem", name: "stash", path: root, writable: true },
|
|
603
|
+
};
|
|
604
|
+
}
|
|
605
|
+
const target = resolveWriteTarget(config);
|
|
606
|
+
return { ...target, source: { ...target.source, path: path.resolve(target.source.path) } };
|
|
607
|
+
}
|
|
571
608
|
export async function akmConsolidate(opts = {}) {
|
|
572
609
|
const startMs = Date.now();
|
|
573
610
|
// Derive a stable PROV-DM token for this run. Callers (e.g. akmImprove)
|
|
@@ -575,8 +612,10 @@ export async function akmConsolidate(opts = {}) {
|
|
|
575
612
|
// standalone `akm consolidate` gets a self-contained token.
|
|
576
613
|
const sourceRun = opts.sourceRun ?? `consolidate-${startMs}`;
|
|
577
614
|
const config = opts.config ?? loadConfig();
|
|
615
|
+
const writeTarget = resolveConsolidationWriteTarget(opts, config);
|
|
616
|
+
opts = { ...opts, target: writeTarget.source.name, writeTarget };
|
|
578
617
|
opts = { ...opts, improveProfile: opts.improveProfile ?? resolveImproveStrategy(undefined, config).config };
|
|
579
|
-
const stashDir =
|
|
618
|
+
const stashDir = writeTarget.source.path;
|
|
580
619
|
if (!resolveProcessEnabled("consolidate", opts.improveProfile ?? resolveImproveStrategy(undefined, config).config)) {
|
|
581
620
|
return makeConsolidateResult({
|
|
582
621
|
dryRun: opts.dryRun ?? false,
|
|
@@ -647,7 +686,7 @@ function createConsolidateAccounting() {
|
|
|
647
686
|
* passes consume. Behavior-identical to the former inlined narrowing block.
|
|
648
687
|
*/
|
|
649
688
|
async function narrowConsolidationPool(opts, config, stashDir, startMs, warnings, sharedStateDb) {
|
|
650
|
-
let memories = loadMemoriesForSource(opts.
|
|
689
|
+
let memories = loadMemoriesForSource(opts.writeTarget?.source.path, stashDir, warnings);
|
|
651
690
|
// Pre-flight: filter out stale DB entries whose files no longer exist on
|
|
652
691
|
// disk. Without this, memories deleted by a prior run (but not yet
|
|
653
692
|
// reindexed) appear in chunk prompts, causing the LLM to generate plans
|
|
@@ -782,7 +821,7 @@ async function narrowConsolidationPool(opts, config, stashDir, startMs, warnings
|
|
|
782
821
|
const dbForJudged = sharedStateDb;
|
|
783
822
|
if (dbForJudged) {
|
|
784
823
|
try {
|
|
785
|
-
cachedMap = getConsolidationJudgedMap(dbForJudged, memories.map((m) => `memory:${m.name}
|
|
824
|
+
cachedMap = getConsolidationJudgedMap(dbForJudged, memories.map((m) => durableImproveRef(`memory:${m.name}`, opts.target)));
|
|
786
825
|
}
|
|
787
826
|
catch {
|
|
788
827
|
cachedMap = new Map();
|
|
@@ -790,7 +829,7 @@ async function narrowConsolidationPool(opts, config, stashDir, startMs, warnings
|
|
|
790
829
|
}
|
|
791
830
|
else {
|
|
792
831
|
try {
|
|
793
|
-
cachedMap = withStateDb((localDb) => getConsolidationJudgedMap(localDb, memories.map((m) => `memory:${m.name}
|
|
832
|
+
cachedMap = withStateDb((localDb) => getConsolidationJudgedMap(localDb, memories.map((m) => durableImproveRef(`memory:${m.name}`, opts.target))));
|
|
794
833
|
}
|
|
795
834
|
catch {
|
|
796
835
|
// State DB unavailable → fail open: judge the full pool this run.
|
|
@@ -804,7 +843,7 @@ async function narrowConsolidationPool(opts, config, stashDir, startMs, warnings
|
|
|
804
843
|
// No readable hash → keep (fail open; let the LLM judge it).
|
|
805
844
|
if (cur === undefined)
|
|
806
845
|
return true;
|
|
807
|
-
const cached = cachedMap.get(`memory:${m.name}
|
|
846
|
+
const cached = cachedMap.get(durableImproveRef(`memory:${m.name}`, opts.target));
|
|
808
847
|
// Skip only when previously judged AND content is byte-identical since.
|
|
809
848
|
return !(cached !== undefined && cached.content_hash === cur);
|
|
810
849
|
});
|
|
@@ -992,7 +1031,7 @@ async function planConsolidation(opts, config, stashDir, startMs, memories, warn
|
|
|
992
1031
|
// Consolidate output merges memories (non-wiki) → stash authoring standards.
|
|
993
1032
|
// Resolved ONCE per run and passed to each chunk prompt (facts not re-read
|
|
994
1033
|
// per chunk).
|
|
995
|
-
const standardsContext =
|
|
1034
|
+
const standardsContext = resolveStandardsContext("memory:_consolidated", stashDir);
|
|
996
1035
|
const chunkOpsArrays = [];
|
|
997
1036
|
// judgedNoAction tracks memories the LLM saw inside a chunk but proposed
|
|
998
1037
|
// no op for. Computed per chunk as `chunk.length − unique(targetRefs in ops)`.
|
|
@@ -1192,7 +1231,7 @@ async function planConsolidation(opts, config, stashDir, startMs, memories, warn
|
|
|
1192
1231
|
if (hash === undefined)
|
|
1193
1232
|
continue;
|
|
1194
1233
|
upsertConsolidationJudged(db, {
|
|
1195
|
-
entryKey: `memory:${name}`,
|
|
1234
|
+
entryKey: durableImproveRef(`memory:${name}`, opts.target),
|
|
1196
1235
|
contentHash: hash,
|
|
1197
1236
|
judgedAt,
|
|
1198
1237
|
outcome,
|
|
@@ -1230,9 +1269,8 @@ async function planConsolidation(opts, config, stashDir, startMs, memories, warn
|
|
|
1230
1269
|
* the op-handlers' `pushSkipReason`. Behavior-identical to the former inlined
|
|
1231
1270
|
* write block. Never invoked on the dry-run or aborted-confirm paths.
|
|
1232
1271
|
*/
|
|
1233
|
-
async function applyConsolidationPlan(config, stashDir, sourceRun, memories, warnings, allOps, accounting, dedupCollapsed, activeProfile, llmConfig) {
|
|
1272
|
+
async function applyConsolidationPlan(config, stashDir, target, sourceRun, memories, warnings, allOps, accounting, dedupCollapsed, activeProfile, llmConfig) {
|
|
1234
1273
|
// -- Phase B + writes -------------------------------------------------------
|
|
1235
|
-
const target = resolveWriteTarget(config);
|
|
1236
1274
|
const timestamp = timestampForFilename();
|
|
1237
1275
|
const backupDir = getBackupDir(stashDir, timestamp);
|
|
1238
1276
|
// Write journal before any mutations
|
|
@@ -1381,7 +1419,7 @@ async function akmConsolidateInner(opts, config, stashDir, startMs, sourceRun, w
|
|
|
1381
1419
|
}
|
|
1382
1420
|
}
|
|
1383
1421
|
// -- Pass 3: execute the plan against the filesystem ------------------------
|
|
1384
|
-
const { merged, deleted, contradicted, mergeFloorViolations, mergedSecondaries, promoted } = await applyConsolidationPlan(config, stashDir, sourceRun, memories, warnings, allOps, accounting, dedupCollapsed, opts.improveProfile, Object.hasOwn(opts, "llmConfig")
|
|
1422
|
+
const { merged, deleted, contradicted, mergeFloorViolations, mergedSecondaries, promoted } = await applyConsolidationPlan(config, stashDir, opts.writeTarget, sourceRun, memories, warnings, allOps, accounting, dedupCollapsed, opts.improveProfile, Object.hasOwn(opts, "llmConfig")
|
|
1385
1423
|
? (opts.llmConfig ?? null)
|
|
1386
1424
|
: (resolveConsolidateLlmConfig(config, opts.improveProfile) ?? null));
|
|
1387
1425
|
const runDurationMs = Date.now() - startMs;
|
|
@@ -1505,7 +1543,7 @@ export async function handleMergeOp(op, opIndex, ctx) {
|
|
|
1505
1543
|
emitMergeFailureSkips("merge_read_failed");
|
|
1506
1544
|
return;
|
|
1507
1545
|
}
|
|
1508
|
-
const mergeResult = await generateMergedContent(config, op.primary, primaryBody, op.secondaries, memoryByRef, ctx.improveProfile, ctx.llmConfig);
|
|
1546
|
+
const mergeResult = await (ctx.generateMergedContentFn ?? generateMergedContent)(config, op.primary, primaryBody, op.secondaries, memoryByRef, ctx.improveProfile, ctx.llmConfig);
|
|
1509
1547
|
if ("error" in mergeResult) {
|
|
1510
1548
|
warnings.push(`Merge: ${mergeResult.error} for ${mergeResult.detail}.`);
|
|
1511
1549
|
emitMergeFailureSkips(mergeResult.error);
|
|
@@ -1565,6 +1603,28 @@ export async function handleMergeOp(op, opIndex, ctx) {
|
|
|
1565
1603
|
emitMergeFailureSkips("merge_participant_blocked");
|
|
1566
1604
|
return;
|
|
1567
1605
|
}
|
|
1606
|
+
const allParticipants = [op.primary, ...op.secondaries];
|
|
1607
|
+
// Generation and provenance are mandatory merge metadata, independent of
|
|
1608
|
+
// whether the optional anti-collapse refusal/advisory checks are enabled.
|
|
1609
|
+
const participantInfo = allParticipants.map((ref) => {
|
|
1610
|
+
const e = memoryByRef.get(ref);
|
|
1611
|
+
if (!e)
|
|
1612
|
+
return { ref, generation: 0, body: "", sourceRefs: [] };
|
|
1613
|
+
try {
|
|
1614
|
+
const raw = fs.readFileSync(e.filePath, "utf8");
|
|
1615
|
+
const parsed = parseFrontmatter(raw);
|
|
1616
|
+
const fm = parsed.data;
|
|
1617
|
+
const sourceRefs = [
|
|
1618
|
+
...(Array.isArray(fm.xrefs) ? fm.xrefs.map(String) : []),
|
|
1619
|
+
...(Array.isArray(fm.source_refs) ? fm.source_refs.map(String) : []),
|
|
1620
|
+
];
|
|
1621
|
+
return { ref, generation: readAssetGeneration(fm), body: stripFrontmatterBody(raw), sourceRefs };
|
|
1622
|
+
}
|
|
1623
|
+
catch {
|
|
1624
|
+
return { ref, generation: 0, body: "", sourceRefs: [] };
|
|
1625
|
+
}
|
|
1626
|
+
});
|
|
1627
|
+
const sourceGenerations = participantInfo.map((p) => p.generation);
|
|
1568
1628
|
// WS-3b: Anti-collapse generation guard (step 8a).
|
|
1569
1629
|
// DEFAULT ON since R5 (opt out via antiCollapse.enabled: false). Refuses
|
|
1570
1630
|
// to merge two assets both above generation N (default 2) — prevents the
|
|
@@ -1572,25 +1632,6 @@ export async function handleMergeOp(op, opIndex, ctx) {
|
|
|
1572
1632
|
// source fidelity of the original episodes.
|
|
1573
1633
|
const antiCollapseConfig = getImproveProcessConfig(config, "consolidate", ctx.improveProfile)?.antiCollapse ?? {};
|
|
1574
1634
|
if (antiCollapseConfig.enabled !== false) {
|
|
1575
|
-
const allParticipants = [op.primary, ...op.secondaries];
|
|
1576
|
-
// One read per participant: generation counter, stripped body (for the
|
|
1577
|
-
// information floor), and existing source_refs (for the provenance union).
|
|
1578
|
-
const participantInfo = allParticipants.map((ref) => {
|
|
1579
|
-
const e = memoryByRef.get(ref);
|
|
1580
|
-
if (!e)
|
|
1581
|
-
return { ref, generation: 0, body: "", sourceRefs: [] };
|
|
1582
|
-
try {
|
|
1583
|
-
const raw = fs.readFileSync(e.filePath, "utf8");
|
|
1584
|
-
const parsed = parseFrontmatter(raw);
|
|
1585
|
-
const fm = parsed.data;
|
|
1586
|
-
const sourceRefs = Array.isArray(fm.source_refs) ? fm.source_refs.map(String) : [];
|
|
1587
|
-
return { ref, generation: readAssetGeneration(fm), body: stripFrontmatterBody(raw), sourceRefs };
|
|
1588
|
-
}
|
|
1589
|
-
catch {
|
|
1590
|
-
return { ref, generation: 0, body: "", sourceRefs: [] };
|
|
1591
|
-
}
|
|
1592
|
-
});
|
|
1593
|
-
const sourceGenerations = participantInfo.map((p) => p.generation);
|
|
1594
1635
|
const generationCheck = checkGenerationGuard(sourceGenerations, antiCollapseConfig);
|
|
1595
1636
|
if (generationCheck.refused) {
|
|
1596
1637
|
warnings.push(`Merge: ${generationCheck.reason}`);
|
|
@@ -1608,19 +1649,19 @@ export async function handleMergeOp(op, opIndex, ctx) {
|
|
|
1608
1649
|
warnings.push(`Merge: cluster around ${op.primary} has low lexical diversity (${diversityCheck.diversity?.toFixed(2) ?? "?"} < 0.30) — likely correlated extraction; merge proceeds but review is recommended.`);
|
|
1609
1650
|
}
|
|
1610
1651
|
}
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
|
|
1615
|
-
|
|
1616
|
-
|
|
1652
|
+
}
|
|
1653
|
+
// merged.generation = max(sourceGenerations) + 1. xrefs is the UNION of
|
|
1654
|
+
// participants + canonical and legacy provenance already carried by them.
|
|
1655
|
+
const provenanceUnion = [...new Set([...allParticipants, ...participantInfo.flatMap((p) => p.sourceRefs)])];
|
|
1656
|
+
mergedContent = injectGenerationFrontmatter(mergedContent, sourceGenerations, provenanceUnion);
|
|
1657
|
+
if (antiCollapseConfig.enabled !== false) {
|
|
1617
1658
|
// R5 §4.2: merge-information floor — ADVISORY in v1. A merge that
|
|
1618
1659
|
// shrinks provenance or genericizes below the retention floor is
|
|
1619
1660
|
// counted + warned, never refused (promotion path: design doc §7).
|
|
1620
1661
|
try {
|
|
1621
1662
|
const mergedParsed = parseFrontmatter(mergedContent);
|
|
1622
1663
|
const mergedFm = mergedParsed.data;
|
|
1623
|
-
const mergedSourceRefs = Array.isArray(mergedFm.
|
|
1664
|
+
const mergedSourceRefs = Array.isArray(mergedFm.xrefs) ? mergedFm.xrefs.map(String) : [];
|
|
1624
1665
|
const floorCheck = checkMergeInformationFloor(mergedParsed.content, mergedSourceRefs, participantInfo, antiCollapseConfig);
|
|
1625
1666
|
if (!floorCheck.passed) {
|
|
1626
1667
|
counts.mergeFloorViolations++;
|
|
@@ -1758,17 +1799,22 @@ export async function handlePromoteOp(op, ctx) {
|
|
|
1758
1799
|
pushSkipReason("promote", op.ref, "promote_already_promoted_this_run");
|
|
1759
1800
|
return;
|
|
1760
1801
|
}
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1802
|
+
const proposedName = op.knowledgeRef
|
|
1803
|
+
.replace(/^knowledge:/, "")
|
|
1804
|
+
.split("/")
|
|
1805
|
+
.filter(Boolean)
|
|
1806
|
+
.at(-1) ??
|
|
1807
|
+
entry.name.split("/").filter(Boolean).at(-1) ??
|
|
1808
|
+
"promoted-memory";
|
|
1809
|
+
const slug = proposedName
|
|
1810
|
+
.replace(/[^a-z0-9-]/gi, "-")
|
|
1811
|
+
.replace(/-+/g, "-")
|
|
1812
|
+
.replace(/^-|-$/g, "")
|
|
1813
|
+
.toLowerCase();
|
|
1814
|
+
const knowledgeRef = `knowledge:${slug}`;
|
|
1815
|
+
parseAssetRef(knowledgeRef);
|
|
1816
|
+
if (knowledgeRef !== op.knowledgeRef) {
|
|
1817
|
+
warnings.push(`Normalized generated ref "${op.knowledgeRef}" → "${knowledgeRef}"`);
|
|
1772
1818
|
}
|
|
1773
1819
|
// Idempotency: check pending proposals by target ref
|
|
1774
1820
|
const existingProposals = listProposals(stashDir, { ref: knowledgeRef });
|
|
@@ -1879,6 +1925,9 @@ export async function handlePromoteOp(op, ctx) {
|
|
|
1879
1925
|
const mergedBodyFm = {
|
|
1880
1926
|
...(parsedMemory.data ?? {}),
|
|
1881
1927
|
description,
|
|
1928
|
+
xrefs: [
|
|
1929
|
+
...new Set([...(Array.isArray(parsedMemory.data?.xrefs) ? parsedMemory.data.xrefs.map(String) : []), op.ref]),
|
|
1930
|
+
],
|
|
1882
1931
|
};
|
|
1883
1932
|
const serializedMergedFm = serializeFrontmatter(mergedBodyFm);
|
|
1884
1933
|
const proposalContent = assembleAssetFromString(serializedMergedFm, parsedMemory.content);
|
|
@@ -1904,7 +1953,7 @@ export async function handlePromoteOp(op, ctx) {
|
|
|
1904
1953
|
sourceRun,
|
|
1905
1954
|
payload: {
|
|
1906
1955
|
content: proposalContent,
|
|
1907
|
-
frontmatter: { description },
|
|
1956
|
+
frontmatter: { description, xrefs: [op.ref] },
|
|
1908
1957
|
},
|
|
1909
1958
|
...(typeof op.confidence === "number" ? { confidence: op.confidence } : {}),
|
|
1910
1959
|
});
|
|
@@ -2134,14 +2183,22 @@ function loadMemoriesForSource(source, stashDir, warnings) {
|
|
|
2134
2183
|
const memoriesDir = path.join(source ?? stashDir, "memories");
|
|
2135
2184
|
const fsStashDir = source ?? stashDir;
|
|
2136
2185
|
if (fs.existsSync(memoriesDir)) {
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
const
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2186
|
+
const pending = [memoriesDir];
|
|
2187
|
+
while (pending.length > 0) {
|
|
2188
|
+
const current = pending.pop();
|
|
2189
|
+
for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
|
|
2190
|
+
const filePath = path.join(current, entry.name);
|
|
2191
|
+
if (entry.isDirectory()) {
|
|
2192
|
+
pending.push(filePath);
|
|
2193
|
+
continue;
|
|
2194
|
+
}
|
|
2195
|
+
if (!entry.isFile() || !entry.name.endsWith(".md"))
|
|
2196
|
+
continue;
|
|
2197
|
+
const name = path.relative(memoriesDir, filePath).replace(/\.md$/, "").split(path.sep).join("/");
|
|
2198
|
+
if (!isConsolidationEligibleMemoryName(name))
|
|
2199
|
+
continue;
|
|
2200
|
+
memories.push({ name, filePath, description: "", tags: [], stashDir: fsStashDir });
|
|
2201
|
+
}
|
|
2145
2202
|
}
|
|
2146
2203
|
}
|
|
2147
2204
|
if (memories.length > 0) {
|
|
@@ -22,6 +22,7 @@ import { appendEvent } from "../../../core/events.js";
|
|
|
22
22
|
import { parseEmbeddedJsonResponse } from "../../../llm/client.js";
|
|
23
23
|
import { createProposal, isProposalSkipped } from "../../proposal/repository.js";
|
|
24
24
|
import { assessMemoryKnowledgePromotionCandidate } from "../distill-promotion-policy.js";
|
|
25
|
+
import { durableImproveRef } from "../source-identity.js";
|
|
25
26
|
import { persistOutputEncodingSalience, runLessonQualityJudge, writeQualityRejection } from "./quality-gate.js";
|
|
26
27
|
/**
|
|
27
28
|
* Run the memory→knowledge promotion branch. Returns the finished distill
|
|
@@ -31,10 +32,11 @@ import { persistOutputEncodingSalience, runLessonQualityJudge, writeQualityRejec
|
|
|
31
32
|
*/
|
|
32
33
|
export async function promoteMemoryToKnowledge(ctx) {
|
|
33
34
|
const { targetKind, inputRef, assetContent, config, chat, stash, lookup, fetchSimilarLessonsFn, existingRefVocabulary, outcomeWeightEnabled, eligMeta, exclusionSetSize, filteredFeedbackCount, feedbackFullyFiltered, } = ctx;
|
|
35
|
+
const durableInputRef = ctx.durableInputRef ?? inputRef;
|
|
34
36
|
const promotion = targetKind === "lesson"
|
|
35
37
|
? null
|
|
36
38
|
: assessMemoryKnowledgePromotionCandidate({
|
|
37
|
-
inputRef,
|
|
39
|
+
inputRef: durableInputRef,
|
|
38
40
|
assetContent,
|
|
39
41
|
feedbackEvents: ctx.filteredEvents.map((event) => ({
|
|
40
42
|
...(event.metadata !== undefined ? { metadata: event.metadata } : {}),
|
|
@@ -48,7 +50,7 @@ export async function promoteMemoryToKnowledge(ctx) {
|
|
|
48
50
|
// overwriting. Follows mem0 ADD/UPDATE/DELETE/NOOP pattern (arXiv:2504.19413 §3.2)
|
|
49
51
|
// and A-MEM dynamic linking (arXiv:2502.12110).
|
|
50
52
|
let resolvedPromotionContent = promotion.content;
|
|
51
|
-
const existingKnowledgePath = await lookup(promotion.knowledgeRef);
|
|
53
|
+
const existingKnowledgePath = await lookup(durableImproveRef(promotion.knowledgeRef, ctx.sourceName));
|
|
52
54
|
const existingKnowledgeContent = existingKnowledgePath && fs.existsSync(existingKnowledgePath)
|
|
53
55
|
? (() => {
|
|
54
56
|
try {
|
|
@@ -87,7 +89,7 @@ export async function promoteMemoryToKnowledge(ctx) {
|
|
|
87
89
|
// Existing content is authoritative — no update needed.
|
|
88
90
|
appendEvent({
|
|
89
91
|
eventType: "distill_invoked",
|
|
90
|
-
ref:
|
|
92
|
+
ref: durableInputRef,
|
|
91
93
|
metadata: {
|
|
92
94
|
outcome: "skipped",
|
|
93
95
|
lessonRef: promotion.knowledgeRef,
|
|
@@ -168,7 +170,7 @@ export async function promoteMemoryToKnowledge(ctx) {
|
|
|
168
170
|
if (isProposalSkipped(proposalResult)) {
|
|
169
171
|
appendEvent({
|
|
170
172
|
eventType: "distill_invoked",
|
|
171
|
-
ref:
|
|
173
|
+
ref: durableInputRef,
|
|
172
174
|
metadata: {
|
|
173
175
|
outcome: "skipped",
|
|
174
176
|
lessonRef: promotion.knowledgeRef,
|
|
@@ -189,10 +191,10 @@ export async function promoteMemoryToKnowledge(ctx) {
|
|
|
189
191
|
const proposal = proposalResult;
|
|
190
192
|
// G4: content-score the distilled OUTPUT so it carries a real encoding
|
|
191
193
|
// salience (encoding_source='content') from creation.
|
|
192
|
-
persistOutputEncodingSalience(promotion.knowledgeRef, resolvedPromotionContent, existingRefVocabulary, outcomeWeightEnabled);
|
|
194
|
+
persistOutputEncodingSalience(durableImproveRef(promotion.knowledgeRef, ctx.sourceName), resolvedPromotionContent, existingRefVocabulary, outcomeWeightEnabled);
|
|
193
195
|
appendEvent({
|
|
194
196
|
eventType: "distill_invoked",
|
|
195
|
-
ref:
|
|
197
|
+
ref: durableInputRef,
|
|
196
198
|
metadata: {
|
|
197
199
|
outcome: "queued",
|
|
198
200
|
lessonRef: promotion.knowledgeRef,
|
|
@@ -169,7 +169,7 @@ export function writeQualityRejection(stash, inputRef, lessonRef, content, score
|
|
|
169
169
|
const rejectDir = path.join(stash, ".akm", "distill-rejected");
|
|
170
170
|
fs.mkdirSync(rejectDir, { recursive: true });
|
|
171
171
|
const ts = timestampForFilename();
|
|
172
|
-
fs.writeFileSync(path.join(rejectDir, `${ts}-${lessonRef}.md`), `---\nscore: ${score}\nreason: ${reason}\noutcome: ${outcome}\n---\n\n${content}`, "utf8");
|
|
172
|
+
fs.writeFileSync(path.join(rejectDir, `${ts}-${lessonRef.replace(/[:/\\]/g, "-")}.md`), `---\nscore: ${score}\nreason: ${reason}\noutcome: ${outcome}\n---\n\n${content}`, "utf8");
|
|
173
173
|
appendEvent({
|
|
174
174
|
eventType: "distill_invoked",
|
|
175
175
|
ref: inputRef,
|
|
@@ -96,7 +96,7 @@ export function checkDistillFidelity(proposalBody, sourceBodies, config) {
|
|
|
96
96
|
}
|
|
97
97
|
}
|
|
98
98
|
}
|
|
99
|
-
// Also flag proposals whose
|
|
99
|
+
// Also flag proposals whose xrefs are empty (broken provenance).
|
|
100
100
|
// This is a degradation signal, not a contradiction, but worth surfacing.
|
|
101
101
|
return { contradictionDetected: false };
|
|
102
102
|
}
|