akm-cli 0.9.0-beta.11 → 0.9.0-beta.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +163 -0
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +5 -1
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/templates/html/health.html +25 -27
- package/dist/cli.js +2 -2
- package/dist/commands/agent/contribute-cli.js +16 -3
- package/dist/commands/feedback-cli.js +48 -44
- package/dist/commands/health/html-report.js +140 -16
- package/dist/commands/health.js +277 -1
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/consolidate.js +595 -105
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill.js +119 -64
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +115 -1
- package/dist/commands/improve/extract-prompt.js +32 -1
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +210 -30
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/homeostatic.js +467 -0
- package/dist/commands/improve/improve-auto-accept.js +80 -7
- package/dist/commands/improve/improve-profiles.js +8 -0
- package/dist/commands/improve/improve.js +991 -61
- package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
- package/dist/commands/improve/outcome-loop.js +256 -0
- package/dist/commands/improve/proactive-maintenance.js +9 -35
- package/dist/commands/improve/procedural.js +409 -0
- package/dist/commands/improve/recombine.js +488 -0
- package/dist/commands/improve/reflect.js +20 -1
- package/dist/commands/improve/related-sessions.js +120 -0
- package/dist/commands/improve/salience.js +386 -0
- package/dist/commands/improve/triage.js +95 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +34 -1
- package/dist/commands/lint/index.js +30 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +5 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/proposal/validators/proposals.js +4 -0
- package/dist/commands/read/curate.js +284 -86
- package/dist/commands/read/search-cli.js +7 -0
- package/dist/commands/read/search.js +1 -0
- package/dist/commands/sources/installed-stashes.js +5 -1
- package/dist/core/asset/frontmatter.js +166 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/config/config-schema.js +211 -3
- package/dist/core/config/config.js +2 -2
- package/dist/core/logs-db.js +4 -3
- package/dist/core/state-db.js +555 -29
- package/dist/indexer/db/db.js +250 -27
- package/dist/indexer/db/graph-db.js +81 -86
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/passes/memory-inference.js +10 -3
- package/dist/indexer/passes/staleness-detect.js +2 -5
- package/dist/indexer/search/db-search.js +15 -4
- package/dist/indexer/search/ranking.js +4 -0
- package/dist/integrations/harnesses/claude/session-log.js +10 -0
- package/dist/integrations/harnesses/opencode/session-log.js +9 -0
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/embedder.js +27 -3
- package/dist/llm/embedders/local.js +66 -2
- package/dist/llm/graph-extract.js +2 -1
- package/dist/llm/memory-infer.js +4 -8
- package/dist/llm/metadata-enhance.js +9 -1
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/text/helpers.js +9 -0
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +1025 -567
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +435 -269
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/workflows/db.js +3 -4
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/data-and-telemetry.md +1 -0
- package/package.json +5 -4
|
@@ -4,170 +4,43 @@
|
|
|
4
4
|
/**
|
|
5
5
|
* Shared frontmatter parsing utilities.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
7
|
+
* Uses the `yaml` library for all YAML parsing so that the full YAML spec
|
|
8
|
+
* (block scalars, multi-line strings, nested objects, flow sequences, escape
|
|
9
|
+
* sequences) is handled correctly without a brittle hand-rolled state machine.
|
|
9
10
|
*/
|
|
11
|
+
import { parse as yamlParse, stringify as yamlStringify } from "yaml";
|
|
10
12
|
/**
|
|
11
|
-
* Parse YAML
|
|
13
|
+
* Parse YAML frontmatter from a Markdown (or similar) string.
|
|
12
14
|
*
|
|
13
15
|
* Returns the parsed key-value data and the remaining body content.
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* - **Top-level values**: string, boolean, and number scalars are supported,
|
|
18
|
-
* as well as top-level list-valued keys using YAML block sequences
|
|
19
|
-
* (`- item`) or flow arrays (`[a, b, c]`).
|
|
20
|
-
* - **List item types**: list items must be scalar values and may be strings,
|
|
21
|
-
* booleans, or numbers.
|
|
22
|
-
* - **No nested objects beyond one level**: Only a single level of indented
|
|
23
|
-
* key-value pairs is supported.
|
|
24
|
-
* - **Block scalars**: `|` (literal), `|-` (strip), and `|+` (keep) block
|
|
25
|
-
* scalars are supported for multi-line string values as emitted by the
|
|
26
|
-
* `yaml` library's `stringify`.
|
|
16
|
+
* Delegates all YAML parsing to the `yaml` library; the only responsibility
|
|
17
|
+
* of this function is extracting the `---…---` block and normalizing the
|
|
18
|
+
* parsed result (e.g. converting YAML timestamp values to ISO date strings).
|
|
27
19
|
*/
|
|
28
20
|
export function parseFrontmatter(raw) {
|
|
29
21
|
const parsedBlock = parseFrontmatterBlock(raw);
|
|
30
22
|
if (!parsedBlock) {
|
|
31
23
|
return { data: {}, content: raw, frontmatter: null, bodyStartLine: 1 };
|
|
32
24
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
/** Lines collected while in "block" mode. */
|
|
44
|
-
let blockLines = null;
|
|
45
|
-
/** Block scalar chomping: "clip" (|), "strip" (|-), "keep" (|+). */
|
|
46
|
-
let blockChomping = "clip";
|
|
47
|
-
const flushPending = () => {
|
|
48
|
-
// Called when we start a new top-level key and the previous key was still "pending".
|
|
49
|
-
// An empty-value key followed by another top-level key means it was an empty scalar.
|
|
50
|
-
if (mode === "pending" && currentKey !== null) {
|
|
51
|
-
data[currentKey] = "";
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
const flushBlock = () => {
|
|
55
|
-
// Commit the accumulated block-scalar lines to `data[currentKey]`.
|
|
56
|
-
if (mode !== "block" || currentKey === null || blockLines === null)
|
|
57
|
-
return;
|
|
58
|
-
// De-indent: strip the common 2-space prefix `yaml.stringify` emits.
|
|
59
|
-
const deindented = blockLines.map((l) => (l.startsWith(" ") ? l.slice(2) : l));
|
|
60
|
-
// Chomping: apply trailing-newline policy.
|
|
61
|
-
// "clip" (|): single trailing newline.
|
|
62
|
-
// "strip" (|-): no trailing newline.
|
|
63
|
-
// "keep" (|+): keep all trailing newlines as-is.
|
|
64
|
-
if (blockChomping === "keep") {
|
|
65
|
-
data[currentKey] = deindented.join("\n");
|
|
66
|
-
}
|
|
67
|
-
else if (blockChomping === "strip") {
|
|
68
|
-
data[currentKey] = deindented.join("\n").replace(/\n+$/, "");
|
|
69
|
-
}
|
|
70
|
-
else {
|
|
71
|
-
// "clip": exactly one trailing newline
|
|
72
|
-
data[currentKey] = `${deindented.join("\n").replace(/\n+$/, "")}\n`;
|
|
73
|
-
}
|
|
74
|
-
};
|
|
75
|
-
for (const line of parsedBlock.frontmatter.split(/\r?\n/)) {
|
|
76
|
-
// If we are in block-scalar mode, collect indented lines or end the block.
|
|
77
|
-
if (mode === "block") {
|
|
78
|
-
if (line.startsWith(" ") || line === "") {
|
|
79
|
-
// Continuation of the block scalar (indented content or blank line).
|
|
80
|
-
blockLines.push(line);
|
|
81
|
-
continue;
|
|
25
|
+
let data = {};
|
|
26
|
+
if (parsedBlock.frontmatter.trim()) {
|
|
27
|
+
try {
|
|
28
|
+
const parsed = yamlParse(parsedBlock.frontmatter);
|
|
29
|
+
if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
30
|
+
// Normalize Date objects: the yaml "core" schema parses YYYY-MM-DD
|
|
31
|
+
// literals as JS Date instances. Convert them back to ISO date strings
|
|
32
|
+
// to preserve the string type that callers (and yaml.stringify on write)
|
|
33
|
+
// expect.
|
|
34
|
+
data = normalizeYamlValues(parsed);
|
|
82
35
|
}
|
|
83
|
-
// Non-indented line ends the block scalar — flush and fall through to
|
|
84
|
-
// parse the new line as a top-level key.
|
|
85
|
-
flushBlock();
|
|
86
|
-
mode = "scalar";
|
|
87
|
-
blockLines = null;
|
|
88
36
|
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
// First block-sequence item after an empty-value key — switch to list mode
|
|
95
|
-
currentList = [];
|
|
96
|
-
data[currentKey] = currentList;
|
|
97
|
-
mode = "list";
|
|
98
|
-
}
|
|
99
|
-
currentList.push(parseYamlScalar(seqItem[1].trim()));
|
|
100
|
-
continue;
|
|
101
|
-
}
|
|
102
|
-
// Plain-style multi-line scalar continuation: a 2-space-indented line that
|
|
103
|
-
// is not a sequence item or nested key. YAML plain scalars fold newlines
|
|
104
|
-
// into a single space, so we append with a space. This handles LLM-emitted
|
|
105
|
-
// descriptions like:
|
|
106
|
-
// description: Use 4-colon outer containers when mixing
|
|
107
|
-
// nesting depths in markdown-it-container plugins.
|
|
108
|
-
// Without this, only the first line is captured and the truncation
|
|
109
|
-
// heuristic wrongly flags it as cut off mid-sentence.
|
|
110
|
-
if (mode === "scalar" && currentKey !== null && /^ {2}\S/.test(line)) {
|
|
111
|
-
data[currentKey] = `${String(data[currentKey])} ${line.trim()}`;
|
|
112
|
-
continue;
|
|
113
|
-
}
|
|
114
|
-
// Indented nested key-value (object under a key with empty value)
|
|
115
|
-
const indented = line.match(/^ {2}(\w[\w-]*):\s*(.+)$/);
|
|
116
|
-
if (indented && currentKey !== null && (mode === "object" || mode === "pending")) {
|
|
117
|
-
if (mode === "pending") {
|
|
118
|
-
// First indented k-v after an empty-value key — switch to object mode
|
|
119
|
-
nested = {};
|
|
120
|
-
data[currentKey] = nested;
|
|
121
|
-
mode = "object";
|
|
122
|
-
}
|
|
123
|
-
nested[indented[1]] = parseYamlScalar(indented[2].trim());
|
|
124
|
-
continue;
|
|
37
|
+
catch {
|
|
38
|
+
// Malformed YAML (e.g. unterminated quotes from LLM output corruption).
|
|
39
|
+
// Fall back to line-by-line best-effort extraction so callers still get
|
|
40
|
+
// whatever scalar values they can rather than a completely empty record.
|
|
41
|
+
data = parseFrontmatterLenient(parsedBlock.frontmatter);
|
|
125
42
|
}
|
|
126
|
-
// Top-level key (possibly with inline value)
|
|
127
|
-
const top = line.match(/^(\w[\w-]*):\s*(.*)$/);
|
|
128
|
-
if (!top) {
|
|
129
|
-
continue;
|
|
130
|
-
}
|
|
131
|
-
// Starting a new top-level key — flush any pending empty-value key
|
|
132
|
-
flushPending();
|
|
133
|
-
currentKey = top[1];
|
|
134
|
-
const value = top[2].trim();
|
|
135
|
-
if (value === "|" || value === "|-" || value === "|+") {
|
|
136
|
-
// Block scalar header — collect subsequent indented lines.
|
|
137
|
-
mode = "block";
|
|
138
|
-
blockLines = [];
|
|
139
|
-
blockChomping = value === "|-" ? "strip" : value === "|+" ? "keep" : "clip";
|
|
140
|
-
nested = null;
|
|
141
|
-
currentList = null;
|
|
142
|
-
}
|
|
143
|
-
else if (value === "") {
|
|
144
|
-
// Defer mode decision until we see the next line
|
|
145
|
-
mode = "pending";
|
|
146
|
-
nested = null;
|
|
147
|
-
currentList = null;
|
|
148
|
-
// Don't store anything yet — flushPending will set "" if no continuation
|
|
149
|
-
}
|
|
150
|
-
else if (value.startsWith("[") && value.endsWith("]")) {
|
|
151
|
-
// Inline flow array: tags: [ops, networking]
|
|
152
|
-
mode = "list";
|
|
153
|
-
nested = null;
|
|
154
|
-
currentList = null;
|
|
155
|
-
currentList = parseFlowArray(value);
|
|
156
|
-
data[currentKey] = currentList;
|
|
157
|
-
}
|
|
158
|
-
else {
|
|
159
|
-
mode = "scalar";
|
|
160
|
-
nested = null;
|
|
161
|
-
currentList = null;
|
|
162
|
-
data[currentKey] = parseYamlScalar(value);
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
// Flush any in-progress block scalar at end of frontmatter.
|
|
166
|
-
if (mode === "block") {
|
|
167
|
-
flushBlock();
|
|
168
43
|
}
|
|
169
|
-
// Flush the last key if it was still pending (empty value, no continuation)
|
|
170
|
-
flushPending();
|
|
171
44
|
return {
|
|
172
45
|
data,
|
|
173
46
|
content: parsedBlock.content,
|
|
@@ -176,29 +49,89 @@ export function parseFrontmatter(raw) {
|
|
|
176
49
|
};
|
|
177
50
|
}
|
|
178
51
|
/**
|
|
179
|
-
*
|
|
52
|
+
* Normalize YAML-parsed values to match expected AKM frontmatter types.
|
|
53
|
+
*
|
|
54
|
+
* Two conversions:
|
|
55
|
+
* 1. `Date` → YYYY-MM-DD string: the yaml "core" schema parses bare date
|
|
56
|
+
* scalars like `2026-06-18` as JS Date instances. AKM frontmatter treats
|
|
57
|
+
* `updated:` and similar fields as plain strings.
|
|
58
|
+
* 2. `null` → `""`: the yaml library parses empty-value keys (`key:` with no
|
|
59
|
+
* value) as `null`, but AKM callers historically received `""` from the
|
|
60
|
+
* hand-rolled parser. Convert to preserve backward compatibility.
|
|
180
61
|
*/
|
|
181
|
-
function
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
62
|
+
function normalizeYamlValues(value) {
|
|
63
|
+
if (value instanceof Date) {
|
|
64
|
+
const y = value.getUTCFullYear();
|
|
65
|
+
const m = String(value.getUTCMonth() + 1).padStart(2, "0");
|
|
66
|
+
const d = String(value.getUTCDate()).padStart(2, "0");
|
|
67
|
+
return `${y}-${m}-${d}`;
|
|
68
|
+
}
|
|
69
|
+
if (value === null)
|
|
70
|
+
return "";
|
|
71
|
+
if (Array.isArray(value))
|
|
72
|
+
return value.map(normalizeYamlValues);
|
|
73
|
+
if (typeof value === "object") {
|
|
74
|
+
return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, normalizeYamlValues(v)]));
|
|
75
|
+
}
|
|
76
|
+
return value;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Best-effort line-by-line frontmatter extraction for malformed YAML.
|
|
80
|
+
*
|
|
81
|
+
* Used as a fallback when yaml.parse throws (e.g. unterminated quotes from LLM
|
|
82
|
+
* output corruption). Extracts simple `key: value` scalar pairs only — nested
|
|
83
|
+
* objects and sequences are skipped. Values that are individually parseable by
|
|
84
|
+
* yaml are normalized; otherwise stored as raw strings.
|
|
85
|
+
*/
|
|
86
|
+
function parseFrontmatterLenient(frontmatter) {
|
|
87
|
+
const data = {};
|
|
88
|
+
for (const line of frontmatter.split(/\r?\n/)) {
|
|
89
|
+
const m = line.match(/^([\w][\w-]*):\s*(.*)$/);
|
|
90
|
+
if (!m)
|
|
91
|
+
continue;
|
|
92
|
+
const key = m[1];
|
|
93
|
+
const rawValue = (m[2] ?? "").trim();
|
|
94
|
+
try {
|
|
95
|
+
const singleEntry = yamlParse(`k: ${rawValue}`);
|
|
96
|
+
if (singleEntry !== null && typeof singleEntry === "object" && !Array.isArray(singleEntry)) {
|
|
97
|
+
const v = singleEntry.k;
|
|
98
|
+
data[key] = v === null || v === undefined ? "" : v;
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
data[key] = rawValue;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
data[key] = rawValue;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return data;
|
|
186
109
|
}
|
|
187
110
|
export function parseFrontmatterBlock(raw) {
|
|
188
111
|
// Handle both LF and CRLF line endings throughout.
|
|
189
112
|
// The closing --- may be preceded by \r\n; capture and strip trailing \r
|
|
190
113
|
// from the frontmatter block so key parsing sees clean LF-terminated lines.
|
|
191
114
|
const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
|
|
192
|
-
if (
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
}
|
|
115
|
+
if (match) {
|
|
116
|
+
// Strip any \r characters from the frontmatter block to normalise CRLF → LF
|
|
117
|
+
const frontmatter = match[1].replace(/\r/g, "");
|
|
118
|
+
const content = match[2];
|
|
119
|
+
return {
|
|
120
|
+
frontmatter,
|
|
121
|
+
content,
|
|
122
|
+
bodyStartLine: countLines(raw.slice(0, match[0].length - match[2].length)) + 1,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
// Empty frontmatter (---\n---): the content-bearing regex above requires at
|
|
126
|
+
// least one character between the fences. Handle the degenerate case so
|
|
127
|
+
// callers can reconstruct `---\nkey: val\n---\n\nbody` from a previously
|
|
128
|
+
// empty-frontmatter file without corrupting it by wrapping the entire raw
|
|
129
|
+
// string as body content.
|
|
130
|
+
const emptyMatch = raw.match(/^---\r?\n---(?:\r\n|\r|\n)([\s\S]*)$/);
|
|
131
|
+
if (emptyMatch) {
|
|
132
|
+
return { frontmatter: "", content: emptyMatch[1], bodyStartLine: 3 };
|
|
133
|
+
}
|
|
134
|
+
return null;
|
|
202
135
|
}
|
|
203
136
|
function countLines(text) {
|
|
204
137
|
if (text.length === 0)
|
|
@@ -206,7 +139,13 @@ function countLines(text) {
|
|
|
206
139
|
return text.split(/\r?\n/).length - 1;
|
|
207
140
|
}
|
|
208
141
|
/**
|
|
209
|
-
* Parse a
|
|
142
|
+
* Parse a YAML scalar value (string, boolean, or number).
|
|
143
|
+
*
|
|
144
|
+
* For quoted strings (single or double), delegates to the `yaml` library so
|
|
145
|
+
* escape sequences are handled correctly per spec. The previous hand-rolled
|
|
146
|
+
* `slice(1, -1)` only stripped one layer of quoting and left inner quotes and
|
|
147
|
+
* escape sequences as literal characters in the stored value, causing visible
|
|
148
|
+
* corruption when `yaml.stringify` re-quoted them on the next write.
|
|
210
149
|
*/
|
|
211
150
|
export function parseYamlScalar(value) {
|
|
212
151
|
if (value === "")
|
|
@@ -219,7 +158,67 @@ export function parseYamlScalar(value) {
|
|
|
219
158
|
if (!Number.isNaN(asNumber))
|
|
220
159
|
return asNumber;
|
|
221
160
|
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
|
161
|
+
try {
|
|
162
|
+
const parsed = yamlParse(value);
|
|
163
|
+
if (typeof parsed === "string")
|
|
164
|
+
return parsed;
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
// Fall through to raw slice on malformed YAML — better than throwing.
|
|
168
|
+
}
|
|
222
169
|
return value.slice(1, -1);
|
|
223
170
|
}
|
|
224
171
|
return value;
|
|
225
172
|
}
|
|
173
|
+
// ── Minimum score delta to trigger a frontmatter salience rewrite ─────────────
|
|
174
|
+
const SALIENCE_WRITE_DELTA_THRESHOLD = 0.05;
|
|
175
|
+
/**
|
|
176
|
+
* Idempotently write `salience` and `salienceInputs` fields into the YAML
|
|
177
|
+
* frontmatter of a raw asset string.
|
|
178
|
+
*
|
|
179
|
+
* Skips the write when the existing `salience` field differs from `score` by
|
|
180
|
+
* less than {@link SALIENCE_WRITE_DELTA_THRESHOLD}, to avoid churn for minor
|
|
181
|
+
* floating-point drift. Returns the raw string unchanged when no write is needed
|
|
182
|
+
* or when no frontmatter block is present.
|
|
183
|
+
*
|
|
184
|
+
* The `salienceInputs` field is written for auditability only; no pipeline code
|
|
185
|
+
* reads it back. `state.db :: asset_salience` is the canonical store.
|
|
186
|
+
*/
|
|
187
|
+
export function writeSalienceToFrontmatter(raw, score, inputs) {
|
|
188
|
+
const parsed = parseFrontmatterBlock(raw);
|
|
189
|
+
if (!parsed)
|
|
190
|
+
return raw;
|
|
191
|
+
const existingData = parseFrontmatter(raw).data;
|
|
192
|
+
const existingSalience = typeof existingData.salience === "number" ? existingData.salience : undefined;
|
|
193
|
+
if (existingSalience !== undefined && Math.abs(existingSalience - score) < SALIENCE_WRITE_DELTA_THRESHOLD) {
|
|
194
|
+
return raw;
|
|
195
|
+
}
|
|
196
|
+
// Parse existing frontmatter into an object, then set/overwrite salience fields.
|
|
197
|
+
let fm = {};
|
|
198
|
+
if (parsed.frontmatter.trim()) {
|
|
199
|
+
try {
|
|
200
|
+
const p = yamlParse(parsed.frontmatter);
|
|
201
|
+
if (p !== null && typeof p === "object" && !Array.isArray(p)) {
|
|
202
|
+
fm = p;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
catch {
|
|
206
|
+
// Malformed YAML — rebuild from best-effort parse
|
|
207
|
+
fm = parseFrontmatterLenient(parsed.frontmatter);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
fm.salience = roundTo2dp(score);
|
|
211
|
+
fm.salienceInputs = {
|
|
212
|
+
novelty: roundTo2dp(inputs.novelty),
|
|
213
|
+
magnitude: roundTo2dp(inputs.magnitude),
|
|
214
|
+
predictionError: roundTo2dp(inputs.predictionError),
|
|
215
|
+
};
|
|
216
|
+
const newFrontmatter = yamlStringify(fm).trimEnd();
|
|
217
|
+
const body = parsed.content;
|
|
218
|
+
// Preserve original line ending style between frontmatter and body
|
|
219
|
+
const separator = body.startsWith("\n") ? "" : "\n";
|
|
220
|
+
return `---\n${newFrontmatter}\n---\n${separator}${body}`;
|
|
221
|
+
}
|
|
222
|
+
function roundTo2dp(n) {
|
|
223
|
+
return Math.round(n * 100) / 100;
|
|
224
|
+
}
|
|
@@ -8,7 +8,15 @@ export function parseMarkdownToc(content) {
|
|
|
8
8
|
const headings = [];
|
|
9
9
|
const parsed = parseFrontmatter(content);
|
|
10
10
|
const start = parsed.frontmatter ? parsed.bodyStartLine - 1 : 0;
|
|
11
|
+
let inFence = false;
|
|
11
12
|
for (let i = start; i < lines.length; i++) {
|
|
13
|
+
// Track fenced code blocks (``` or ~~~) so headings inside them are skipped.
|
|
14
|
+
if (/^\s*(`{3,}|~{3,})/.test(lines[i])) {
|
|
15
|
+
inFence = !inFence;
|
|
16
|
+
continue;
|
|
17
|
+
}
|
|
18
|
+
if (inFence)
|
|
19
|
+
continue;
|
|
12
20
|
const match = lines[i].match(/^(#{1,6})\s+(.+)$/);
|
|
13
21
|
if (match) {
|
|
14
22
|
headings.push({
|
|
@@ -133,6 +133,34 @@ export const ImproveProcessConfigSchema = z
|
|
|
133
133
|
// consolidation pass skips entirely (emits `pool_below_min_size`). 0 disables
|
|
134
134
|
// the guard. Only meaningful on the `consolidate` process. Default 500.
|
|
135
135
|
minPoolSize: z.number().int().min(0).optional(),
|
|
136
|
+
// Consolidate process: deterministic near-duplicate dedup pre-pass (#617).
|
|
137
|
+
// A cheap, no-LLM fast path that collapses obvious duplicates (`.derived`
|
|
138
|
+
// origin pairs + content twins) before the LLM consolidation. Default OFF
|
|
139
|
+
// — when absent the consolidate pass behaves byte-identically to today.
|
|
140
|
+
// `cosineThreshold` is a strict floor in [0, 1] (default 0.97) for the
|
|
141
|
+
// optional embedding-similarity match; exact normalized content-hash
|
|
142
|
+
// equality always collapses regardless of the threshold. Only meaningful
|
|
143
|
+
// on the `consolidate` process.
|
|
144
|
+
dedup: z
|
|
145
|
+
.object({
|
|
146
|
+
enabled: z.boolean().optional(),
|
|
147
|
+
cosineThreshold: z.number().min(0).max(1).optional(),
|
|
148
|
+
// WS-3a: maximum pool size for the O(n²) cosine-similarity twin compare.
|
|
149
|
+
// Only the first `cosineCandidateLimit` memories are cosine-compared;
|
|
150
|
+
// exact-hash matches still run over the full pool. Default 500. Raise
|
|
151
|
+
// with care — cost is O(n²).
|
|
152
|
+
cosineCandidateLimit: z.number().int().positive().optional(),
|
|
153
|
+
})
|
|
154
|
+
.strict()
|
|
155
|
+
.optional(),
|
|
156
|
+
// Consolidate process: judged-state cache (#581). When enabled, a memory
|
|
157
|
+
// whose current content hash equals its cached judged hash is SKIPPED from
|
|
158
|
+
// the LLM pool (judged-unchanged → no re-judge), letting one run sweep the
|
|
159
|
+
// whole corpus at O(changed/new) cost instead of narrowing to a recent
|
|
160
|
+
// time-window slice. Default OFF — when absent the consolidate pass behaves
|
|
161
|
+
// byte-identically to today (the incrementalSince path is unaffected). Only
|
|
162
|
+
// meaningful on the `consolidate` process.
|
|
163
|
+
judgedCache: z.object({ enabled: z.boolean().optional() }).strict().optional(),
|
|
136
164
|
qualityGate: z.object({ enabled: z.boolean().optional() }).strict().optional(),
|
|
137
165
|
contradictionDetection: z.object({ enabled: z.boolean().optional() }).strict().optional(),
|
|
138
166
|
// Extract process config (only meaningful for extract process)
|
|
@@ -162,9 +190,6 @@ export const ImproveProcessConfigSchema = z
|
|
|
162
190
|
// proactiveMaintenance process: top-N bound per run (default 25). Alias for
|
|
163
191
|
// `limit`; `maxPerRun` wins when both are set.
|
|
164
192
|
maxPerRun: positiveInt.optional(),
|
|
165
|
-
// proactiveMaintenance process: optional per-type importance overrides,
|
|
166
|
-
// merged over the built-in defaults. Only meaningful on `proactiveMaintenance`.
|
|
167
|
-
importanceWeights: z.record(z.string().min(1), z.number()).optional(),
|
|
168
193
|
// MemoryInference process: minimum pending memory count to run the pass.
|
|
169
194
|
minPendingCount: z.number().int().min(0).optional(),
|
|
170
195
|
// Extract process: minimum number of new (unseen, in-window) candidate
|
|
@@ -182,6 +207,114 @@ export const ImproveProcessConfigSchema = z
|
|
|
182
207
|
// #561 — minimum session duration in minutes for session indexing. 0
|
|
183
208
|
// disables the gate. Absent = default 5. Only meaningful on `extract`.
|
|
184
209
|
minSessionDuration: z.number().min(0).optional(),
|
|
210
|
+
// Consolidate process: fallback p90 wall-clock time per consolidation chunk
|
|
211
|
+
// in seconds, used for cold-start budget estimation when no telemetry
|
|
212
|
+
// history exists. The actual p90 is derived from observed run durations
|
|
213
|
+
// once sufficient history accumulates; this value is only used on the very
|
|
214
|
+
// first run. Default 30 s. Only meaningful on the `consolidate` process.
|
|
215
|
+
p90ChunkSecondsDefault: z.number().finite().positive().optional(),
|
|
216
|
+
// WS-3b: Homeostatic demotion (step 0a). Before any LLM merge, demote
|
|
217
|
+
// retrievalSalience for stale/low-value assets so the merge pool is bounded
|
|
218
|
+
// and high-SNR. Demotion is state.db-only (file content untouched);
|
|
219
|
+
// re-promotable on re-retrieval. Default OFF. Only meaningful on the
|
|
220
|
+
// `consolidate` process.
|
|
221
|
+
homeostaticDemotion: z
|
|
222
|
+
.object({
|
|
223
|
+
enabled: z.boolean().optional(),
|
|
224
|
+
// Minimum days since last retrieval to consider an asset stale (default 30).
|
|
225
|
+
staleDays: z.number().int().min(0).optional(),
|
|
226
|
+
// Demotion factor: multiply retrievalSalience by this when stale (default 0.5).
|
|
227
|
+
demotionFactor: z.number().min(0).max(1).optional(),
|
|
228
|
+
})
|
|
229
|
+
.strict()
|
|
230
|
+
.optional(),
|
|
231
|
+
// WS-3b: Schema-similarity gate (step 0b). At intake, if a new candidate's
|
|
232
|
+
// body embedding is within epsilon of an existing derived-layer lesson/knowledge
|
|
233
|
+
// node, mark it schema-consistent and lower its priority. Default OFF.
|
|
234
|
+
// Only meaningful on the `consolidate` and `extract` processes.
|
|
235
|
+
schemaSimilarity: z
|
|
236
|
+
.object({
|
|
237
|
+
enabled: z.boolean().optional(),
|
|
238
|
+
// Epsilon: cosine similarity threshold above which a candidate is schema-consistent
|
|
239
|
+
// (default 0.85 — looser than dedup's 0.97 since we want to catch conceptual overlap).
|
|
240
|
+
epsilon: z.number().min(0).max(1).optional(),
|
|
241
|
+
// Multiplicative factor applied to candidate confidence when schema-consistent.
|
|
242
|
+
// Default 0.5 — halves the confidence so schema-consistent candidates are less likely
|
|
243
|
+
// to pass the quality gate and create redundant stash entries.
|
|
244
|
+
confidencePenalty: z.number().min(0).max(1).optional(),
|
|
245
|
+
})
|
|
246
|
+
.strict()
|
|
247
|
+
.optional(),
|
|
248
|
+
// WS-3b: Hot-probation intake buffer (step 0c, #604). New system-generated
|
|
249
|
+
// extractions enter captureMode: hot-probation and spend ONE consolidation
|
|
250
|
+
// cycle in probation. Dedup + quality second-pass runs before promotion.
|
|
251
|
+
// Default OFF. Only meaningful on the `extract` process.
|
|
252
|
+
hotProbation: z
|
|
253
|
+
.object({
|
|
254
|
+
enabled: z.boolean().optional(),
|
|
255
|
+
})
|
|
256
|
+
.strict()
|
|
257
|
+
.optional(),
|
|
258
|
+
// WS-3b: Anti-collapse guards (step 8). Prevents the consolidation pipeline
|
|
259
|
+
// from collapsing too aggressively and losing diversity.
|
|
260
|
+
// - maxGeneration: refuse to merge two assets both above this generation (default 2).
|
|
261
|
+
// - lexicalDiversityCheck: low n-gram diversity ⇒ raise merge threshold.
|
|
262
|
+
// - randomClusterFraction: occasional random (non-similar) cluster in pool (default 0.05).
|
|
263
|
+
// Default OFF. Only meaningful on the `consolidate` process.
|
|
264
|
+
antiCollapse: z
|
|
265
|
+
.object({
|
|
266
|
+
enabled: z.boolean().optional(),
|
|
267
|
+
maxGeneration: z.number().int().min(1).optional(),
|
|
268
|
+
lexicalDiversityCheck: z.boolean().optional(),
|
|
269
|
+
randomClusterFraction: z.number().min(0).max(1).optional(),
|
|
270
|
+
})
|
|
271
|
+
.strict()
|
|
272
|
+
.optional(),
|
|
273
|
+
// WS-3b: CLS (Complementary Learning System) interleaving (step 9).
|
|
274
|
+
// distill/memoryInference prompts include embedding-retrieved existing adjacent
|
|
275
|
+
// lessons/knowledge to prevent catastrophic interference with prior generalizations.
|
|
276
|
+
// Default OFF. Only meaningful on `distill` and `memoryInference` processes.
|
|
277
|
+
cls: z
|
|
278
|
+
.object({
|
|
279
|
+
enabled: z.boolean().optional(),
|
|
280
|
+
// Number of adjacent lessons/knowledge to include in prompts (default 3).
|
|
281
|
+
adjacentCount: z.number().int().min(1).optional(),
|
|
282
|
+
})
|
|
283
|
+
.strict()
|
|
284
|
+
.optional(),
|
|
285
|
+
// WS-3b: Distill→source fidelity check (step 10). After a distill proposal,
|
|
286
|
+
// check it against its cited source memories; a contradiction flag forces
|
|
287
|
+
// human review. Default OFF. Only meaningful on `distill` process.
|
|
288
|
+
fidelityCheck: z
|
|
289
|
+
.object({
|
|
290
|
+
enabled: z.boolean().optional(),
|
|
291
|
+
})
|
|
292
|
+
.strict()
|
|
293
|
+
.optional(),
|
|
294
|
+
// #609 — recombine process: minimum related-memory cluster size before an
|
|
295
|
+
// LLM generalization call. Default 3. Only meaningful on `recombine`.
|
|
296
|
+
minClusterSize: z.number().int().min(2).optional(),
|
|
297
|
+
// #609 — recombine process: hard cap on clusters processed per run (one
|
|
298
|
+
// bounded LLM call each). Default 5. Only meaningful on `recombine`.
|
|
299
|
+
maxClustersPerRun: positiveInt.optional(),
|
|
300
|
+
// #609 — recombine process: relatedness signal used to form clusters
|
|
301
|
+
// (tags | graph | both). Clustering is by relatedness, never embedding
|
|
302
|
+
// similarity. Default "tags". Only meaningful on `recombine`.
|
|
303
|
+
relatednessSource: z.enum(["tags", "graph", "both"]).optional(),
|
|
304
|
+
// #609 — recombine process: consecutive re-inductions required before a
|
|
305
|
+
// hypothesis is promoted to a lesson. Default 2. Only meaningful on
|
|
306
|
+
// `recombine`.
|
|
307
|
+
confirmThreshold: z.number().int().min(1).optional(),
|
|
308
|
+
// #615 — procedural process: minimum number of distinct assets sharing the
|
|
309
|
+
// same successful normalized ordered-action sequence before it is compiled
|
|
310
|
+
// into a workflow proposal. Default 3. Only meaningful on `procedural`.
|
|
311
|
+
minRecurrence: z.number().int().min(2).optional(),
|
|
312
|
+
// #615 — procedural process: hard cap on workflow proposals emitted per run
|
|
313
|
+
// (one bounded LLM call each). Default 3. Only meaningful on `procedural`.
|
|
314
|
+
maxProposalsPerRun: positiveInt.optional(),
|
|
315
|
+
// #615 — procedural process: asset type a compiled sequence is emitted as.
|
|
316
|
+
// Reserved; v1 always emits "workflow". Only meaningful on `procedural`.
|
|
317
|
+
emitAs: z.enum(["workflow", "skill"]).optional(),
|
|
185
318
|
// Triage process config (only meaningful for the `triage` process)
|
|
186
319
|
applyMode: z.enum(["queue", "promote"]).optional(),
|
|
187
320
|
policy: z.string().min(1).optional(),
|
|
@@ -208,6 +341,8 @@ const ImproveProfileProcessesSchema = z
|
|
|
208
341
|
validation: ImproveProcessConfigSchema.optional(),
|
|
209
342
|
triage: ImproveProcessConfigSchema.optional(),
|
|
210
343
|
proactiveMaintenance: ImproveProcessConfigSchema.optional(),
|
|
344
|
+
recombine: ImproveProcessConfigSchema.optional(),
|
|
345
|
+
procedural: ImproveProcessConfigSchema.optional(),
|
|
211
346
|
})
|
|
212
347
|
.passthrough()
|
|
213
348
|
.superRefine((val, ctx) => {
|
|
@@ -232,6 +367,8 @@ const ImproveProfileProcessesSchema = z
|
|
|
232
367
|
"extract",
|
|
233
368
|
"triage",
|
|
234
369
|
"proactiveMaintenance",
|
|
370
|
+
"recombine",
|
|
371
|
+
"procedural",
|
|
235
372
|
]);
|
|
236
373
|
for (const k of Object.keys(raw)) {
|
|
237
374
|
if (!allowed.has(k)) {
|
|
@@ -249,6 +386,12 @@ export const ImproveProfileConfigSchema = z
|
|
|
249
386
|
processes: ImproveProfileProcessesSchema.optional(),
|
|
250
387
|
autoAccept: nonNegativeNumber.optional(),
|
|
251
388
|
limit: positiveInt.optional(),
|
|
389
|
+
// #614 — symmetric valence weighting in the eligibility sort. When true,
|
|
390
|
+
// the attention term becomes |valence| MAGNITUDE so BOTH strong positive
|
|
391
|
+
// and strong negative feedback drive attention (utility stays dominant) and
|
|
392
|
+
// strong-signed assets are routed to a fix/reinforce lane. DEFAULT OFF —
|
|
393
|
+
// false/absent preserves the legacy negative-only ranking byte-for-byte.
|
|
394
|
+
symmetricValence: z.boolean().optional(),
|
|
252
395
|
sync: z
|
|
253
396
|
.object({
|
|
254
397
|
enabled: z.boolean().optional(),
|
|
@@ -365,6 +508,7 @@ const SearchGraphBoostSchema = z
|
|
|
365
508
|
export const SearchConfigSchema = z
|
|
366
509
|
.object({
|
|
367
510
|
minScore: nonNegativeNumber.optional(),
|
|
511
|
+
defaultExcludeTypes: z.array(nonEmptyString).optional(),
|
|
368
512
|
curateRerank: z.object({ enabled: z.boolean().optional() }).strict().optional(),
|
|
369
513
|
graphBoost: SearchGraphBoostSchema.optional(),
|
|
370
514
|
})
|
|
@@ -383,10 +527,74 @@ const ImproveUtilityDecaySchema = z
|
|
|
383
527
|
feedbackStabilityBoost: z.number().finite().min(1).optional(),
|
|
384
528
|
})
|
|
385
529
|
.strict();
|
|
530
|
+
// #612 / WS-4 — auto-accept gate calibration + bounded, opt-in per-phase
|
|
531
|
+
// threshold auto-tune. DEFAULT OFF: when absent (or `autoTune: false`) no
|
|
532
|
+
// tuning occurs, so the gate behaves byte-identically to today.
|
|
533
|
+
// WS-4 adds: per-phase persistence (state.db) + auto-tune ceiling default 85.
|
|
534
|
+
const ImproveCalibrationSchema = z
|
|
535
|
+
.object({
|
|
536
|
+
/** Master switch for the bounded threshold auto-tune. Default false (parity). */
|
|
537
|
+
autoTune: z.boolean().optional(),
|
|
538
|
+
/** Lower bound (0-100) the tuned threshold may never drop below. */
|
|
539
|
+
minThreshold: z.number().int().min(0).max(100).optional(),
|
|
540
|
+
/**
|
|
541
|
+
* Upper bound (0-100) the tuned threshold may never rise above.
|
|
542
|
+
* WS-4 default: 85 (prevents gate converging to pure exploitation).
|
|
543
|
+
*/
|
|
544
|
+
maxThreshold: z.number().int().min(0).max(100).optional(),
|
|
545
|
+
/** Maximum adjustment magnitude (points) applied in one tune step. */
|
|
546
|
+
maxStep: positiveInt.optional(),
|
|
547
|
+
/** Minimum acted-on sample count required before any adjustment. */
|
|
548
|
+
minSamples: nonNegativeNumber.optional(),
|
|
549
|
+
/** Target realized accept rate in [0, 1]. Default 0.9. */
|
|
550
|
+
targetAcceptRate: z.number().finite().min(0).max(1).optional(),
|
|
551
|
+
})
|
|
552
|
+
.strict();
|
|
553
|
+
// WS-4 — exploration budget: a fixed fraction of proposals accepted per run
|
|
554
|
+
// regardless of confidence. DEFAULT OFF.
|
|
555
|
+
const ImproveExplorationSchema = z
|
|
556
|
+
.object({
|
|
557
|
+
/**
|
|
558
|
+
* Enable the exploration budget lane. Default false (parity).
|
|
559
|
+
* When true, a fraction of proposals are accepted regardless of confidence.
|
|
560
|
+
*/
|
|
561
|
+
enabled: z.boolean().optional(),
|
|
562
|
+
/**
|
|
563
|
+
* Fraction of proposals per run to accept as exploration [0, 1].
|
|
564
|
+
* Default 0.05 (5%). Clamped to [0, 1] at read time.
|
|
565
|
+
*/
|
|
566
|
+
budgetFraction: z.number().finite().min(0).max(1).optional(),
|
|
567
|
+
})
|
|
568
|
+
.strict();
|
|
569
|
+
const ImproveSalienceSchema = z
|
|
570
|
+
.object({
|
|
571
|
+
/**
|
|
572
|
+
* WS-2 Part-V gate: enable the outcome-weight term in the salience projection.
|
|
573
|
+
* Default false (parity — WS-1 weights w_e=0.30, w_r=0.70 until Part-V confirms
|
|
574
|
+
* no regression). Set to true after running scripts/akm-eval + health report.
|
|
575
|
+
*/
|
|
576
|
+
outcomeWeightEnabled: z.boolean().optional(),
|
|
577
|
+
/**
|
|
578
|
+
* Minimum encoding salience score [0, 1] for a zero-feedback asset to be
|
|
579
|
+
* admitted to the high-salience improve lane (#608).
|
|
580
|
+
* Default 0.75. Set to 1.0 to disable the lane entirely.
|
|
581
|
+
*/
|
|
582
|
+
salienceThreshold: z.number().min(0).max(1).optional(),
|
|
583
|
+
/**
|
|
584
|
+
* Per-run additive replay budget (#610). Up to this many top-salience refs are
|
|
585
|
+
* revisited even with no reactive signal and regardless of cooldown. Additive
|
|
586
|
+
* on top of --limit. Default 0 = no replay.
|
|
587
|
+
*/
|
|
588
|
+
replayBudget: z.number().int().min(0).optional(),
|
|
589
|
+
})
|
|
590
|
+
.strict();
|
|
386
591
|
export const ImproveConfigSchema = z
|
|
387
592
|
.object({
|
|
388
593
|
utilityDecay: ImproveUtilityDecaySchema.optional(),
|
|
389
594
|
eventRetentionDays: nonNegativeNumber.optional(),
|
|
595
|
+
calibration: ImproveCalibrationSchema.optional(),
|
|
596
|
+
exploration: ImproveExplorationSchema.optional(),
|
|
597
|
+
salience: ImproveSalienceSchema.optional(),
|
|
390
598
|
})
|
|
391
599
|
.strict();
|
|
392
600
|
// ── Index / per-pass ────────────────────────────────────────────────────────
|