akm-cli 0.9.0-beta.3 → 0.9.0-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +613 -0
- package/dist/assets/prompts/consolidate-system.md +23 -0
- package/dist/assets/prompts/contradiction-judge.md +33 -0
- package/dist/assets/prompts/distill-knowledge-system.md +22 -0
- package/dist/assets/prompts/distill-lesson-system.md +36 -0
- package/dist/assets/prompts/extract-session.md +5 -1
- package/dist/assets/prompts/graph-extract-system.md +1 -0
- package/dist/assets/prompts/memory-infer-system.md +1 -0
- package/dist/assets/prompts/memory-infer-user.md +5 -0
- package/dist/assets/prompts/metadata-enhance-system.md +1 -0
- package/dist/assets/prompts/procedural-system.md +44 -0
- package/dist/assets/prompts/recombine-system.md +40 -0
- package/dist/assets/prompts/staleness-detect-system.md +6 -0
- package/dist/assets/prompts/validate-summary-judge.md +1 -0
- package/dist/assets/templates/html/health.html +281 -111
- package/dist/cli.js +14 -3
- package/dist/commands/agent/contribute-cli.js +16 -3
- package/dist/commands/feedback-cli.js +15 -6
- package/dist/commands/graph/graph.js +75 -71
- package/dist/commands/health/checks.js +48 -0
- package/dist/commands/health/html-report.js +422 -80
- package/dist/commands/health.js +381 -9
- package/dist/commands/improve/calibration.js +161 -0
- package/dist/commands/improve/consolidate.js +634 -111
- package/dist/commands/improve/dedup.js +482 -0
- package/dist/commands/improve/distill.js +145 -69
- package/dist/commands/improve/encoding-salience.js +205 -0
- package/dist/commands/improve/extract-cli.js +115 -1
- package/dist/commands/improve/extract-prompt.js +33 -2
- package/dist/commands/improve/extract-watch.js +140 -0
- package/dist/commands/improve/extract.js +244 -35
- package/dist/commands/improve/feedback-valence.js +54 -0
- package/dist/commands/improve/homeostatic.js +467 -0
- package/dist/commands/improve/improve-auto-accept.js +113 -6
- package/dist/commands/improve/improve-profiles.js +12 -0
- package/dist/commands/improve/improve.js +1974 -614
- package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
- package/dist/commands/improve/outcome-loop.js +256 -0
- package/dist/commands/improve/proactive-maintenance.js +87 -0
- package/dist/commands/improve/procedural.js +409 -0
- package/dist/commands/improve/recombine.js +593 -0
- package/dist/commands/improve/reflect.js +26 -1
- package/dist/commands/improve/related-sessions.js +120 -0
- package/dist/commands/improve/salience.js +386 -0
- package/dist/commands/improve/triage.js +95 -0
- package/dist/commands/lint/agent-linter.js +19 -24
- package/dist/commands/lint/base-linter.js +173 -60
- package/dist/commands/lint/command-linter.js +19 -24
- package/dist/commands/lint/env-key-rules.js +34 -1
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +31 -13
- package/dist/commands/lint/memory-linter.js +1 -1
- package/dist/commands/lint/registry.js +7 -2
- package/dist/commands/lint/task-linter.js +3 -3
- package/dist/commands/lint/workflow-linter.js +26 -1
- package/dist/commands/proposal/proposal.js +5 -0
- package/dist/commands/proposal/validators/proposals.js +71 -54
- package/dist/commands/read/curate.js +344 -80
- package/dist/commands/read/search-cli.js +7 -0
- package/dist/commands/read/search.js +1 -0
- package/dist/commands/read/show.js +67 -2
- package/dist/commands/sources/installed-stashes.js +5 -1
- package/dist/commands/sources/stash-cli.js +10 -2
- package/dist/core/asset/asset-registry.js +2 -0
- package/dist/core/asset/asset-spec.js +14 -0
- package/dist/core/asset/frontmatter.js +166 -167
- package/dist/core/asset/markdown.js +8 -0
- package/dist/core/config/config-schema.js +259 -2
- package/dist/core/config/config.js +2 -2
- package/dist/core/logs-db.js +4 -3
- package/dist/core/paths.js +3 -0
- package/dist/core/state-db.js +649 -30
- package/dist/indexer/db/db.js +364 -38
- package/dist/indexer/db/graph-db.js +129 -86
- package/dist/indexer/ensure-index.js +152 -17
- package/dist/indexer/graph/graph-boost.js +51 -41
- package/dist/indexer/graph/graph-extraction.js +203 -3
- package/dist/indexer/index-writer-lock.js +99 -0
- package/dist/indexer/indexer.js +114 -111
- package/dist/indexer/passes/memory-inference.js +10 -3
- package/dist/indexer/passes/staleness-detect.js +2 -5
- package/dist/indexer/search/db-search.js +15 -4
- package/dist/indexer/search/ranking-contributors.js +22 -0
- package/dist/indexer/search/ranking.js +4 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/integrations/agent/prompts.js +1 -0
- package/dist/integrations/harnesses/claude/session-log.js +11 -1
- package/dist/integrations/harnesses/opencode/session-log.js +9 -0
- package/dist/integrations/session-logs/index.js +16 -0
- package/dist/llm/client.js +23 -4
- package/dist/llm/embedder.js +27 -3
- package/dist/llm/embedders/local.js +66 -2
- package/dist/llm/graph-extract.js +2 -1
- package/dist/llm/memory-infer.js +4 -8
- package/dist/llm/metadata-enhance.js +9 -1
- package/dist/output/renderers.js +73 -1
- package/dist/output/shapes/curate.js +14 -2
- package/dist/output/text/helpers.js +9 -0
- package/dist/runtime.js +25 -1
- package/dist/scripts/migrate-storage.js +1242 -594
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +473 -270
- package/dist/sources/providers/tar-utils.js +16 -8
- package/dist/storage/sqlite-pragmas.js +146 -0
- package/dist/workflows/db.js +3 -4
- package/dist/workflows/validate-summary.js +2 -7
- package/docs/data-and-telemetry.md +1 -0
- package/package.json +9 -6
|
@@ -22,18 +22,22 @@ import { parseAssetRef } from "../../core/asset/asset-ref.js";
|
|
|
22
22
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
23
23
|
import { META_DIR, parseMetaRef, resolveMetaFilePath } from "../../core/asset/stash-meta.js";
|
|
24
24
|
import { asNonEmptyString } from "../../core/common.js";
|
|
25
|
-
import { loadConfig } from "../../core/config/config.js";
|
|
25
|
+
import { getIndexPassConfig, loadConfig } from "../../core/config/config.js";
|
|
26
26
|
import { NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
27
27
|
import { appendEvent, readEvents } from "../../core/events.js";
|
|
28
|
-
import { findEntryIdByRef } from "../../indexer/db/db.js";
|
|
28
|
+
import { closeDatabase, computeBodyHash, findEntryIdByRef, openExistingDatabase } from "../../indexer/db/db.js";
|
|
29
|
+
import { hasGraphData } from "../../indexer/db/graph-db.js";
|
|
29
30
|
import { ensureIndex } from "../../indexer/ensure-index.js";
|
|
30
31
|
import { listRelatedPathsForFile } from "../../indexer/graph/graph-boost.js";
|
|
32
|
+
import { extractGraphForSingleFile } from "../../indexer/graph/graph-extraction.js";
|
|
31
33
|
import { lookup } from "../../indexer/indexer.js";
|
|
32
34
|
import { buildEditHint, findSourceForPath, isEditable, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
33
35
|
import { insertUsageEvent } from "../../indexer/usage/usage-events.js";
|
|
34
36
|
import { buildFileContext, buildRenderContext, getRenderer, runMatchers } from "../../indexer/walk/file-context.js";
|
|
35
37
|
import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
|
|
38
|
+
import { resolveIndexPassLLM } from "../../llm/index-passes.js";
|
|
36
39
|
import { resolveSourcesForOrigin } from "../../registry/origin-resolve.js";
|
|
40
|
+
import { resolveStorageLocations } from "../../storage/locations.js";
|
|
37
41
|
import { withIndexDb } from "../../storage/repositories/index-db.js";
|
|
38
42
|
// Eagerly import source providers to trigger self-registration.
|
|
39
43
|
import "../../sources/providers/index.js";
|
|
@@ -383,6 +387,15 @@ export async function showLocal(input) {
|
|
|
383
387
|
if (activeRun) {
|
|
384
388
|
fullResponse.activeRun = activeRun;
|
|
385
389
|
}
|
|
390
|
+
// #624-P3: opt-in inline graph extraction. Default OFF — when the flag is
|
|
391
|
+
// unset this whole block is skipped (no hasGraphData check, no LLM call), so
|
|
392
|
+
// behavior is byte-identical to today. When ON, it extracts graph data for an
|
|
393
|
+
// ungraphed asset, but ONLY when a model is configured (model-available
|
|
394
|
+
// guard) and ALWAYS bounded by a 30s timeout so `show` can never hang. Any
|
|
395
|
+
// timeout/model-unavailable/error path returns the response unchanged.
|
|
396
|
+
if (getIndexPassConfig(config.index, "graph")?.lazyGraphExtraction === true) {
|
|
397
|
+
await maybeExtractGraphInline(config, sourceStashDir, assetPath);
|
|
398
|
+
}
|
|
386
399
|
if (input.detail === "brief") {
|
|
387
400
|
return buildBriefResponse(fullResponse, assetPath);
|
|
388
401
|
}
|
|
@@ -391,6 +404,58 @@ export async function showLocal(input) {
|
|
|
391
404
|
}
|
|
392
405
|
return fullResponse;
|
|
393
406
|
}
|
|
407
|
+
/**
|
|
408
|
+
* #624-P3 — opt-in inline graph extraction for `akm show`. Best-effort and
|
|
409
|
+
* timeout-bounded: never throws, never hangs, never mutates the response.
|
|
410
|
+
*
|
|
411
|
+
* Preconditions (caller already checked the flag): a model must be configured
|
|
412
|
+
* (model-available guard via {@link resolveIndexPassLLM}) and the asset must be
|
|
413
|
+
* ungraphed ({@link hasGraphData}). Extraction races a 30s timeout so `show`
|
|
414
|
+
* cannot block on a slow provider; any timeout/error/missing-model path is
|
|
415
|
+
* swallowed and `show` returns its already-assembled response unchanged.
|
|
416
|
+
*/
|
|
417
|
+
async function maybeExtractGraphInline(config, sourceStashDir, assetPath) {
|
|
418
|
+
try {
|
|
419
|
+
// Model-available guard — no provider configured ⇒ silent skip, no LLM call.
|
|
420
|
+
if (!resolveIndexPassLLM("graph", config))
|
|
421
|
+
return;
|
|
422
|
+
let alreadyGraphed = false;
|
|
423
|
+
let bodyHash;
|
|
424
|
+
try {
|
|
425
|
+
const raw = fs.readFileSync(assetPath, "utf8");
|
|
426
|
+
bodyHash = computeBodyHash(parseFrontmatter(raw).content.trim());
|
|
427
|
+
}
|
|
428
|
+
catch {
|
|
429
|
+
return; // file gone/unreadable ⇒ nothing to extract
|
|
430
|
+
}
|
|
431
|
+
withIndexDb((db) => {
|
|
432
|
+
alreadyGraphed = hasGraphData(db, sourceStashDir, assetPath);
|
|
433
|
+
});
|
|
434
|
+
if (alreadyGraphed)
|
|
435
|
+
return;
|
|
436
|
+
// Open the db for the async extraction ourselves: `withIndexDb` is
|
|
437
|
+
// synchronous and would close the connection the instant the async fn
|
|
438
|
+
// returns its Promise (before extraction completes). Close it explicitly
|
|
439
|
+
// after the race settles instead.
|
|
440
|
+
const db = openExistingDatabase(resolveStorageLocations().indexDb);
|
|
441
|
+
let timer;
|
|
442
|
+
const timeout = new Promise((resolve) => {
|
|
443
|
+
timer = setTimeout(resolve, 30_000);
|
|
444
|
+
});
|
|
445
|
+
try {
|
|
446
|
+
await Promise.race([extractGraphForSingleFile(db, sourceStashDir, assetPath, bodyHash, { config }), timeout]);
|
|
447
|
+
}
|
|
448
|
+
finally {
|
|
449
|
+
if (timer)
|
|
450
|
+
clearTimeout(timer);
|
|
451
|
+
closeDatabase(db);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
catch (err) {
|
|
455
|
+
rethrowIfTestIsolationError(err);
|
|
456
|
+
// Any other failure: silently return the unchanged show response.
|
|
457
|
+
}
|
|
458
|
+
}
|
|
394
459
|
/**
|
|
395
460
|
* Minimal `show`: ref → indexer lookup → file contents. Used by callers that
|
|
396
461
|
* just need the raw file (e.g. clone, write-source) and don't want the full
|
|
@@ -205,7 +205,11 @@ async function updateRegistryEntry(entry, force) {
|
|
|
205
205
|
const synced = await syncFromRef(entry.ref, { force });
|
|
206
206
|
const installedEntry = {
|
|
207
207
|
id: synced.id,
|
|
208
|
-
source
|
|
208
|
+
// Preserve the original source classification. syncFromRef() re-derives the
|
|
209
|
+
// source type from the ref scheme (e.g. "github:" → source: "github"), but
|
|
210
|
+
// an update should not reclassify an existing entry. A writable entry stored
|
|
211
|
+
// as source: "git" would fail config validation if rewritten to "github".
|
|
212
|
+
source: entry.source,
|
|
209
213
|
ref: synced.ref,
|
|
210
214
|
artifactUrl: synced.artifactUrl,
|
|
211
215
|
resolvedVersion: synced.resolvedVersion,
|
|
@@ -71,6 +71,11 @@ export const indexCommand = defineCommand({
|
|
|
71
71
|
description: "When combined with --clean, report stale entries without deleting them.",
|
|
72
72
|
default: false,
|
|
73
73
|
},
|
|
74
|
+
background: {
|
|
75
|
+
type: "boolean",
|
|
76
|
+
description: "Run as a background process (suppresses interactive output, manages PID file).",
|
|
77
|
+
default: false,
|
|
78
|
+
},
|
|
74
79
|
},
|
|
75
80
|
async run({ args }) {
|
|
76
81
|
await runWithJsonErrors(async () => {
|
|
@@ -80,6 +85,7 @@ export const indexCommand = defineCommand({
|
|
|
80
85
|
if (getHyphenatedBoolean(args, "re-enrich") || parseFlagValue(process.argv, "--re-enrich") !== undefined) {
|
|
81
86
|
throw new UsageError("`akm index --re-enrich` has been removed. Re-enrichment of index-time LLM passes is not exposed in this slice.");
|
|
82
87
|
}
|
|
88
|
+
const isBackground = args.background === true;
|
|
83
89
|
const outputMode = getOutputMode();
|
|
84
90
|
const controller = new AbortController();
|
|
85
91
|
const abort = () => controller.abort(new Error("index interrupted"));
|
|
@@ -88,7 +94,7 @@ export const indexCommand = defineCommand({
|
|
|
88
94
|
const indexLogFile = path.join(getCacheDir(), "logs", "index", `${new Date().toISOString().replace(/[:.]/g, "-")}.log`);
|
|
89
95
|
setLogFile(indexLogFile);
|
|
90
96
|
const verbose = isVerbose();
|
|
91
|
-
const spin = !verbose && outputMode.format === "text" ? p.spinner() : null;
|
|
97
|
+
const spin = !verbose && !isBackground && outputMode.format === "text" ? p.spinner() : null;
|
|
92
98
|
if (spin) {
|
|
93
99
|
spin.start(`Building search index${args.full ? " (full rebuild)" : ""}...`);
|
|
94
100
|
}
|
|
@@ -114,7 +120,9 @@ export const indexCommand = defineCommand({
|
|
|
114
120
|
if (spin) {
|
|
115
121
|
spin.stop(`Indexed ${result.totalEntries} assets.`);
|
|
116
122
|
}
|
|
117
|
-
|
|
123
|
+
if (!isBackground) {
|
|
124
|
+
output("index", result);
|
|
125
|
+
}
|
|
118
126
|
}
|
|
119
127
|
catch (error) {
|
|
120
128
|
if (spin) {
|
|
@@ -29,6 +29,7 @@ export const TYPE_TO_RENDERER = {
|
|
|
29
29
|
wiki: "wiki-md",
|
|
30
30
|
task: "task-yaml",
|
|
31
31
|
session: "session-md",
|
|
32
|
+
fact: "fact-md",
|
|
32
33
|
};
|
|
33
34
|
/** Map asset types to action builder functions for search results. */
|
|
34
35
|
export const ACTION_BUILDERS = {
|
|
@@ -45,6 +46,7 @@ export const ACTION_BUILDERS = {
|
|
|
45
46
|
wiki: (ref) => `akm show ${ref} -> read the wiki page`,
|
|
46
47
|
task: (ref) => `akm tasks show ${ref.replace(/^task:/, "")} -> inspect; akm tasks run <id> -> run now; akm tasks remove <id> -> unschedule`,
|
|
47
48
|
session: (ref) => `akm show ${ref} -> read the session summary; follow the \`access\` frontmatter to open the raw log at \`log_path\``,
|
|
49
|
+
fact: (ref) => `akm show ${ref} -> read the stash fact and apply it as durable context`,
|
|
48
50
|
};
|
|
49
51
|
/**
|
|
50
52
|
* Register a type-to-renderer mapping.
|
|
@@ -154,6 +154,20 @@ const ASSET_SPECS_INTERNAL = {
|
|
|
154
154
|
rendererName: "session-md",
|
|
155
155
|
actionBuilder: (ref) => `akm show ${ref} -> read the session summary; follow the \`access\` frontmatter to open the raw log at \`log_path\``,
|
|
156
156
|
},
|
|
157
|
+
// Durable stash-level semantic knowledge — facts about the user, team, or
|
|
158
|
+
// project (personal details, team tool stacks, coding conventions /
|
|
159
|
+
// "constitution", and stash-meta like naming conventions or the active
|
|
160
|
+
// projects list). Unlike `memory` (episodic, recency-decayed) these are
|
|
161
|
+
// mostly-static declarations meant to be reliably surfaced as context. A
|
|
162
|
+
// plain markdown spec; `category` frontmatter scopes the fact and
|
|
163
|
+
// `pinned: true` marks the small always-injected core. See
|
|
164
|
+
// docs/design/fact-asset-type.md.
|
|
165
|
+
fact: {
|
|
166
|
+
stashDir: "facts",
|
|
167
|
+
...markdownSpec,
|
|
168
|
+
rendererName: "fact-md",
|
|
169
|
+
actionBuilder: (ref) => `akm show ${ref} -> read the stash fact and apply it as durable context`,
|
|
170
|
+
},
|
|
157
171
|
};
|
|
158
172
|
export const ASSET_SPECS = ASSET_SPECS_INTERNAL;
|
|
159
173
|
/**
|
|
@@ -4,170 +4,43 @@
|
|
|
4
4
|
/**
|
|
5
5
|
* Shared frontmatter parsing utilities.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
7
|
+
* Uses the `yaml` library for all YAML parsing so that the full YAML spec
|
|
8
|
+
* (block scalars, multi-line strings, nested objects, flow sequences, escape
|
|
9
|
+
* sequences) is handled correctly without a brittle hand-rolled state machine.
|
|
9
10
|
*/
|
|
11
|
+
import { parse as yamlParse, stringify as yamlStringify } from "yaml";
|
|
10
12
|
/**
|
|
11
|
-
* Parse YAML
|
|
13
|
+
* Parse YAML frontmatter from a Markdown (or similar) string.
|
|
12
14
|
*
|
|
13
15
|
* Returns the parsed key-value data and the remaining body content.
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
* - **Top-level values**: string, boolean, and number scalars are supported,
|
|
18
|
-
* as well as top-level list-valued keys using YAML block sequences
|
|
19
|
-
* (`- item`) or flow arrays (`[a, b, c]`).
|
|
20
|
-
* - **List item types**: list items must be scalar values and may be strings,
|
|
21
|
-
* booleans, or numbers.
|
|
22
|
-
* - **No nested objects beyond one level**: Only a single level of indented
|
|
23
|
-
* key-value pairs is supported.
|
|
24
|
-
* - **Block scalars**: `|` (literal), `|-` (strip), and `|+` (keep) block
|
|
25
|
-
* scalars are supported for multi-line string values as emitted by the
|
|
26
|
-
* `yaml` library's `stringify`.
|
|
16
|
+
* Delegates all YAML parsing to the `yaml` library; the only responsibility
|
|
17
|
+
* of this function is extracting the `---…---` block and normalizing the
|
|
18
|
+
* parsed result (e.g. converting YAML timestamp values to ISO date strings).
|
|
27
19
|
*/
|
|
28
20
|
export function parseFrontmatter(raw) {
|
|
29
21
|
const parsedBlock = parseFrontmatterBlock(raw);
|
|
30
22
|
if (!parsedBlock) {
|
|
31
23
|
return { data: {}, content: raw, frontmatter: null, bodyStartLine: 1 };
|
|
32
24
|
}
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
/** Lines collected while in "block" mode. */
|
|
44
|
-
let blockLines = null;
|
|
45
|
-
/** Block scalar chomping: "clip" (|), "strip" (|-), "keep" (|+). */
|
|
46
|
-
let blockChomping = "clip";
|
|
47
|
-
const flushPending = () => {
|
|
48
|
-
// Called when we start a new top-level key and the previous key was still "pending".
|
|
49
|
-
// An empty-value key followed by another top-level key means it was an empty scalar.
|
|
50
|
-
if (mode === "pending" && currentKey !== null) {
|
|
51
|
-
data[currentKey] = "";
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
const flushBlock = () => {
|
|
55
|
-
// Commit the accumulated block-scalar lines to `data[currentKey]`.
|
|
56
|
-
if (mode !== "block" || currentKey === null || blockLines === null)
|
|
57
|
-
return;
|
|
58
|
-
// De-indent: strip the common 2-space prefix `yaml.stringify` emits.
|
|
59
|
-
const deindented = blockLines.map((l) => (l.startsWith(" ") ? l.slice(2) : l));
|
|
60
|
-
// Chomping: apply trailing-newline policy.
|
|
61
|
-
// "clip" (|): single trailing newline.
|
|
62
|
-
// "strip" (|-): no trailing newline.
|
|
63
|
-
// "keep" (|+): keep all trailing newlines as-is.
|
|
64
|
-
if (blockChomping === "keep") {
|
|
65
|
-
data[currentKey] = deindented.join("\n");
|
|
66
|
-
}
|
|
67
|
-
else if (blockChomping === "strip") {
|
|
68
|
-
data[currentKey] = deindented.join("\n").replace(/\n+$/, "");
|
|
69
|
-
}
|
|
70
|
-
else {
|
|
71
|
-
// "clip": exactly one trailing newline
|
|
72
|
-
data[currentKey] = `${deindented.join("\n").replace(/\n+$/, "")}\n`;
|
|
73
|
-
}
|
|
74
|
-
};
|
|
75
|
-
for (const line of parsedBlock.frontmatter.split(/\r?\n/)) {
|
|
76
|
-
// If we are in block-scalar mode, collect indented lines or end the block.
|
|
77
|
-
if (mode === "block") {
|
|
78
|
-
if (line.startsWith(" ") || line === "") {
|
|
79
|
-
// Continuation of the block scalar (indented content or blank line).
|
|
80
|
-
blockLines.push(line);
|
|
81
|
-
continue;
|
|
25
|
+
let data = {};
|
|
26
|
+
if (parsedBlock.frontmatter.trim()) {
|
|
27
|
+
try {
|
|
28
|
+
const parsed = yamlParse(parsedBlock.frontmatter);
|
|
29
|
+
if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
30
|
+
// Normalize Date objects: the yaml "core" schema parses YYYY-MM-DD
|
|
31
|
+
// literals as JS Date instances. Convert them back to ISO date strings
|
|
32
|
+
// to preserve the string type that callers (and yaml.stringify on write)
|
|
33
|
+
// expect.
|
|
34
|
+
data = normalizeYamlValues(parsed);
|
|
82
35
|
}
|
|
83
|
-
// Non-indented line ends the block scalar — flush and fall through to
|
|
84
|
-
// parse the new line as a top-level key.
|
|
85
|
-
flushBlock();
|
|
86
|
-
mode = "scalar";
|
|
87
|
-
blockLines = null;
|
|
88
36
|
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
// First block-sequence item after an empty-value key — switch to list mode
|
|
95
|
-
currentList = [];
|
|
96
|
-
data[currentKey] = currentList;
|
|
97
|
-
mode = "list";
|
|
98
|
-
}
|
|
99
|
-
currentList.push(parseYamlScalar(seqItem[1].trim()));
|
|
100
|
-
continue;
|
|
101
|
-
}
|
|
102
|
-
// Plain-style multi-line scalar continuation: a 2-space-indented line that
|
|
103
|
-
// is not a sequence item or nested key. YAML plain scalars fold newlines
|
|
104
|
-
// into a single space, so we append with a space. This handles LLM-emitted
|
|
105
|
-
// descriptions like:
|
|
106
|
-
// description: Use 4-colon outer containers when mixing
|
|
107
|
-
// nesting depths in markdown-it-container plugins.
|
|
108
|
-
// Without this, only the first line is captured and the truncation
|
|
109
|
-
// heuristic wrongly flags it as cut off mid-sentence.
|
|
110
|
-
if (mode === "scalar" && currentKey !== null && /^ {2}\S/.test(line)) {
|
|
111
|
-
data[currentKey] = `${String(data[currentKey])} ${line.trim()}`;
|
|
112
|
-
continue;
|
|
113
|
-
}
|
|
114
|
-
// Indented nested key-value (object under a key with empty value)
|
|
115
|
-
const indented = line.match(/^ {2}(\w[\w-]*):\s*(.+)$/);
|
|
116
|
-
if (indented && currentKey !== null && (mode === "object" || mode === "pending")) {
|
|
117
|
-
if (mode === "pending") {
|
|
118
|
-
// First indented k-v after an empty-value key — switch to object mode
|
|
119
|
-
nested = {};
|
|
120
|
-
data[currentKey] = nested;
|
|
121
|
-
mode = "object";
|
|
122
|
-
}
|
|
123
|
-
nested[indented[1]] = parseYamlScalar(indented[2].trim());
|
|
124
|
-
continue;
|
|
37
|
+
catch {
|
|
38
|
+
// Malformed YAML (e.g. unterminated quotes from LLM output corruption).
|
|
39
|
+
// Fall back to line-by-line best-effort extraction so callers still get
|
|
40
|
+
// whatever scalar values they can rather than a completely empty record.
|
|
41
|
+
data = parseFrontmatterLenient(parsedBlock.frontmatter);
|
|
125
42
|
}
|
|
126
|
-
// Top-level key (possibly with inline value)
|
|
127
|
-
const top = line.match(/^(\w[\w-]*):\s*(.*)$/);
|
|
128
|
-
if (!top) {
|
|
129
|
-
continue;
|
|
130
|
-
}
|
|
131
|
-
// Starting a new top-level key — flush any pending empty-value key
|
|
132
|
-
flushPending();
|
|
133
|
-
currentKey = top[1];
|
|
134
|
-
const value = top[2].trim();
|
|
135
|
-
if (value === "|" || value === "|-" || value === "|+") {
|
|
136
|
-
// Block scalar header — collect subsequent indented lines.
|
|
137
|
-
mode = "block";
|
|
138
|
-
blockLines = [];
|
|
139
|
-
blockChomping = value === "|-" ? "strip" : value === "|+" ? "keep" : "clip";
|
|
140
|
-
nested = null;
|
|
141
|
-
currentList = null;
|
|
142
|
-
}
|
|
143
|
-
else if (value === "") {
|
|
144
|
-
// Defer mode decision until we see the next line
|
|
145
|
-
mode = "pending";
|
|
146
|
-
nested = null;
|
|
147
|
-
currentList = null;
|
|
148
|
-
// Don't store anything yet — flushPending will set "" if no continuation
|
|
149
|
-
}
|
|
150
|
-
else if (value.startsWith("[") && value.endsWith("]")) {
|
|
151
|
-
// Inline flow array: tags: [ops, networking]
|
|
152
|
-
mode = "list";
|
|
153
|
-
nested = null;
|
|
154
|
-
currentList = null;
|
|
155
|
-
currentList = parseFlowArray(value);
|
|
156
|
-
data[currentKey] = currentList;
|
|
157
|
-
}
|
|
158
|
-
else {
|
|
159
|
-
mode = "scalar";
|
|
160
|
-
nested = null;
|
|
161
|
-
currentList = null;
|
|
162
|
-
data[currentKey] = parseYamlScalar(value);
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
// Flush any in-progress block scalar at end of frontmatter.
|
|
166
|
-
if (mode === "block") {
|
|
167
|
-
flushBlock();
|
|
168
43
|
}
|
|
169
|
-
// Flush the last key if it was still pending (empty value, no continuation)
|
|
170
|
-
flushPending();
|
|
171
44
|
return {
|
|
172
45
|
data,
|
|
173
46
|
content: parsedBlock.content,
|
|
@@ -176,29 +49,89 @@ export function parseFrontmatter(raw) {
|
|
|
176
49
|
};
|
|
177
50
|
}
|
|
178
51
|
/**
|
|
179
|
-
*
|
|
52
|
+
* Normalize YAML-parsed values to match expected AKM frontmatter types.
|
|
53
|
+
*
|
|
54
|
+
* Two conversions:
|
|
55
|
+
* 1. `Date` → YYYY-MM-DD string: the yaml "core" schema parses bare date
|
|
56
|
+
* scalars like `2026-06-18` as JS Date instances. AKM frontmatter treats
|
|
57
|
+
* `updated:` and similar fields as plain strings.
|
|
58
|
+
* 2. `null` → `""`: the yaml library parses empty-value keys (`key:` with no
|
|
59
|
+
* value) as `null`, but AKM callers historically received `""` from the
|
|
60
|
+
* hand-rolled parser. Convert to preserve backward compatibility.
|
|
180
61
|
*/
|
|
181
|
-
function
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
62
|
+
function normalizeYamlValues(value) {
|
|
63
|
+
if (value instanceof Date) {
|
|
64
|
+
const y = value.getUTCFullYear();
|
|
65
|
+
const m = String(value.getUTCMonth() + 1).padStart(2, "0");
|
|
66
|
+
const d = String(value.getUTCDate()).padStart(2, "0");
|
|
67
|
+
return `${y}-${m}-${d}`;
|
|
68
|
+
}
|
|
69
|
+
if (value === null)
|
|
70
|
+
return "";
|
|
71
|
+
if (Array.isArray(value))
|
|
72
|
+
return value.map(normalizeYamlValues);
|
|
73
|
+
if (typeof value === "object") {
|
|
74
|
+
return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, normalizeYamlValues(v)]));
|
|
75
|
+
}
|
|
76
|
+
return value;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Best-effort line-by-line frontmatter extraction for malformed YAML.
|
|
80
|
+
*
|
|
81
|
+
* Used as a fallback when yaml.parse throws (e.g. unterminated quotes from LLM
|
|
82
|
+
* output corruption). Extracts simple `key: value` scalar pairs only — nested
|
|
83
|
+
* objects and sequences are skipped. Values that are individually parseable by
|
|
84
|
+
* yaml are normalized; otherwise stored as raw strings.
|
|
85
|
+
*/
|
|
86
|
+
function parseFrontmatterLenient(frontmatter) {
|
|
87
|
+
const data = {};
|
|
88
|
+
for (const line of frontmatter.split(/\r?\n/)) {
|
|
89
|
+
const m = line.match(/^([\w][\w-]*):\s*(.*)$/);
|
|
90
|
+
if (!m)
|
|
91
|
+
continue;
|
|
92
|
+
const key = m[1];
|
|
93
|
+
const rawValue = (m[2] ?? "").trim();
|
|
94
|
+
try {
|
|
95
|
+
const singleEntry = yamlParse(`k: ${rawValue}`);
|
|
96
|
+
if (singleEntry !== null && typeof singleEntry === "object" && !Array.isArray(singleEntry)) {
|
|
97
|
+
const v = singleEntry.k;
|
|
98
|
+
data[key] = v === null || v === undefined ? "" : v;
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
data[key] = rawValue;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
data[key] = rawValue;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
return data;
|
|
186
109
|
}
|
|
187
110
|
export function parseFrontmatterBlock(raw) {
|
|
188
111
|
// Handle both LF and CRLF line endings throughout.
|
|
189
112
|
// The closing --- may be preceded by \r\n; capture and strip trailing \r
|
|
190
113
|
// from the frontmatter block so key parsing sees clean LF-terminated lines.
|
|
191
114
|
const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
|
|
192
|
-
if (
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
}
|
|
115
|
+
if (match) {
|
|
116
|
+
// Strip any \r characters from the frontmatter block to normalise CRLF → LF
|
|
117
|
+
const frontmatter = match[1].replace(/\r/g, "");
|
|
118
|
+
const content = match[2];
|
|
119
|
+
return {
|
|
120
|
+
frontmatter,
|
|
121
|
+
content,
|
|
122
|
+
bodyStartLine: countLines(raw.slice(0, match[0].length - match[2].length)) + 1,
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
// Empty frontmatter (---\n---): the content-bearing regex above requires at
|
|
126
|
+
// least one character between the fences. Handle the degenerate case so
|
|
127
|
+
// callers can reconstruct `---\nkey: val\n---\n\nbody` from a previously
|
|
128
|
+
// empty-frontmatter file without corrupting it by wrapping the entire raw
|
|
129
|
+
// string as body content.
|
|
130
|
+
const emptyMatch = raw.match(/^---\r?\n---(?:\r\n|\r|\n)([\s\S]*)$/);
|
|
131
|
+
if (emptyMatch) {
|
|
132
|
+
return { frontmatter: "", content: emptyMatch[1], bodyStartLine: 3 };
|
|
133
|
+
}
|
|
134
|
+
return null;
|
|
202
135
|
}
|
|
203
136
|
function countLines(text) {
|
|
204
137
|
if (text.length === 0)
|
|
@@ -206,7 +139,13 @@ function countLines(text) {
|
|
|
206
139
|
return text.split(/\r?\n/).length - 1;
|
|
207
140
|
}
|
|
208
141
|
/**
|
|
209
|
-
* Parse a
|
|
142
|
+
* Parse a YAML scalar value (string, boolean, or number).
|
|
143
|
+
*
|
|
144
|
+
* For quoted strings (single or double), delegates to the `yaml` library so
|
|
145
|
+
* escape sequences are handled correctly per spec. The previous hand-rolled
|
|
146
|
+
* `slice(1, -1)` only stripped one layer of quoting and left inner quotes and
|
|
147
|
+
* escape sequences as literal characters in the stored value, causing visible
|
|
148
|
+
* corruption when `yaml.stringify` re-quoted them on the next write.
|
|
210
149
|
*/
|
|
211
150
|
export function parseYamlScalar(value) {
|
|
212
151
|
if (value === "")
|
|
@@ -219,7 +158,67 @@ export function parseYamlScalar(value) {
|
|
|
219
158
|
if (!Number.isNaN(asNumber))
|
|
220
159
|
return asNumber;
|
|
221
160
|
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
|
161
|
+
try {
|
|
162
|
+
const parsed = yamlParse(value);
|
|
163
|
+
if (typeof parsed === "string")
|
|
164
|
+
return parsed;
|
|
165
|
+
}
|
|
166
|
+
catch {
|
|
167
|
+
// Fall through to raw slice on malformed YAML — better than throwing.
|
|
168
|
+
}
|
|
222
169
|
return value.slice(1, -1);
|
|
223
170
|
}
|
|
224
171
|
return value;
|
|
225
172
|
}
|
|
173
|
+
// ── Minimum score delta to trigger a frontmatter salience rewrite ─────────────
|
|
174
|
+
const SALIENCE_WRITE_DELTA_THRESHOLD = 0.05;
|
|
175
|
+
/**
|
|
176
|
+
* Idempotently write `salience` and `salienceInputs` fields into the YAML
|
|
177
|
+
* frontmatter of a raw asset string.
|
|
178
|
+
*
|
|
179
|
+
* Skips the write when the existing `salience` field differs from `score` by
|
|
180
|
+
* less than {@link SALIENCE_WRITE_DELTA_THRESHOLD}, to avoid churn for minor
|
|
181
|
+
* floating-point drift. Returns the raw string unchanged when no write is needed
|
|
182
|
+
* or when no frontmatter block is present.
|
|
183
|
+
*
|
|
184
|
+
* The `salienceInputs` field is written for auditability only; no pipeline code
|
|
185
|
+
* reads it back. `state.db :: asset_salience` is the canonical store.
|
|
186
|
+
*/
|
|
187
|
+
export function writeSalienceToFrontmatter(raw, score, inputs) {
|
|
188
|
+
const parsed = parseFrontmatterBlock(raw);
|
|
189
|
+
if (!parsed)
|
|
190
|
+
return raw;
|
|
191
|
+
const existingData = parseFrontmatter(raw).data;
|
|
192
|
+
const existingSalience = typeof existingData.salience === "number" ? existingData.salience : undefined;
|
|
193
|
+
if (existingSalience !== undefined && Math.abs(existingSalience - score) < SALIENCE_WRITE_DELTA_THRESHOLD) {
|
|
194
|
+
return raw;
|
|
195
|
+
}
|
|
196
|
+
// Parse existing frontmatter into an object, then set/overwrite salience fields.
|
|
197
|
+
let fm = {};
|
|
198
|
+
if (parsed.frontmatter.trim()) {
|
|
199
|
+
try {
|
|
200
|
+
const p = yamlParse(parsed.frontmatter);
|
|
201
|
+
if (p !== null && typeof p === "object" && !Array.isArray(p)) {
|
|
202
|
+
fm = p;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
catch {
|
|
206
|
+
// Malformed YAML — rebuild from best-effort parse
|
|
207
|
+
fm = parseFrontmatterLenient(parsed.frontmatter);
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
fm.salience = roundTo2dp(score);
|
|
211
|
+
fm.salienceInputs = {
|
|
212
|
+
novelty: roundTo2dp(inputs.novelty),
|
|
213
|
+
magnitude: roundTo2dp(inputs.magnitude),
|
|
214
|
+
predictionError: roundTo2dp(inputs.predictionError),
|
|
215
|
+
};
|
|
216
|
+
const newFrontmatter = yamlStringify(fm).trimEnd();
|
|
217
|
+
const body = parsed.content;
|
|
218
|
+
// Preserve original line ending style between frontmatter and body
|
|
219
|
+
const separator = body.startsWith("\n") ? "" : "\n";
|
|
220
|
+
return `---\n${newFrontmatter}\n---\n${separator}${body}`;
|
|
221
|
+
}
|
|
222
|
+
function roundTo2dp(n) {
|
|
223
|
+
return Math.round(n * 100) / 100;
|
|
224
|
+
}
|
|
@@ -8,7 +8,15 @@ export function parseMarkdownToc(content) {
|
|
|
8
8
|
const headings = [];
|
|
9
9
|
const parsed = parseFrontmatter(content);
|
|
10
10
|
const start = parsed.frontmatter ? parsed.bodyStartLine - 1 : 0;
|
|
11
|
+
let inFence = false;
|
|
11
12
|
for (let i = start; i < lines.length; i++) {
|
|
13
|
+
// Track fenced code blocks (``` or ~~~) so headings inside them are skipped.
|
|
14
|
+
if (/^\s*(`{3,}|~{3,})/.test(lines[i])) {
|
|
15
|
+
inFence = !inFence;
|
|
16
|
+
continue;
|
|
17
|
+
}
|
|
18
|
+
if (inFence)
|
|
19
|
+
continue;
|
|
12
20
|
const match = lines[i].match(/^(#{1,6})\s+(.+)$/);
|
|
13
21
|
if (match) {
|
|
14
22
|
headings.push({
|