akm-cli 0.9.0-beta.11 → 0.9.0-beta.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. package/CHANGELOG.md +163 -0
  2. package/dist/assets/prompts/consolidate-system.md +23 -0
  3. package/dist/assets/prompts/contradiction-judge.md +33 -0
  4. package/dist/assets/prompts/distill-knowledge-system.md +22 -0
  5. package/dist/assets/prompts/distill-lesson-system.md +36 -0
  6. package/dist/assets/prompts/extract-session.md +5 -1
  7. package/dist/assets/prompts/graph-extract-system.md +1 -0
  8. package/dist/assets/prompts/memory-infer-system.md +1 -0
  9. package/dist/assets/prompts/memory-infer-user.md +5 -0
  10. package/dist/assets/prompts/metadata-enhance-system.md +1 -0
  11. package/dist/assets/prompts/procedural-system.md +44 -0
  12. package/dist/assets/prompts/recombine-system.md +40 -0
  13. package/dist/assets/prompts/staleness-detect-system.md +6 -0
  14. package/dist/assets/prompts/validate-summary-judge.md +1 -0
  15. package/dist/assets/templates/html/health.html +25 -27
  16. package/dist/cli.js +2 -2
  17. package/dist/commands/agent/contribute-cli.js +16 -3
  18. package/dist/commands/feedback-cli.js +48 -44
  19. package/dist/commands/health/html-report.js +140 -16
  20. package/dist/commands/health.js +277 -1
  21. package/dist/commands/improve/calibration.js +161 -0
  22. package/dist/commands/improve/consolidate.js +595 -105
  23. package/dist/commands/improve/dedup.js +482 -0
  24. package/dist/commands/improve/distill.js +119 -64
  25. package/dist/commands/improve/encoding-salience.js +205 -0
  26. package/dist/commands/improve/extract-cli.js +115 -1
  27. package/dist/commands/improve/extract-prompt.js +32 -1
  28. package/dist/commands/improve/extract-watch.js +140 -0
  29. package/dist/commands/improve/extract.js +210 -30
  30. package/dist/commands/improve/feedback-valence.js +54 -0
  31. package/dist/commands/improve/homeostatic.js +467 -0
  32. package/dist/commands/improve/improve-auto-accept.js +80 -7
  33. package/dist/commands/improve/improve-profiles.js +8 -0
  34. package/dist/commands/improve/improve.js +991 -61
  35. package/dist/commands/improve/memory/memory-contradiction-detect.js +23 -28
  36. package/dist/commands/improve/outcome-loop.js +256 -0
  37. package/dist/commands/improve/proactive-maintenance.js +9 -35
  38. package/dist/commands/improve/procedural.js +409 -0
  39. package/dist/commands/improve/recombine.js +488 -0
  40. package/dist/commands/improve/reflect.js +20 -1
  41. package/dist/commands/improve/related-sessions.js +120 -0
  42. package/dist/commands/improve/salience.js +386 -0
  43. package/dist/commands/improve/triage.js +95 -0
  44. package/dist/commands/lint/agent-linter.js +19 -24
  45. package/dist/commands/lint/base-linter.js +173 -60
  46. package/dist/commands/lint/command-linter.js +19 -24
  47. package/dist/commands/lint/env-key-rules.js +34 -1
  48. package/dist/commands/lint/index.js +30 -13
  49. package/dist/commands/lint/memory-linter.js +1 -1
  50. package/dist/commands/lint/registry.js +5 -2
  51. package/dist/commands/lint/task-linter.js +3 -3
  52. package/dist/commands/lint/workflow-linter.js +26 -1
  53. package/dist/commands/proposal/validators/proposals.js +4 -0
  54. package/dist/commands/read/curate.js +284 -86
  55. package/dist/commands/read/search-cli.js +7 -0
  56. package/dist/commands/read/search.js +1 -0
  57. package/dist/commands/sources/installed-stashes.js +5 -1
  58. package/dist/core/asset/frontmatter.js +166 -167
  59. package/dist/core/asset/markdown.js +8 -0
  60. package/dist/core/config/config-schema.js +211 -3
  61. package/dist/core/config/config.js +2 -2
  62. package/dist/core/logs-db.js +4 -3
  63. package/dist/core/state-db.js +555 -29
  64. package/dist/indexer/db/db.js +250 -27
  65. package/dist/indexer/db/graph-db.js +81 -86
  66. package/dist/indexer/graph/graph-boost.js +51 -41
  67. package/dist/indexer/passes/memory-inference.js +10 -3
  68. package/dist/indexer/passes/staleness-detect.js +2 -5
  69. package/dist/indexer/search/db-search.js +15 -4
  70. package/dist/indexer/search/ranking.js +4 -0
  71. package/dist/integrations/harnesses/claude/session-log.js +10 -0
  72. package/dist/integrations/harnesses/opencode/session-log.js +9 -0
  73. package/dist/integrations/session-logs/index.js +16 -0
  74. package/dist/llm/embedder.js +27 -3
  75. package/dist/llm/embedders/local.js +66 -2
  76. package/dist/llm/graph-extract.js +2 -1
  77. package/dist/llm/memory-infer.js +4 -8
  78. package/dist/llm/metadata-enhance.js +9 -1
  79. package/dist/output/shapes/curate.js +14 -2
  80. package/dist/output/text/helpers.js +9 -0
  81. package/dist/runtime.js +25 -1
  82. package/dist/scripts/migrate-storage.js +1025 -567
  83. package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +435 -269
  84. package/dist/storage/sqlite-pragmas.js +146 -0
  85. package/dist/workflows/db.js +3 -4
  86. package/dist/workflows/validate-summary.js +2 -7
  87. package/docs/data-and-telemetry.md +1 -0
  88. package/package.json +5 -4
@@ -52,22 +52,31 @@
52
52
  */
53
53
  import fs from "node:fs";
54
54
  import path from "node:path";
55
+ import distillKnowledgeSystemPrompt from "../../assets/prompts/distill-knowledge-system.md" with { type: "text" };
56
+ import distillLessonSystemPrompt from "../../assets/prompts/distill-lesson-system.md" with { type: "text" };
55
57
  import { parseAssetRef } from "../../core/asset/asset-ref.js";
56
58
  import { assembleAssetFromString } from "../../core/asset/asset-serialize.js";
57
- import { parseFrontmatter } from "../../core/asset/frontmatter.js";
59
+ import { parseFrontmatter, writeSalienceToFrontmatter } from "../../core/asset/frontmatter.js";
58
60
  import { stripMarkdownFences } from "../../core/asset/markdown.js";
59
61
  import { resolveStashDir, timestampForFilename } from "../../core/common.js";
60
62
  import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
61
63
  import { ConfigError, UsageError } from "../../core/errors.js";
62
64
  import { appendEvent, readEvents } from "../../core/events.js";
63
65
  import { lintLessonContent } from "../../core/lesson-lint.js";
66
+ import { getDbPath } from "../../core/paths.js";
67
+ import { openStateDatabase } from "../../core/state-db.js";
64
68
  import { warnVerbose } from "../../core/warn.js";
69
+ import { closeDatabase, getAllEntries, openDatabase } from "../../indexer/db/db.js";
65
70
  import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
66
71
  import { chatCompletion, parseEmbeddedJsonResponse } from "../../llm/client.js";
67
72
  import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
68
73
  import { createProposal, isProposalSkipped, listProposals, } from "../proposal/validators/proposals.js";
69
74
  import { akmSearch } from "../read/search.js";
75
+ import { stripFrontmatterBody as stripBodyForFidelity } from "./dedup.js";
70
76
  import { assessMemoryKnowledgePromotionCandidate, deriveKnowledgeRef } from "./distill-promotion-policy.js";
77
+ import { buildRefVocabulary, scoreEncodingSalience } from "./encoding-salience.js";
78
+ import { buildClsContext, checkDistillFidelity } from "./homeostatic.js";
79
+ import { computeSalience, upsertAssetSalience } from "./salience.js";
71
80
  /**
72
81
  * Asset-ref types that `akm distill` structurally refuses as inputs.
73
82
  *
@@ -122,68 +131,8 @@ import { repairTruncatedDescription } from "../../core/text-truncation.js";
122
131
  import { detectDoubleFrontmatter, isValidDescription, isValidWhenToUse, } from "../proposal/validators/proposal-quality-validators.js";
123
132
  export { detectDoubleFrontmatter, isValidDescription, isValidWhenToUse };
124
133
  // ── Prompt assembly ─────────────────────────────────────────────────────────
125
- const LESSON_SYSTEM_PROMPT = [
126
- "You are the akm `distill` distiller.",
127
- "Given an asset and recent feedback events about it, produce a single",
128
- "concise *lesson* an agent should remember next time it works on this",
129
- "asset's domain.",
130
- "",
131
- "YOUR RESPONSE MUST START EXACTLY WITH `---` ON THE VERY FIRST LINE.",
132
- "DO NOT output any prose, explanation, or code fences before or after.",
133
- "",
134
- "Required output format — copy this structure exactly:",
135
- "---",
136
- "description: <one complete sentence (ending with `.`) summarising what the lesson teaches>",
137
- "when_to_use: <one complete sentence describing the concrete trigger condition>",
138
- "---",
139
- "",
140
- "<lesson body — plain markdown, 1–3 short paragraphs of practical guidance>",
141
- "",
142
- "## description field (MANDATORY)",
143
- "- A single complete sentence in present tense, 80-200 chars, NO markdown.",
144
- "- Self-contained: a reviewer must understand the lesson from this field alone.",
145
- '- DO NOT start with "When ", "If ", or a connector word — that belongs in when_to_use.',
146
- '- DO NOT copy a section heading ("Key takeaways", "For example", "Key pitfalls").',
147
- "- DO NOT begin with a numbered list marker, code fence, or markdown heading.",
148
- "",
149
- 'GOOD: "Always validate ref existence before promoting a memory to knowledge; missing refs surface as silent 404s during accept."',
150
- 'BAD: "Key pitfalls"',
151
- 'BAD: "When working with the akm CLI"',
152
- 'BAD: "For example, you might..."',
153
- 'BAD: "1. Check the file"',
154
- "",
155
- "RULES:",
156
- "- `when_to_use` MUST be a complete sentence describing a concrete trigger. Never write `When working with <asset-name>` — that is circular and useless.",
157
- "- `description` and `when_to_use` MUST differ from each other.",
158
- "- The lesson body MUST be non-empty markdown prose. Do NOT restate `description:` or `when_to_use:` inside the body (no `**description:** ...` or `**when_to_use:** ...` lines — the frontmatter is the only place those keys belong).",
159
- "- Do NOT emit a second `---` fence after the opening frontmatter — there are exactly two `---` lines in the output, both belonging to the single frontmatter block at the top.",
160
- "- Do NOT reproduce the source asset verbatim — distil what a caller needs to know.",
161
- "- Output ONLY the lesson file. No preamble, no code fences, no trailing prose.",
162
- ].join("\n");
163
- const KNOWLEDGE_SYSTEM_PROMPT = [
164
- "You are the akm `distill` distiller.",
165
- "Given an asset and recent feedback events about it, produce a concise",
166
- "*knowledge* markdown document capturing the durable, reusable facts.",
167
- "Prefer stable guidance over narrative recap.",
168
- "",
169
- "YOUR RESPONSE MUST START EXACTLY WITH `---` ON THE VERY FIRST LINE.",
170
- "DO NOT output any prose, explanation, or code fences before or after.",
171
- "",
172
- "Required output format:",
173
- "---",
174
- "description: <one-line summary of the knowledge asset>",
175
- "tags: [<tag1>, <tag2>]",
176
- "---",
177
- "",
178
- "# <Title>",
179
- "",
180
- "<body — structured markdown, durable facts only>",
181
- "",
182
- "RULES:",
183
- "- `description` MUST be a non-empty single-line string.",
184
- "- Include a meaningful markdown body with a `# Title` heading.",
185
- "- Output ONLY the knowledge file. No preamble, no code fences, no trailing prose.",
186
- ].join("\n");
134
+ const LESSON_SYSTEM_PROMPT = distillLessonSystemPrompt;
135
+ const KNOWLEDGE_SYSTEM_PROMPT = distillKnowledgeSystemPrompt;
187
136
  // ── Structured-output schemas (responseSchema lift) ─────────────────────────
188
137
  //
189
138
  // PR 1 of the asset-writers decision (see knowledge:projects/akm/
@@ -681,15 +630,78 @@ export async function akmDistill(options) {
681
630
  // Best-effort load: when the asset is not yet indexed we still proceed —
682
631
  // the LLM is asked to distil from "available signal" (feedback alone).
683
632
  let assetContent = null;
633
+ let assetFilePath = null;
684
634
  try {
685
635
  const filePath = await lookup(inputRef);
686
636
  if (filePath && fs.existsSync(filePath)) {
637
+ assetFilePath = filePath;
687
638
  assetContent = fs.readFileSync(filePath, "utf8");
688
639
  }
689
640
  }
690
641
  catch {
691
642
  assetContent = null;
692
643
  }
644
+ // ── #608: Encoding-time salience scoring ────────────────────────────────
645
+ // Score the source asset with the three-signal model (novelty × 0.40 +
646
+ // magnitude × 0.35 + predictionError × 0.25) and persist the result to:
647
+ // 1. The asset's frontmatter (human-readable mirror; idempotent delta gate).
648
+ // 2. state.db :: asset_salience (canonical; feeds improve's high-salience gate).
649
+ // Both writes are best-effort — a DB error never blocks distillation.
650
+ if (assetContent && assetFilePath) {
651
+ try {
652
+ const parsedRef = parseAssetRef(inputRef);
653
+ // Build bigram vocabulary from currently-indexed refs for novelty signal.
654
+ let existingRefVocabulary = new Set();
655
+ try {
656
+ const embCfg = config?.embedding;
657
+ const indexDb = openDatabase(getDbPath(), embCfg?.dimension ? { embeddingDim: embCfg.dimension } : undefined);
658
+ try {
659
+ const allRefs = getAllEntries(indexDb).map((e) => e.entryKey);
660
+ existingRefVocabulary = buildRefVocabulary(allRefs);
661
+ }
662
+ finally {
663
+ closeDatabase(indexDb);
664
+ }
665
+ }
666
+ catch {
667
+ // Index not available — novelty defaults to type-floor.
668
+ }
669
+ const salienceResult = scoreEncodingSalience({
670
+ body: assetContent,
671
+ type: parsedRef.type,
672
+ existingRefVocabulary,
673
+ revisionCount: 0,
674
+ });
675
+ // 1. Write salience to the source asset frontmatter (idempotent).
676
+ const updatedContent = writeSalienceToFrontmatter(assetContent, salienceResult.score, salienceResult);
677
+ if (updatedContent !== assetContent) {
678
+ fs.writeFileSync(assetFilePath, updatedContent, "utf8");
679
+ assetContent = updatedContent;
680
+ }
681
+ // 2. Persist encoding_salience to state.db.
682
+ try {
683
+ const stateDb = openStateDatabase();
684
+ try {
685
+ const vector = computeSalience({
686
+ ref: inputRef,
687
+ type: parsedRef.type,
688
+ retrievalFreq: 0,
689
+ encodingSalience: salienceResult.score,
690
+ });
691
+ upsertAssetSalience(stateDb, inputRef, vector);
692
+ }
693
+ finally {
694
+ stateDb.close();
695
+ }
696
+ }
697
+ catch {
698
+ // State DB unavailable — frontmatter mirror is the only persistence.
699
+ }
700
+ }
701
+ catch {
702
+ // Scoring errors never block distillation.
703
+ }
704
+ }
693
705
  const { events } = readEventsImpl({
694
706
  ref: inputRef,
695
707
  type: "feedback",
@@ -909,13 +921,32 @@ export async function akmDistill(options) {
909
921
  reason: p.review?.reason ?? "no reason given",
910
922
  contentPreview: p.payload.content.slice(0, 500),
911
923
  }));
912
- const userPrompt = buildDistillPrompt({
924
+ // WS-3b CLS interleaving (step 9).
925
+ // When cls.enabled, inject embedding-retrieved adjacent lessons/knowledge
926
+ // into the distill prompt so the LLM avoids overwriting prior generalizations
927
+ // (catastrophic interference). DEFAULT OFF.
928
+ const clsConfig = config.profiles?.improve?.default?.processes?.distill?.cls ?? {};
929
+ let clsContext = "";
930
+ if (clsConfig.enabled) {
931
+ try {
932
+ const adjacentCount = clsConfig.adjacentCount ?? 3;
933
+ // Use the asset content or input ref as the query for adjacent retrieval.
934
+ const clsQuery = assetContent ? assetContent.slice(0, 500) : inputRef;
935
+ const adjacentItems = await fetchSimilarLessonsFn(clsQuery, adjacentCount);
936
+ clsContext = buildClsContext(adjacentItems, clsConfig);
937
+ }
938
+ catch {
939
+ // Fail open — CLS is supplemental, never required.
940
+ }
941
+ }
942
+ const baseUserPrompt = buildDistillPrompt({
913
943
  inputRef,
914
944
  assetContent,
915
945
  feedback,
916
946
  proposalKind: effectiveProposalKind,
917
947
  ...(rejectedForRef.length > 0 ? { rejectedProposals: rejectedForRef } : {}),
918
948
  });
949
+ const userPrompt = clsContext ? `${baseUserPrompt}${clsContext}` : baseUserPrompt;
919
950
  const messages = [
920
951
  { role: "system", content: effectiveProposalKind === "knowledge" ? KNOWLEDGE_SYSTEM_PROMPT : LESSON_SYSTEM_PROMPT },
921
952
  { role: "user", content: userPrompt },
@@ -1251,6 +1282,30 @@ export async function akmDistill(options) {
1251
1282
  if (judgeResult.score > 0)
1252
1283
  lessonJudgeConfidence = judgeResult.score / 5;
1253
1284
  }
1285
+ // WS-3b: Distill→source fidelity check (step 10).
1286
+ // When fidelityCheck.enabled, check the distill proposal against its cited
1287
+ // source memories. A contradiction flag routes to human review (not auto-accept).
1288
+ // DEFAULT OFF. Fail-open: any error is treated as no-contradiction.
1289
+ const fidelityConfig = config.profiles?.improve?.default?.processes?.distill?.fidelityCheck ?? {};
1290
+ if (fidelityConfig.enabled && assetContent) {
1291
+ try {
1292
+ const proposalBody = stripBodyForFidelity(content);
1293
+ const sourceBodies = [stripBodyForFidelity(assetContent)];
1294
+ const fidelityResult = checkDistillFidelity(proposalBody, sourceBodies, fidelityConfig);
1295
+ if (fidelityResult.contradictionDetected) {
1296
+ // Route to human review by writing a quality rejection with reviewNeeded=true.
1297
+ return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, 2.0, // below auto-accept threshold, signals review needed
1298
+ fidelityResult.reason ?? "Proposal may contradict cited source memories.", {
1299
+ reviewNeeded: true,
1300
+ fidelityContradiction: true,
1301
+ ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
1302
+ }, options.eligibilitySource);
1303
+ }
1304
+ }
1305
+ catch {
1306
+ // Fail open — fidelity check is supplemental.
1307
+ }
1308
+ }
1254
1309
  // Round-trip the parsed frontmatter so the proposal carries it as a
1255
1310
  // structured payload alongside the raw content (matches the shape used by
1256
1311
  // other proposal sources).
@@ -0,0 +1,205 @@
1
+ // This Source Code Form is subject to the terms of the Mozilla Public
2
+ // License, v. 2.0. If a copy of the MPL was not distributed with this
3
+ // file, You can obtain one at https://mozilla.org/MPL/2.0/.
4
+ /**
5
+ * Encoding-time salience scoring for issue #608.
6
+ *
7
+ * Pure module — no I/O. All inputs are pre-fetched by the caller.
8
+ * Implements the three-signal model: novelty × 0.40 + magnitude × 0.35 +
9
+ * predictionError × 0.25, clamped to [0, 1].
10
+ */
11
+ // ── Weights ────────────────────────────────────────────────────────────────────
12
+ const W_NOVELTY = 0.4;
13
+ const W_MAGNITUDE = 0.35;
14
+ const W_PREDICTION_ERROR = 0.25;
15
+ // Guard: weights must sum to 1.0 (consistent with the pattern in salience.ts).
16
+ // This check fires at module load and catches any future re-tuning mistakes.
17
+ if (Math.abs(W_NOVELTY + W_MAGNITUDE + W_PREDICTION_ERROR - 1.0) > 1e-9) {
18
+ throw new Error(`encoding-salience.ts: sub-signal weights must sum to 1.0 (got ${W_NOVELTY + W_MAGNITUDE + W_PREDICTION_ERROR})`);
19
+ }
20
+ // ── Novelty type floors ────────────────────────────────────────────────────────
21
+ const TYPE_NOVELTY_FLOOR = Object.freeze({
22
+ skill: 0.8,
23
+ agent: 0.8,
24
+ memory: 0.4,
25
+ });
26
+ const DEFAULT_NOVELTY_FLOOR = 0.5;
27
+ // ── Magnitude keyword sets ──────────────────────────────────────────────────────
28
+ const SEVERITY_BUCKET = new Set(["error", "critical", "warning", "incident", "regression"]);
29
+ // Note: "fails" and "urgent" are from the spec's initial keyword list and are included
30
+ // here as strong constraint signals, even though the spec's final bucket table omits them.
31
+ const CONSTRAINT_BUCKET = new Set(["must", "never", "always", "blocked", "breaking", "deprecated", "fails", "urgent"]);
32
+ const ALL_MAGNITUDE_KEYWORDS = new Set([...SEVERITY_BUCKET, ...CONSTRAINT_BUCKET]);
33
+ // Number of distinct keyword matches to reach full magnitude score.
34
+ const MAGNITUDE_FULL_SCORE_THRESHOLD = 4;
35
+ // Single-bucket magnitude cap: when only one semantic bucket matches.
36
+ const MAGNITUDE_SINGLE_BUCKET_CAP = 0.5;
37
+ // ── Stop-words (excluded from bigram tokenization) ────────────────────────────
38
+ const STOP_WORDS = new Set([
39
+ "a",
40
+ "an",
41
+ "the",
42
+ "and",
43
+ "or",
44
+ "but",
45
+ "in",
46
+ "on",
47
+ "at",
48
+ "to",
49
+ "for",
50
+ "of",
51
+ "with",
52
+ "by",
53
+ "from",
54
+ "as",
55
+ "is",
56
+ "it",
57
+ "its",
58
+ "be",
59
+ "are",
60
+ "was",
61
+ "were",
62
+ "has",
63
+ "have",
64
+ "had",
65
+ "do",
66
+ "does",
67
+ "did",
68
+ "not",
69
+ "this",
70
+ "that",
71
+ "these",
72
+ "those",
73
+ "i",
74
+ "we",
75
+ "you",
76
+ "he",
77
+ "she",
78
+ "they",
79
+ "my",
80
+ "our",
81
+ "your",
82
+ "his",
83
+ "her",
84
+ "their",
85
+ "can",
86
+ "will",
87
+ "would",
88
+ "should",
89
+ "could",
90
+ "may",
91
+ "might",
92
+ "shall",
93
+ "so",
94
+ "if",
95
+ "then",
96
+ "than",
97
+ "when",
98
+ "while",
99
+ "what",
100
+ "which",
101
+ "who",
102
+ "how",
103
+ "no",
104
+ "any",
105
+ "all",
106
+ "more",
107
+ "also",
108
+ ]);
109
+ // ── Tokenization helpers ──────────────────────────────────────────────────────
110
+ /**
111
+ * Tokenize text into lowercase words, splitting on whitespace and punctuation,
112
+ * dropping stop-words. Used for body bigram extraction (novelty signal).
113
+ */
114
+ function tokenizeBody(text) {
115
+ return text
116
+ .toLowerCase()
117
+ .split(/[\s\p{P}\p{S}\-_/]+/u)
118
+ .filter((t) => t.length > 1 && !STOP_WORDS.has(t));
119
+ }
120
+ /**
121
+ * Tokenize a ref name or tag into lowercase words without stop-word filtering.
122
+ * Ref names are identifiers, not prose — all tokens are significant.
123
+ */
124
+ function tokenizeRef(ref) {
125
+ return ref
126
+ .toLowerCase()
127
+ .split(/[\s\p{P}\p{S}\-_/]+/u)
128
+ .filter((t) => t.length > 0);
129
+ }
130
+ /**
131
+ * Generate all consecutive bigrams from an array of tokens.
132
+ * Returns them as "token1 token2" strings.
133
+ */
134
+ function bigrams(tokens) {
135
+ const result = [];
136
+ for (let i = 0; i < tokens.length - 1; i++) {
137
+ result.push(`${tokens[i]} ${tokens[i + 1]}`);
138
+ }
139
+ return result;
140
+ }
141
+ /**
142
+ * Build the existing-ref vocabulary: a Set of bigrams derived from tokenizing
143
+ * the provided ref names (and tags). Pass this to `scoreEncodingSalience` as
144
+ * `existingRefVocabulary` so the novelty signal can measure how much of the
145
+ * asset body is already represented in the stash vocabulary.
146
+ */
147
+ export function buildRefVocabulary(refs) {
148
+ const vocab = new Set();
149
+ for (const ref of refs) {
150
+ const tokens = tokenizeRef(ref);
151
+ for (const bg of bigrams(tokens)) {
152
+ vocab.add(bg);
153
+ }
154
+ }
155
+ return vocab;
156
+ }
157
+ // ── Sub-signal computations ────────────────────────────────────────────────────
158
+ function computeNovelty(body, type, vocab) {
159
+ const tokens = tokenizeBody(body);
160
+ const bgs = bigrams(tokens);
161
+ const floor = TYPE_NOVELTY_FLOOR[type] ?? DEFAULT_NOVELTY_FLOOR;
162
+ if (bgs.length === 0)
163
+ return floor;
164
+ const novelCount = bgs.filter((bg) => !vocab.has(bg)).length;
165
+ const bigramNoveltyFraction = novelCount / bgs.length;
166
+ return Math.max(floor, bigramNoveltyFraction);
167
+ }
168
+ function computeMagnitude(body) {
169
+ const lowerBody = body.toLowerCase();
170
+ // Split on word boundaries to avoid partial matches (e.g. "errors" matching "error").
171
+ const words = new Set(lowerBody.split(/\W+/).filter((w) => w.length > 0));
172
+ const matched = [...ALL_MAGNITUDE_KEYWORDS].filter((kw) => words.has(kw));
173
+ if (matched.length === 0)
174
+ return 0;
175
+ const distinctCount = matched.length;
176
+ const rawMagnitude = Math.min(1.0, distinctCount / MAGNITUDE_FULL_SCORE_THRESHOLD);
177
+ // Require at least one match from each bucket to lift the single-bucket cap.
178
+ const hasSeverity = matched.some((kw) => SEVERITY_BUCKET.has(kw));
179
+ const hasConstraint = matched.some((kw) => CONSTRAINT_BUCKET.has(kw));
180
+ if (hasSeverity && hasConstraint)
181
+ return rawMagnitude;
182
+ return Math.min(rawMagnitude, MAGNITUDE_SINGLE_BUCKET_CAP);
183
+ }
184
+ function computePredictionError(revisionCount) {
185
+ if (revisionCount === 0)
186
+ return 1.0;
187
+ return 1 / (1 + Math.log(1 + revisionCount));
188
+ }
189
+ // ── Main export ────────────────────────────────────────────────────────────────
190
+ /**
191
+ * Compute the encoding-time salience score for an asset.
192
+ *
193
+ * Three sub-signals:
194
+ * novelty (0.40) — how much of the body is absent from the stash vocabulary.
195
+ * magnitude (0.35) — presence of severity/constraint keywords.
196
+ * predictionError (0.25) — surprise: high for new assets, decays with revisions.
197
+ */
198
+ export function scoreEncodingSalience(inputs) {
199
+ const novelty = computeNovelty(inputs.body, inputs.type, inputs.existingRefVocabulary);
200
+ const magnitude = computeMagnitude(inputs.body);
201
+ const predictionError = computePredictionError(inputs.revisionCount);
202
+ const raw = W_NOVELTY * novelty + W_MAGNITUDE * magnitude + W_PREDICTION_ERROR * predictionError;
203
+ const score = Math.min(1, Math.max(0, raw));
204
+ return { score, novelty, magnitude, predictionError };
205
+ }
@@ -14,11 +14,14 @@
14
14
  * Output is the AkmExtractResult JSON envelope (or an aggregated one when
15
15
  * `--auto` runs multiple harnesses).
16
16
  */
17
+ import fs from "node:fs";
18
+ import path from "node:path";
17
19
  import { defineCommand } from "citty";
18
20
  import { output, runWithJsonErrors } from "../../cli/shared.js";
19
21
  import { UsageError } from "../../core/errors.js";
20
- import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
22
+ import { getAvailableHarnesses, getWatchTargets } from "../../integrations/session-logs/index.js";
21
23
  import { akmExtract } from "./extract.js";
24
+ import { akmExtractWatch } from "./extract-watch.js";
22
25
  export const extractCommand = defineCommand({
23
26
  meta: {
24
27
  name: "extract",
@@ -60,6 +63,15 @@ export const extractCommand = defineCommand({
60
63
  type: "string",
61
64
  description: "Per-session LLM timeout in ms (default 60000).",
62
65
  },
66
+ watch: {
67
+ type: "boolean",
68
+ description: "Watch harness session-log directories and run extract on change (debounced). Stays alive until SIGINT/SIGTERM. Falls back to the */30 cron when off.",
69
+ default: false,
70
+ },
71
+ "debounce-ms": {
72
+ type: "string",
73
+ description: "Debounce window in ms for --watch (default 2000).",
74
+ },
63
75
  },
64
76
  async run({ args }) {
65
77
  await runWithJsonErrors(async () => {
@@ -76,6 +88,17 @@ export const extractCommand = defineCommand({
76
88
  if (timeoutMs !== undefined && (!Number.isFinite(timeoutMs) || timeoutMs <= 0)) {
77
89
  throw new UsageError(`--timeout-ms must be a positive integer (got "${args["timeout-ms"]}").`, "INVALID_FLAG_VALUE");
78
90
  }
91
+ const watch = args.watch === true;
92
+ const debounceMs = typeof args["debounce-ms"] === "string" && args["debounce-ms"] !== ""
93
+ ? Number.parseInt(args["debounce-ms"], 10)
94
+ : 2000;
95
+ if (watch && (!Number.isFinite(debounceMs) || debounceMs <= 0)) {
96
+ throw new UsageError(`--debounce-ms must be a positive integer (got "${args["debounce-ms"]}").`, "INVALID_FLAG_VALUE");
97
+ }
98
+ if (watch) {
99
+ await runWatchMode({ debounceMs, dryRun, force, ...(since ? { since } : {}) });
100
+ return;
101
+ }
79
102
  if (auto && type) {
80
103
  throw new UsageError("--auto and --type are mutually exclusive. Pick one.", "INVALID_FLAG_VALUE");
81
104
  }
@@ -125,3 +148,94 @@ export const extractCommand = defineCommand({
125
148
  });
126
149
  },
127
150
  });
151
+ /**
152
+ * A thin {@link WatchEventSource} over `fs.watch` for each configured root.
153
+ * This adapter is the ONLY place a real `fs.watch` is created (the core stays
154
+ * injectable + fully unit-tested); it is intentionally not unit-covered.
155
+ *
156
+ * `fs.watch(dir, { recursive: true })` is unreliable for recursive mode on some
157
+ * Node/Bun/Linux combinations — that is acceptable here: missed events degrade
158
+ * to the unchanged scheduled cron fallback, never worse than today.
159
+ */
160
+ function createFsWatchEventSource(roots) {
161
+ return {
162
+ subscribe(listener) {
163
+ const watchers = [];
164
+ for (const root of roots) {
165
+ try {
166
+ const watcher = fs.watch(root, { recursive: true }, (_event, filename) => {
167
+ if (!filename)
168
+ return;
169
+ const filePath = path.isAbsolute(filename.toString())
170
+ ? filename.toString()
171
+ : path.join(root, filename.toString());
172
+ listener({ path: filePath });
173
+ });
174
+ watchers.push(watcher);
175
+ }
176
+ catch {
177
+ // A root that can't be watched is skipped; the cron fallback covers it.
178
+ }
179
+ }
180
+ return () => {
181
+ for (const w of watchers) {
182
+ try {
183
+ w.close();
184
+ }
185
+ catch {
186
+ // best-effort teardown
187
+ }
188
+ }
189
+ };
190
+ },
191
+ };
192
+ }
193
+ /**
194
+ * Run `akm extract --watch`: watch every available harness's session-log
195
+ * roots and run extract (debounced, per-harness) on change. Stays alive until
196
+ * SIGINT/SIGTERM, then stops cleanly. PROCESS-HYGIENE: stop() removes every
197
+ * watcher + pending timer before the process exits.
198
+ */
199
+ async function runWatchMode(opts) {
200
+ const targets = getWatchTargets();
201
+ if (targets.length === 0) {
202
+ output("extract", {
203
+ schemaVersion: 1,
204
+ ok: false,
205
+ shape: "extract-watch-started",
206
+ warnings: ["no watchable harness session-log directories found on this machine"],
207
+ watching: [],
208
+ });
209
+ return;
210
+ }
211
+ const allRoots = targets.flatMap((t) => t.roots);
212
+ const eventSource = createFsWatchEventSource(allRoots);
213
+ const handle = akmExtractWatch({
214
+ roots: targets,
215
+ eventSource,
216
+ debounceMs: opts.debounceMs,
217
+ onTrigger: async (harnessName) => {
218
+ await akmExtract({
219
+ type: harnessName,
220
+ dryRun: opts.dryRun,
221
+ force: opts.force,
222
+ ...(opts.since ? { since: opts.since } : {}),
223
+ });
224
+ },
225
+ });
226
+ output("extract", {
227
+ schemaVersion: 1,
228
+ ok: true,
229
+ shape: "extract-watch-started",
230
+ debounceMs: opts.debounceMs,
231
+ watching: allRoots,
232
+ });
233
+ await new Promise((resolve) => {
234
+ const shutdown = () => {
235
+ handle.stop();
236
+ resolve();
237
+ };
238
+ process.once("SIGINT", shutdown);
239
+ process.once("SIGTERM", shutdown);
240
+ });
241
+ }
@@ -21,12 +21,18 @@ import promptTemplate from "../../assets/prompts/extract-session.md" with { type
21
21
  *
22
22
  * Shape:
23
23
  * {
24
- * "candidates": [{type, name, description, when_to_use?, body, confidence, evidence}, ...],
24
+ * "candidates": [{type, name, description, when_to_use?, body, confidence, evidence,
25
+ * orderedActions?, outcomeData?}, ...],
25
26
  * "rationale_if_empty"?: string
26
27
  * }
27
28
  *
28
29
  * `additionalProperties: false` at each level so any hallucinated keys are
29
30
  * dropped before parsing.
31
+ *
32
+ * `orderedActions` and `outcomeData` are additive fields for #615 procedural
33
+ * compilation (WS-0 data-capture hook). Source transcripts are external logs
34
+ * not guaranteed re-extractable; we capture the ordered-action sequence and
35
+ * outcome now even though the detection/compilation feature is deferred to 0.10+.
30
36
  */
31
37
  export const EXTRACT_JSON_SCHEMA = {
32
38
  type: "object",
@@ -79,6 +85,17 @@ export const EXTRACT_JSON_SCHEMA = {
79
85
  minLength: 5,
80
86
  description: "One-line pointer to the moment in the session that supports this candidate.",
81
87
  },
88
+ orderedActions: {
89
+ type: "array",
90
+ description: "OPTIONAL. Ordered list of discrete actions taken during this episode, as brief imperative phrases (e.g. 'run deploy.sh', 'check VPN status', 'retry with --force'). Capture only when the candidate represents a recurring action sequence that could become a skill or workflow. Omit when the candidate is a standalone fact or observation.",
91
+ items: { type: "string", minLength: 3 },
92
+ maxItems: 20,
93
+ },
94
+ outcomeData: {
95
+ type: "string",
96
+ description: "OPTIONAL. One-sentence description of the outcome when the ordered action sequence completed (e.g. 'deploy succeeded after VPN reconnect', 'build failed with error X'). Required when orderedActions is present; omit otherwise.",
97
+ maxLength: 400,
98
+ },
82
99
  },
83
100
  },
84
101
  },
@@ -207,6 +224,20 @@ export function parseExtractPayload(stdout) {
207
224
  };
208
225
  if (typeof c.when_to_use === "string")
209
226
  candidate.when_to_use = c.when_to_use.trim();
227
+ // #615 WS-0: parse optional ordered-action + outcome fields.
228
+ // Defensive: silently drop malformed entries rather than rejecting the whole candidate.
229
+ if (Array.isArray(c.orderedActions) && c.orderedActions.length > 0) {
230
+ const actions = c.orderedActions
231
+ .filter((a) => typeof a === "string" && a.trim().length >= 3)
232
+ .map((a) => a.trim())
233
+ .slice(0, 20);
234
+ if (actions.length > 0) {
235
+ candidate.orderedActions = actions;
236
+ if (typeof c.outcomeData === "string" && c.outcomeData.trim().length > 0) {
237
+ candidate.outcomeData = c.outcomeData.trim().slice(0, 400);
238
+ }
239
+ }
240
+ }
210
241
  candidates.push(candidate);
211
242
  }
212
243
  const result = { candidates };