akm-cli 0.10.26101101-alpha → 0.10.26101201-alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/dist/commands/improve/consolidate/continuity-check.js +4 -4
- package/dist/commands/improve/reflect.js +2 -33
- package/dist/commands/improve/retrieval-queries.js +37 -0
- package/docs/architecture/0.10-process-rules.md +56 -63
- package/docs/architecture/improvement.md +1 -25
- package/docs/architecture/internals/improve-workflow.md +3 -4
- package/docs/reference/configuration.md +1 -1
- package/package.json +1 -1
- package/dist/assets/prompts/retrieval-relevance-judge.md +0 -6
- package/dist/commands/improve/retrieval-gate.js +0 -134
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.10.26101201] - 2026-10-12
|
|
10
|
+
|
|
11
|
+
### Removed
|
|
12
|
+
|
|
13
|
+
- **Reflect's retrieval regression gate (#1171).** After the quality judge passed a revision, reflect graded the old and the new text against up to five of the asset's own retrieval queries and refused the revision (`quality_rejected`, a 14-day window) when the mean grade fell at all. Re-measured on 0.10 it refused 12-17% of the revisions the judge had passed and lowered defects fixed from 84 to 77 (35B) and from 92 to 77 (27B) of 96, now that reflect edits frontmatter only. A judge-passed revision is now minted as a proposal with no further check. This also removes the `retrieval-ungraded` review reason, the `retrievalRegression`/`retrievalUngraded` event metadata and the `retrieval-relevance-judge.md` prompt. No config key existed for the gate, so no config changes.
|
|
14
|
+
|
|
9
15
|
## [0.10.26101101] - 2026-10-11
|
|
10
16
|
|
|
11
17
|
### Changed
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { searchLocal } from "../../../indexer/search/db-search.js";
|
|
5
5
|
import { stripFrontmatterBody } from "../content-hash.js";
|
|
6
6
|
import { stripBundle } from "../ledger.js";
|
|
7
|
-
import { loadRetrievalQueries } from "../retrieval-
|
|
7
|
+
import { loadRetrievalQueries } from "../retrieval-queries.js";
|
|
8
8
|
/** At most this many of the retired asset's most recent queries are replayed (plan §5.4, §7). */
|
|
9
9
|
export const CONTINUITY_MAX_QUERIES = 5;
|
|
10
10
|
/** "Top 10" per the plan's rule — both the pass/fail cutoff and the search `limit`. */
|
|
@@ -63,8 +63,8 @@ function normalizedBody(raw) {
|
|
|
63
63
|
*
|
|
64
64
|
* S3: two fixes against false flags measured on a real night-1 admission
|
|
65
65
|
* (300 pairs, 6 flags, half spurious):
|
|
66
|
-
* - queries are the
|
|
67
|
-
*
|
|
66
|
+
* - queries are the cleaned set `loadRetrievalQueries` returns
|
|
67
|
+
* (`../retrieval-queries.ts`) — stash-README
|
|
68
68
|
* boilerplate, harness/tool envelopes, pastes over 2,000 characters, and
|
|
69
69
|
* duplicates are dropped before replay, not just capped at 5 raw entries;
|
|
70
70
|
* - when the retired and successor bodies normalize identical, the check
|
|
@@ -92,7 +92,7 @@ export async function checkRetirementContinuity(args) {
|
|
|
92
92
|
// no real risk here to check for.
|
|
93
93
|
if (normalizedBody(args.retiredRaw) === normalizedBody(args.successorRaw))
|
|
94
94
|
return undefined;
|
|
95
|
-
// S3a: the
|
|
95
|
+
// S3a: the cleaned queries from `loadRetrievalQueries` —
|
|
96
96
|
// boilerplate, envelopes, pastes and duplicates dropped before replay.
|
|
97
97
|
const queries = loadRetrievalQueries(args.ledgerAccess, args.retiredRef).slice(0, CONTINUITY_MAX_QUERIES);
|
|
98
98
|
if (queries.length === 0)
|
|
@@ -45,7 +45,6 @@ import { findAssetFilePath } from "./eligibility.js";
|
|
|
45
45
|
import { resolveImproveExecution } from "./execution.js";
|
|
46
46
|
import { recordLedgerAttempt } from "./ledger.js";
|
|
47
47
|
import { classifyReflectChange, findReflectDefect, splitFrontmatter } from "./reflect-noise.js";
|
|
48
|
-
import { loadRetrievalQueries, runRetrievalRegressionGate } from "./retrieval-gate.js";
|
|
49
48
|
import { callStageOnce, errMessage, mintProposal, noticeSet, rejectedProposalContext, resolveQualityGateJudge, runReflectQualityJudge, } from "./stage.js";
|
|
50
49
|
const MAX_FEEDBACK_LINES = 10;
|
|
51
50
|
const MAX_GLOBAL_FEEDBACK_LINES = 20;
|
|
@@ -865,41 +864,12 @@ async function finalizeReflectProposal(args) {
|
|
|
865
864
|
}, `Reflect proposal quality gate rejected: score=${verdict.score}, reason="${verdict.reason}"`);
|
|
866
865
|
}
|
|
867
866
|
}
|
|
868
|
-
// #722: a revision of an existing asset must not grade lower on its own retrieval queries.
|
|
869
|
-
let retrievalUngraded;
|
|
870
|
-
if (verdict?.pass && judge.runner) {
|
|
871
|
-
const retrieval = await runRetrievalRegressionGate({
|
|
872
|
-
ref: payload.ref,
|
|
873
|
-
before: assetContent,
|
|
874
|
-
after: content,
|
|
875
|
-
queries: loadRetrievalQueries({ proposalsCtx: options.ctx, eventsCtx: options.eventsCtx }, payload.ref),
|
|
876
|
-
runner: judge.runner,
|
|
877
|
-
...(options.chat ? { chat: options.chat } : {}),
|
|
878
|
-
...(Object.hasOwn(options, "timeoutMs") ? { timeoutMs: options.timeoutMs } : {}),
|
|
879
|
-
...(options.signal ? { signal: options.signal } : {}),
|
|
880
|
-
onNotices: run.notices.add,
|
|
881
|
-
});
|
|
882
|
-
// A check that could not grade measured nothing: a person decides, as when the judge gave no verdict.
|
|
883
|
-
if (retrieval.ungraded)
|
|
884
|
-
retrievalUngraded = retrieval.reason;
|
|
885
|
-
if (!retrieval.pass && !retrieval.ungraded) {
|
|
886
|
-
return refuse(retrieval.reason, {
|
|
887
|
-
retrievalRegression: true,
|
|
888
|
-
retrievalQueries: retrieval.queries,
|
|
889
|
-
...(retrieval.oldMean !== undefined ? { retrievalGradeBefore: retrieval.oldMean } : {}),
|
|
890
|
-
...(retrieval.newMean !== undefined ? { retrievalGradeAfter: retrieval.newMean } : {}),
|
|
891
|
-
}, `Reflect proposal refused: ${retrieval.reason}`);
|
|
892
|
-
}
|
|
893
|
-
}
|
|
894
867
|
const reviewReasons = [
|
|
895
868
|
...(judge.skippedNoJudge ? ["no-judge-configured"] : []),
|
|
896
869
|
...(judgeFailed ? ["judge-error"] : []),
|
|
897
870
|
...(judgeUncertain ? ["judge-uncertain"] : []),
|
|
898
|
-
...(retrievalUngraded ? ["retrieval-ungraded"] : []),
|
|
899
871
|
];
|
|
900
|
-
const gateReview = judgeFailed || judgeUncertain
|
|
901
|
-
// What the person is shown: the check's words when it could not grade, otherwise the judge's.
|
|
902
|
-
const reviewNote = retrievalUngraded ?? verdict?.reason;
|
|
872
|
+
const gateReview = judgeFailed || judgeUncertain;
|
|
903
873
|
const proposal = mintProposal(run.stash, options.ctx, {
|
|
904
874
|
ref: payload.ref,
|
|
905
875
|
...(options.target ? { target: options.target } : {}),
|
|
@@ -916,7 +886,7 @@ async function finalizeReflectProposal(args) {
|
|
|
916
886
|
// The quality gate's hand-off to a person, as distill's: the triage drain leaves it alone.
|
|
917
887
|
gate: gateReview ? "quality-gate" : "reflect",
|
|
918
888
|
...(gateReview && verdict?.criteria ? { scores: verdict.criteria } : {}),
|
|
919
|
-
...(gateReview &&
|
|
889
|
+
...(gateReview && verdict?.reason ? { judgeReason: verdict.reason } : {}),
|
|
920
890
|
},
|
|
921
891
|
}
|
|
922
892
|
: { judged: verdict });
|
|
@@ -930,7 +900,6 @@ async function finalizeReflectProposal(args) {
|
|
|
930
900
|
...(judge.skippedNoJudge ? { qualityGateSkippedNoJudge: true } : {}),
|
|
931
901
|
...(judgeFailed || judgeUncertain ? { qualityReason: verdict?.reason } : {}),
|
|
932
902
|
...(judgeUncertain ? { qualityScore: verdict?.score } : {}),
|
|
933
|
-
...(retrievalUngraded ? { qualityReason: retrievalUngraded, retrievalUngraded: true } : {}),
|
|
934
903
|
...telemetry,
|
|
935
904
|
},
|
|
936
905
|
}, options.eventsCtx);
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
+
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
+
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
+
/**
|
|
5
|
+
* An asset's own retrieval queries from the usage log, cleaned for replay.
|
|
6
|
+
* The consolidate continuity check replays them against a retiring asset's
|
|
7
|
+
* successor.
|
|
8
|
+
*/
|
|
9
|
+
import { nonTaskInput } from "../../core/non-task-input.js";
|
|
10
|
+
import { listRetrievalQueries } from "../../indexer/usage/usage-events.js";
|
|
11
|
+
import { readLedgerDb, stripBundle } from "./ledger.js";
|
|
12
|
+
/** Queries kept per asset. */
|
|
13
|
+
const MAX_QUERIES = 5;
|
|
14
|
+
/** The retrieval suite treats a longer input as a paste, not a query. */
|
|
15
|
+
const MAX_QUERY_CHARS = 2000;
|
|
16
|
+
/** Up to five distinct task queries, in the given order, whitespace collapsed. */
|
|
17
|
+
export function usableRetrievalQueries(raw) {
|
|
18
|
+
const out = [];
|
|
19
|
+
for (const text of raw) {
|
|
20
|
+
const query = text.replace(/\s+/g, " ").trim();
|
|
21
|
+
if (!query || query.length > MAX_QUERY_CHARS || nonTaskInput(query) || out.includes(query))
|
|
22
|
+
continue;
|
|
23
|
+
out.push(query);
|
|
24
|
+
if (out.length === MAX_QUERIES)
|
|
25
|
+
break;
|
|
26
|
+
}
|
|
27
|
+
return out;
|
|
28
|
+
}
|
|
29
|
+
/** The asset's own retrieval queries from the usage log (none when state.db cannot be read). */
|
|
30
|
+
export function loadRetrievalQueries(access, ref) {
|
|
31
|
+
try {
|
|
32
|
+
return usableRetrievalQueries(readLedgerDb(access, (db) => listRetrievalQueries(db, stripBundle(ref))) ?? []);
|
|
33
|
+
}
|
|
34
|
+
catch {
|
|
35
|
+
return [];
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -2,16 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
*akm release/0.10.0 · read from this branch (708e1e585 plus the 14 fixes and the policy batch in this PR) · published as 0.10.26101002-alpha · read 2026-10-11 · corrected after review #1168 on 2026-10-11*
|
|
4
4
|
|
|
5
|
-
Every rule below was read from the 0.10 source on this branch (708e1e585 plus the fixes and the policy batch in this PR), not from docs or memory. Each one names what it does, why it exists, and the file and line that implements it (paths are relative to the akm repository). A reason comes from a code comment, the CHANGELOG, the docs or a linked issue; where none exists the entry says *Why: not stated in code/docs*.
|
|
5
|
+
Every rule below was read from the 0.10 source on this branch (708e1e585 plus the fixes and the policy batch in this PR), not from docs or memory. Each one names what it does, why it exists, and the file and line that implements it (paths are relative to the akm repository). A reason comes from a code comment, the CHANGELOG, the docs or a linked issue; where none exists the entry says *Why: not stated in code/docs*. 456 rules across 13 processes.
|
|
6
6
|
|
|
7
7
|
This version applies the corrections from the code review of the first version (PR #1168): inverted rules, wrong citations, and the findings that change what a rule says. Rules that describe a bug fixed on this branch say "(fixed in this PR, #N)", with N the number in the Fixed in this PR list. Figures that come only from a GitHub issue (#986's 24,051 auto-accepted proposals, #1167's 129 pending proposals, #900's 19-21 minutes of CPU, #1092's 21 retirements) appear nowhere in the repository and are quoted as issue claims.
|
|
8
8
|
|
|
9
|
-
This version adds the policy batch: Policy decisions 1, 3, 4, 5 and
|
|
9
|
+
This version adds the policy batch: Policy decisions 1, 3, 4, 5, 7 and 9 are implemented in this PR (decision 9 removes reflect's retrieval gate), decisions 2, 6 and 8 are resolved with no code change. A rule whose behaviour changed says "(changed in this PR, decision N)", and every citation to the files those commits touched was moved to the line numbers at this branch's HEAD.
|
|
10
10
|
|
|
11
11
|
## TLDR
|
|
12
12
|
|
|
13
13
|
- **The improve run** plans from fresh feedback plus a capped proactive lane, works only on the write-target bundle, and only touches assets someone retrieved in the last 90 days or new material. It takes a PID lock, syncs a git-backed bundle like `akm sync` (commit pending managed-path edits, fetch, merge, push) before it reads anything, has a 2-hour budget, commits and pushes only the files it wrote, and records every run in `improve_runs`.
|
|
14
|
-
- **Reflect** runs only on fresh negative feedback (or the proactive lane) and may change only `description`, `when_to_use` and a title, never the body text (a title is added as a `# heading` when the body has none). The judge must score every criterion 4 or higher,
|
|
14
|
+
- **Reflect** runs only on fresh negative feedback (or the proactive lane) and may change only `description`, `when_to_use` and a title, never the body text (a title is added as a `# heading` when the body has none). The judge must score every criterion 4 or higher, and nothing is checked after it. A non-pass with a mean of 3.5 or more and a judge outage go to a person as a pending proposal; a lower non-pass or a defect is refused with no proposal and a hard 14-day window.
|
|
15
15
|
- **Distill** needs feedback in the last 30 days, and skips a memory whose only feedback is a positive with no reason or note (10 of 11 lessons from such memories were rejected). Lessons that pass the judge still go to a person, never the drain. The proactive lane never feeds distill.
|
|
16
16
|
- **Consolidate** only promotes memories to knowledge; the pair pass retires a duplicate only when the claim lists show nothing would be lost. A retirement is staged for the drain only if the claims are clean, there is no continuity risk and the second look agrees. There is no confidence cutoff. A `captureMode: hot` memory is retired, promoted and archived like any other. The pair pass, the coverage gate and the continuity check all need embeddings, which are off by default.
|
|
17
17
|
- **Triage** accepts only what a deterministic gate passed (a proposal whose judge stamp still matches its content hash, or a retirement the pair pass staged), up to 200 per run. The judgment step is off by default and may only accept consolidate promotions. Proactive-lane proposals, distill lessons and uncertain judge verdicts wait for a person.
|
|
@@ -19,23 +19,21 @@ This version adds the policy batch: Policy decisions 1, 3, 4, 5 and 7 are implem
|
|
|
19
19
|
|
|
20
20
|
## Policy decisions
|
|
21
21
|
|
|
22
|
-
*The owner's answers, 2026-10-11. Decisions 1, 3, 4, 5 and
|
|
22
|
+
*The owner's answers, 2026-10-11. Decisions 1, 3, 4, 5, 7 and 9 are implemented in this PR, and 2, 6 and 8 are resolved with no code change. Each rule a decision changed says "(changed in this PR, decision N)".*
|
|
23
23
|
|
|
24
24
|
1. **Feedback and ranking** — *implemented in this PR.* `akm search` and `akm curate` multiply each hit's fused RRF score by 1 + 0.03 × (ln(1+pos) − ln(1+neg)), where pos and neg count the asset's user-attributed (`source='user'`) feedback events, so 5 positives lift a hit more than 2 and each further vote counts for less. Improve-attributed feedback does not move ranking. `utility_scores` is untouched and still read only by improve eligibility, salience and metrics; search does not read it. (`src/indexer/search/ranking.ts:62-93`, `src/indexer/search/db-search.ts:336-338`, `src/indexer/usage/usage-events.ts:122-148`)
|
|
25
|
-
2. **Proactive lane** — *
|
|
25
|
+
2. **Proactive lane** — *decided: keep it on at cap 15; its proposals wait for a person ([itlackey/akm#1172](https://github.com/itlackey/akm/issues/1172)).* Re-measured on 0.10 without the drain (akm-eval nightly, public corpus, 6 silent defects × 3 runs, two lab models): the lane left a correct fix pending for a person on 12 of 18 silent defects (35B) and 18 of 18 (27B), against 0 with the lane off; a cap of 50 found nothing extra, nothing was applied unattended, and the cost was about 25% more model calls. The earlier "0 → 6-7 defects fixed" figure was taken with the drain accepting the picks, which #1147 removed, and is not comparable. No code change.
|
|
26
26
|
3. **Exact fixes typed by a person** (`--replace/--with/--source`, `--superseded-by`, `--outdated`) — *implemented in this PR.* `akm feedback` mints the proposal, stamps gate `auto-accepted` / `manual-fix` and accepts it in the same command through `akmProposalAccept`: a normal accept with a normal commit, and `akm proposal revert` works. A refused accept rejects the proposal and fails the command. (`src/commands/feedback-cli.ts:793-824`)
|
|
27
|
-
4. **Uncertain verdicts go to a person** — *implemented in this PR.* A reflect non-pass with a mean of 3.5 or more (`REFLECT_REVIEW_MIN_SCORE`) becomes a pending `deferred` proposal, reason `judge-uncertain`, with no hard window; below 3.5 it is refused as before. A
|
|
27
|
+
4. **Uncertain verdicts go to a person** — *implemented in this PR.* A reflect non-pass with a mean of 3.5 or more (`REFLECT_REVIEW_MIN_SCORE`) becomes a pending `deferred` proposal, reason `judge-uncertain`, with no hard window; below 3.5 it is refused as before. A distill judge with no engine records `review_needed` instead of a 30-day `quality_rejected`. Triage never auto-accepts any of these. The band's data (shipped judge, lab 27B Q2): at 3.5 or more, 17 good and 9 bad among 113 frontmatter-only proposals; below 3.5, 396 labelled body rewrites were 6 good and 285 bad. (`src/commands/improve/reflect.ts:1004-1018,1091-1105,1118-1123`, `src/commands/improve/stage.ts:607-613`)
|
|
28
28
|
5. **Hot memories** — *implemented in this PR.* There is no hot-specific retire guard, chunk skip or prompt annotation, and `hasHotCaptureMode` and `isHotCapturedMemory` are gone: a `captureMode: hot` memory is retired, promoted and archived like any memory. (`src/commands/improve/consolidate/pair-pass.ts:765-773`, `src/commands/improve/consolidate.ts:642-646,798`, `src/commands/improve/consolidate/chunking.ts:31-35,57`)
|
|
29
|
-
6. **Distill pass bar and grounding** — *
|
|
29
|
+
6. **Distill pass bar and grounding** — *decided: no grounding in the pass bar; the outcome is unchanged because pass and review both go to a person ([itlackey/akm#1173](https://github.com/itlackey/akm/issues/1173)).* Replaying judged lessons with grounding added to the bar (akm-eval distill/pass-bar, two lab models) only moves lessons from pass to review (6 good lessons on the 35B, 2 on the 27B). A passing lesson is minted for a person (`distill-review`) and so is one sent to review, and only the reject line (grounding 2 or less) drops a lesson, which is the same under both bars. No code change.
|
|
30
30
|
7. **Sync before every improve run** — *implemented in this PR.* A live run calls the same `saveGitStash` sync as `akm sync` right after taking its lock: it commits pending managed-path edits, fetches, fast-forwards or merges, and pushes per the run's push setting. The outcome is `result.preRunSync`, and `stash_synced` events carry a `phase`. A conflict or fetch failure warns and the run continues; there is no agent hand-off (the owner wants one only if it proves needed). `akm sync` itself fetches and merges (fixed in this PR, #5). (`src/commands/improve/improve.ts:286-288,364,977-1004`)
|
|
31
31
|
8. **Nightly memory-inference warning** — *resolved: by design.* The warning `[improve] memoryInference skipped … Set experimental.improveAutonomy: true` fires only on installs without the opt-in, because a gated lane must never be a silent no-op (`src/commands/improve/improve.ts:1073-1074`, `src/commands/improve/autonomy-gate.ts:26-30,113-116`). On the owner's install `improveAutonomy` is true: memory inference ran every night 10-09 to 10-11 (9, 21 and 4 facts) and state.db holds 0 `autonomy_gated` events since 09-12. No code change.
|
|
32
|
-
9. **Re-measure** the defect filter, the retrieval gate, staged-retirement precision and the proactive-lane benefit with akm-eval, adding tooling there if it is missing — *
|
|
32
|
+
9. **Re-measure** the defect filter, the retrieval gate, staged-retirement precision and the proactive-lane benefit with akm-eval, adding tooling there if it is missing — *done on 0.10 (akm `release/0.10.0` at 3f1156232, two lab models, `--repeat 3`); the retrieval gate is removed in this PR ([itlackey/akm#1174](https://github.com/itlackey/akm/issues/1174)).* The reflect defect filter is kept: it refused 0 of 420 frontmatter-only edits the two models wrote, and on 106 labelled real edits 5 of 78 bad and 0 of 28 good. The retrieval gate is removed ([itlackey/akm#1171](https://github.com/itlackey/akm/issues/1171)): it refused 12-17% of the revisions the judge passed and lowered defects fixed from 84 to 77 (35B) and from 92 to 77 (27B) of 96. Staged-retirement precision is kept as is: 298 of 298 staged retirements were safe, but the consolidate eval uses the deterministic embedder and records no queries, so the continuity check never fires there and is untested by the eval. The proactive lane is kept (decision 2).
|
|
33
33
|
|
|
34
34
|
## Needs a decision
|
|
35
35
|
|
|
36
|
-
- **Docs say reflect runs only from negative feedback.** The proactive lane also plans reflect on assets with no feedback, and since #1147 its proposals wait for a person, so unattended the lane fixes nothing. Re-measurement in progress (akm-eval): Policy decision 2. (`default.json:2`, `docs/guides/improve-the-library.md:106`)
|
|
37
36
|
- **Positive feedback without a reason starves distill.** Distill skips it by design, but the shipped hint files tell agents to run `akm feedback <ref> --positive` with no reason (`src/assets/hints/cli-hints-short.md:11,42`, `cli-hints-full.md:96-104`, `src/assets/stash-skeleton/README.md:81`, `docs/reference/cli.md:1792-1795`). The `AGENTS.md` text in your Codex and OpenPalm worktrees says the same. `docs/reference/cli.md:1793` also shows `akm feedback agents/reviewer --negative` with no `--reason`, which is a usage error by default. The bare-positive guard also blocks the promotion path, which needs no writer.
|
|
38
|
-
- **The distill pass bar and re-measurement.** Re-measurement in progress (akm-eval): see Policy decisions 6 and 9.
|
|
39
37
|
- **Stale wording in code:** a health check still mentions a staging "confidence threshold" that no longer exists (`src/commands/health/checks.ts:1533`); the salience and valence header comments describe behaviour the code no longer has; `consolidate.json:2` says "merges, deduplicates, and prunes" (the promote pass is promote-only) and "every 4h" although a strategy encodes no schedule; the `--promote` help says "Default is queue mode" (`proposal-cli.ts:436`); `dir-staleness.ts:15,58` describe a fingerprint the code no longer computes; `memory-inference.ts:6,22-30` and `memory-infer.ts:6` describe an `akm index` path; `extract-cli.ts:50` says "Default 24h" (the floor is 48h); `improve-processes.ts:188-189` says `minContentChars` is a pre-filter count (it measures raw chars); `improve-strategies.ts:351-357` describes `allowAllDisabled`; `consolidate.ts:919-920` still says a judged memory "waits out its revisit window". Each section's "Code and doc mismatches" list has the details.
|
|
40
38
|
|
|
41
39
|
*Correction to an earlier report: the 38 pair proposals with no gate decision on 10-11 were not held by a confidence cutoff. Nothing gates on confidence; they failed one of the three staging conditions above.*
|
|
@@ -146,7 +144,7 @@ These 14 bugs are fixed on this branch; the rules below describe the fixed behav
|
|
|
146
144
|
|
|
147
145
|
- **Feedback becomes a signal event** — `akm feedback` appends a `feedback` event keyed on the entry's durable `item_ref`. Its metadata carries `signal`, optional `reason`, `tags`, a `contentHash` (body hash) of the text the feedback judged, and `fix` when `--replace`/`--superseded-by`/`--outdated` was used. For a user-attributed event it also updates utility at once. *Why:* the hash lets improve tell whether the body has been edited since the complaint. (`src/commands/feedback-cli.ts:661-665,734-736,776,787-791`)
|
|
148
146
|
- **Negative feedback requires a reason** — Negative feedback without `--reason` is a usage error by default. Setting `feedback.requireReason:false` in akm.json downgrades it to a warning. *Why:* a bare negative says nothing about what is wrong. (`feedback-cli.ts:648-658`)
|
|
149
|
-
- **Utility update (one formula, shared with `akm index`)** (changed in this PR) — `akm feedback` no longer has its own utility formula. For a user-attributed event it calls `recomputeUtilityScores(indexDb, stateDb, entryId)`, the function the `akm index` finalize pass runs for every entry, scoped to this one entry: `utility = prev × 0.7^days + max(selectRate, max(0, pos − neg) / total) × (1 − 0.7^days)`, with `prev` 0 for an entry with no row yet, `days` the time since the last full recompute (at least 1), and only `source='user'` events counted. So the value stored at feedback time is the value `akm index` would compute at that moment; a scoped call skips the 90-day usage-event purge and leaves the `last_utility_computed_at` watermark alone, so the next full `akm index` decays the value one more step, exactly as running `akm index` twice does. A positive therefore raises utility (from 0 to 0.3, then 0.51 on a second), a negative alone only decays it, and one positive plus one negative is rate 0. If a utility at or above 0.5 falls below 0.5, an `improve_review_needed` event is emitted; nothing consumes the event (only its emitter exists). *Why:* two writers with two formulas left the stored value depending on which ran last; the bounded-step EMA it replaced (`FEEDBACK_LR` 0.1 toward the cumulative positive share; MemRL arXiv:2601.03192, F-5 / #386) is deleted with `utility-policy.ts`. `akm search` reads neither: it counts the feedback events itself (see Search: Feedback scales the fused score; changed in this PR, decision 1). (`src/commands/feedback-cli.ts:173-240,834-854`, `src/indexer/indexer.ts:1839-
|
|
147
|
+
- **Utility update (one formula, shared with `akm index`)** (changed in this PR) — `akm feedback` no longer has its own utility formula. For a user-attributed event it calls `recomputeUtilityScores(indexDb, stateDb, entryId)`, the function the `akm index` finalize pass runs for every entry, scoped to this one entry: `utility = prev × 0.7^days + max(selectRate, max(0, pos − neg) / total) × (1 − 0.7^days)`, with `prev` 0 for an entry with no row yet, `days` the time since the last full recompute (at least 1), and only `source='user'` events counted. So the value stored at feedback time is the value `akm index` would compute at that moment; a scoped call skips the 90-day usage-event purge and leaves the `last_utility_computed_at` watermark alone, so the next full `akm index` decays the value one more step, exactly as running `akm index` twice does. A positive therefore raises utility (from 0 to 0.3, then 0.51 on a second), a negative alone only decays it, and one positive plus one negative is rate 0. If a utility at or above 0.5 falls below 0.5, an `improve_review_needed` event is emitted; nothing consumes the event (only its emitter exists). *Why:* two writers with two formulas left the stored value depending on which ran last; the bounded-step EMA it replaced (`FEEDBACK_LR` 0.1 toward the cumulative positive share; MemRL arXiv:2601.03192, F-5 / #386) is deleted with `utility-policy.ts`. `akm search` reads neither: it counts the feedback events itself (see Search: Feedback scales the fused score; changed in this PR, decision 1). (`src/commands/feedback-cli.ts:173-240,834-854`, `src/indexer/indexer.ts:1839-1950`; test `tests/integration/feedback-command.test.ts`, "stores the utility the index finalize pass would compute")
|
|
150
148
|
- **What counts as a signal event** — A feedback event with a string `signal` or a string `note`. Only these count inside the 30-day window. The `note` clause is dead for 0.10's own writer: `akm feedback` has no `--note` flag and writes `signal`, `reason` and `tags`; `note` survives only in the legacy inline-ref grammar. (`preparation.ts:678-684`, `feedback-cli.ts:648-665`, `src/integrations/session-logs/inline-refs.ts`)
|
|
151
149
|
- **Signal window is 30 days** — `FEEDBACK_SIGNAL_WINDOW_DAYS = 30`. *Why:* not stated. (`preparation.ts:678`)
|
|
152
150
|
- **Reflect is planned only from fresh negative feedback** — A ref passes the reflect delta only if it has a negative signal in the window, newer than the reflect ledger row's `lastAttemptAt`, and not blocked by a hard ledger window. Positive and note-only signals never plan a reflect. *Why:* a rewrite is planned from negative feedback only, never from positive signal or on a proactive cadence. (`preparation.ts:855-863`, `docs/architecture/internals/improve-workflow.md` intro and "Ledger pre-filter")
|
|
@@ -190,7 +188,7 @@ These 14 bugs are fixed on this branch; the rules below describe the fixed behav
|
|
|
190
188
|
- **Final ordering** — Candidates sort by (dampened) salience descending, then by ref ascending. The cap that follows takes reflect-path refs before distill-only refs (see Limit ordering). (`preparation.ts:963-969`, `planner.ts:84-90`)
|
|
191
189
|
- **Retrieval counts and last-use** — Counts come from `getRetrievalCounts(indexDb, stateDb, refs, {sourceName})`. Last-use comes from the utility `lastUsedAt`. The pass warns if there are no `show` events in `usage_events`, because the retrieval scope would then match only search-retrieved assets. (`preparation.ts:1055-1078`, `salience.ts:136-161`)
|
|
192
190
|
- **Reflect noise gate** — A candidate edit identical up to trailing whitespace is `noop`. One identical after YAML-canonicalising frontmatter and unwrapping prose is `cosmetic`. Neither becomes a proposal. Anything uncertain is `substantive`, because letting a cosmetic edit through costs a review but suppressing a real fix loses work. *Why:* #580 (empty and cosmetic edits each cost an LLM call and a review slot). (`src/commands/improve/reflect-noise.ts:5-37,218-311`, #580)
|
|
193
|
-
- **Reflect defect filter** — A revision is refused unjudged if it adds a placeholder phrase (default list is "please confirm", "please verify", "to be confirmed/determined/verified"), meta-commentary (for example "feedback signal(s)", "this revision", "the source asset/note/memory", "quality gate rejected", "proposal rejected"), or copies a frontmatter key as a body line or copies a provenance value (`sources`, `xrefs`, `contradictedBy`, 12+ chars) into the body. It counts only what the revision adds. `processes.reflect.defectFilter` lists replace a rule's defaults; an empty list turns the rule off. *Why:* on 396 labelled reflect edits (83 good, 313 bad) the default lists hit 22 bad edits and no good one. TODO/TBD/FIXME are deliberately not listed because an asset may carry them on purpose. The measurement was taken on body-rewriting reflect; reflect is now frontmatter-only, so `placeholder_added` and `meta_commentary_added` can still fire (they scan the whole text including frontmatter values), the key-line half of `frontmatter_copied_into_body` cannot (it reads body prose lines only), and the provenance half only in a contrived case. Re-
|
|
191
|
+
- **Reflect defect filter** — A revision is refused unjudged if it adds a placeholder phrase (default list is "please confirm", "please verify", "to be confirmed/determined/verified"), meta-commentary (for example "feedback signal(s)", "this revision", "the source asset/note/memory", "quality gate rejected", "proposal rejected"), or copies a frontmatter key as a body line or copies a provenance value (`sources`, `xrefs`, `contradictedBy`, 12+ chars) into the body. It counts only what the revision adds. `processes.reflect.defectFilter` lists replace a rule's defaults; an empty list turns the rule off. *Why:* on 396 labelled reflect edits (83 good, 313 bad) the default lists hit 22 bad edits and no good one. TODO/TBD/FIXME are deliberately not listed because an asset may carry them on purpose. The measurement was taken on body-rewriting reflect; reflect is now frontmatter-only, so `placeholder_added` and `meta_commentary_added` can still fire (they scan the whole text including frontmatter values), the key-line half of `frontmatter_copied_into_body` cannot (it reads body prose lines only), and the provenance half only in a contrived case. Re-measured on 0.10 and kept: it refused 0 of 420 frontmatter-only edits the two lab models wrote, and on 106 labelled real edits 5 of 78 bad and 0 of 28 good (Policy decision 9). (`reflect-noise.ts:39-184`)
|
|
194
192
|
- **Distill-loop guards (planned-ref level)** — For a planned memory, distill is skipped when any of these hold, unless the scope is an explicit ref. Each skip is recorded as `distill-skipped` with an `improve_skipped` event. The three content guards (flagged wrong, bare positive, deprecated) each record ledger `unchanged`; only `memory_distill_requires_feedback` writes no row. The guards run on the planned refs after the limit cut, so a skipped ref has already used a slot, and the selection-time delta counts a bare positive as a signal.
|
|
195
193
|
- It was reached without a feedback signal (so not as distill-only): `memory_distill_requires_feedback`.
|
|
196
194
|
- It was flagged wrong and not edited since: `distill_flagged_wrong`, ledger `unchanged`.
|
|
@@ -206,7 +204,7 @@ These 14 bugs are fixed on this branch; the rules below describe the fixed behav
|
|
|
206
204
|
|
|
207
205
|
### Proactive lane
|
|
208
206
|
|
|
209
|
-
- **What it is** — A lane that plans stable, useful assets with no fresh feedback, merged with the signal-bearing refs and ranked together. It runs only for `all` or `type` scope, not `ref` scope. It is off if `proactiveMaintenance` is disabled in the strategy or `--require-feedback-signal` is passed. *Why:* measured on 8 planted silent defects with no feedback: 0 fixed with the picks scored only, 6-7 with them planned, no harm, about 5x the model calls (#1129). That measurement was taken with the drain accepting the picks, which #1147 removed (see Its proposals wait for a person);
|
|
207
|
+
- **What it is** — A lane that plans stable, useful assets with no fresh feedback, merged with the signal-bearing refs and ranked together. It runs only for `all` or `type` scope, not `ref` scope. It is off if `proactiveMaintenance` is disabled in the strategy or `--require-feedback-signal` is passed. *Why:* measured on 8 planted silent defects with no feedback: 0 fixed with the picks scored only, 6-7 with them planned, no harm, about 5x the model calls (#1129). That measurement was taken with the drain accepting the picks, which #1147 removed (see Its proposals wait for a person); re-measured without the drain on 0.10, the lane is kept at cap 15 (Policy decision 2). (`preparation.ts:1080-1102,925,940-946`, `improve-cli.ts:354-359`, CHANGELOG #1129, #1129)
|
|
210
208
|
- **Candidate pool** — Candidates are the no-feedback pool (no in-window signal, not reflect-ledger-blocked) minus those outside the retrieval scope. (`preparation.ts:932-938`)
|
|
211
209
|
- **Due rule** — An asset is due if never reflected or distilled, or if its last attempt was strictly more than `dueDays` ago. Either a reflect or distill attempt resets the clock. Default `dueDays` is 30 (0 or more). *Why:* it doubles as the rotation cooldown, so each asset comes up at most once per `dueDays`. This is the first part of the cap against a proposal flood. (`src/commands/improve/proactive-maintenance.ts:1-14,22-23,67-86`, `improve-processes.ts:220-225`, #1129)
|
|
212
210
|
- **`maxPerRun` bound** — The top `maxPerRun` due assets are taken (default 15, shipped 15; the code fallback was 25 before #1129 and is now 15). Reflect's `limit` (25) also bounds the run, but the cut is by salience with no lane priority, so the picks can displace feedback-bearing refs (see Limit ordering). *Why:* part of the flood cap. (`proactive-maintenance.ts:25-26,89-92,125-131`, `default.json:14`, `docs/architecture/improvement.md` data-flow step 3, CHANGELOG 0.10.26101001)
|
|
@@ -284,7 +282,7 @@ All paths are relative to the `rules-010` worktree. Citation form is `path:line`
|
|
|
284
282
|
- **Newer feedback lifts a soft revisit window, never a rejection.** `isLedgerBlocked(row, now, feedbackAt)` ignores a signal newer than the last attempt for soft windows. For `rejected`, `quality_rejected` and `expired` rows the block holds regardless. *Why:* code comment "Newer feedback lifts a revisit window, never a rejection". (`preparation.ts:857`; `src/storage/repositories/improve-ledger-repository.ts:131-140,191-196`)
|
|
285
283
|
- **Ledger windows set the cooldowns.** Reflect `rejected` or `quality_rejected` blocks 14 days (distill 30, other sources 7). `expired` blocks 1 day. `unchanged`, `proposed`, `review_needed` and `judged_no_action` block 7 days. `accepted` and `failed` have no window. *Why:* "the constants the rejection backoff always used". The ledger replaced several projections that "none of which agreed with each other". (`improve-ledger-repository.ts:6-18,70-82,142-163`)
|
|
286
284
|
- **Explicit `--scope <ref>` bypasses all gates.** Signal delta, ledger window and retrieval scope are skipped, and the lane is tagged `scope`. *Why:* explicit intent is its own evidence. (`preparation.ts:833-834,875-878,950-953`; `src/commands/improve/retrieval-scope.ts:7-12`)
|
|
287
|
-
- **Proactive-maintenance lane also feeds reflect (on in `default`).** Assets with no fresh feedback are planned if due: never touched, or last reflected or distilled more than `dueDays` (default 30) ago. At most `maxPerRun` (15) are picked, ranked by salience, then staleness, then ref. They go to reflect with lane `proactive` (they are never distilled: a pick has no feedback, so the loop skips distill with `memory_distill_requires_feedback`), and `--require-feedback-signal` turns the lane off. *Why:* measured on 8 planted silent defects with no feedback, 0 fixed with picks scored only versus 6-7 when planned, no harm, about 5x the model calls (#1129). The cap is "the cap against a proposal flood". The measurement assumed the drain accepted the picks, which #1147 removed (Policy decision 2). (`src/commands/improve/proactive-maintenance.ts:5-14,23,26,67-86`; `src/assets/improve-strategies/default.json:14`; `preparation.ts:924,944-952`; CHANGELOG 0.10.26101001 "Changed", #1129)
|
|
285
|
+
- **Proactive-maintenance lane also feeds reflect (on in `default`).** Assets with no fresh feedback are planned if due: never touched, or last reflected or distilled more than `dueDays` (default 30) ago. At most `maxPerRun` (15) are picked, ranked by salience, then staleness, then ref. They go to reflect with lane `proactive` (they are never distilled: a pick has no feedback, so the loop skips distill with `memory_distill_requires_feedback`), and `--require-feedback-signal` turns the lane off. *Why:* measured on 8 planted silent defects with no feedback, 0 fixed with picks scored only versus 6-7 when planned, no harm, about 5x the model calls (#1129). The cap is "the cap against a proposal flood". The measurement assumed the drain accepted the picks, which #1147 removed; re-measured without the drain on 0.10, the lane finds most silent defects and a person decides on each one, so it is kept at cap 15 (Policy decision 2). (`src/commands/improve/proactive-maintenance.ts:5-14,23,26,67-86`; `src/assets/improve-strategies/default.json:14`; `preparation.ts:924,944-952`; CHANGELOG 0.10.26101001 "Changed", #1129)
|
|
288
286
|
- **The proactive lane only picks assets retrieval returned, or new material.** Without feedback or an explicit ref, an asset must have been returned by user `search`, `curate` or `show` in the last 90 days (the usage-log retention), or be newly captured material improve never processed. *Why:* in July the proactive lane rewrote 3,069 assets, 3,059 of which had never been retrieved; the pool dropped from 15,686 to 6,450 (#986). (`retrieval-scope.ts:1-12`; `preparation.ts:925-935`; `src/indexer/usage/usage-events.ts:201`; CHANGELOG 0.9.17)
|
|
289
287
|
- **There is no explicit `no_signal` reason, but refs with nothing new are dropped and counted.** Candidates not surviving the signal delta are logged as `distill-skipped`, with reason "no new signal since last proposal" or "not retrieved inside the usage window". Aggregate `improve_skipped` events are `no_new_signal` and `not_retrieved`. The plan gates are `retrieval`, `signal`, `disk` and `limit`. (`preparation.ts:991-1013,1031-1045`)
|
|
290
288
|
- **Reflect `limit` is 25 and is the run cap.** `--limit` beats `processes.reflect.limit`, which beats the strategy `limit`. Candidates are ranked by salience, and a ref whose last 3 runs were no-ops sorts at 0.5x its score. *Why:* "reflect's `limit` (25) bounds the run" against proposal flood. The no-op dampener is described as consolidation-selection only in salience.ts, yet `selectLoopCandidates` applies it to the reflect/distill loop too. (`src/commands/improve/improve.ts:477-485`; `preparation.ts:956-969`; `src/commands/improve/salience.ts:129-134`)
|
|
@@ -293,33 +291,33 @@ All paths are relative to the `rules-010` worktree. Citation form is `path:line`
|
|
|
293
291
|
- **Strategy filters.** Reflect `allowedTypes` defaults to agent, command, knowledge, lesson, memory, skill and workflow. `excludeRefPrefixes` is reflect-only. A disabled process skips. A ref both reflect and distill would refuse is dropped as `strategy_filtered_all_passes`. *Why:* R12: raw wiki-ingest snapshots are type `knowledge`, so type filtering alone can't exclude them. (`src/commands/improve/improve-strategies.ts:36-40,56-83,98-100`; `src/core/config/schema/improve-processes.ts:48-60`)
|
|
294
292
|
- **`.derived` memories are never reflected.** They are machine-generated. (`src/commands/improve/eligibility.ts:284`; `loop-stages.ts:201-206`)
|
|
295
293
|
- **The run only improves its own write-target bundle.** Candidates come only from the writable bundle the run writes to. *Why:* reflect reads from the owning bundle but files the proposal in the write target, so another bundle's asset would be read from one and filed in another (#1000). (`eligibility.ts:142-148,204-206`; `src/commands/proposal/repository.ts:410-416`)
|
|
296
|
-
- **Refuses before calling the model when the file is not where a proposal would write.** A skill's `references/a.md` is indexed as `knowledge/skills/<name>/references/a`, so a proposal would be a `create` and accepting it adds a second file. The result is `unsupported_type` with subreason `file_outside_layout`, recorded as a skip. *Why:* #1052. (`reflect.ts:
|
|
297
|
-
- **Reflect time budget.** Reflect gets `timeoutMs` equal to the improve run's remaining budget, and the shared abort signal. With no timeout set it uses `DEFAULT_LLM_TIMEOUT_MS` (600,000 ms). A call cut by the budget returns a failure and writes no ledger row, so the feedback still plans the ref next run (see A transient failure leaves the feedback); the abort message is also kept out of "Avoid These Patterns" (fixed in this PR, #1). *Why:* "Model work is bounded". (`loop-stages.ts:214,230-232`; `reflect.ts:
|
|
294
|
+
- **Refuses before calling the model when the file is not where a proposal would write.** A skill's `references/a.md` is indexed as `knowledge/skills/<name>/references/a`, so a proposal would be a `create` and accepting it adds a second file. The result is `unsupported_type` with subreason `file_outside_layout`, recorded as a skip. *Why:* #1052. (`reflect.ts:714-736,778-786`; CHANGELOG 0.9.27-alpha.1)
|
|
295
|
+
- **Reflect time budget.** Reflect gets `timeoutMs` equal to the improve run's remaining budget, and the shared abort signal. With no timeout set it uses `DEFAULT_LLM_TIMEOUT_MS` (600,000 ms). A call cut by the budget returns a failure and writes no ledger row, so the feedback still plans the ref next run (see A transient failure leaves the feedback); the abort message is also kept out of "Avoid These Patterns" (fixed in this PR, #1). *Why:* "Model work is bounded". (`loop-stages.ts:214,230-232`; `reflect.ts:507-513`; `src/integrations/agent/config.ts:9`)
|
|
298
296
|
|
|
299
297
|
### What reflect may change
|
|
300
298
|
|
|
301
|
-
- **Only `description`, `when_to_use` and a title. The body text is the source's own, except that a title adds a `# title` heading.** The engine returns `confidence` plus `frontmatterPatch {description, when_to_use, title}`, each a non-empty single-line string or `null`. akm applies the patch itself. When the patch carries a title and the body has no level-1 heading, akm prepends `# <title>` and strips every leading blank line of the body, so the body is not byte for byte unchanged; #1147 lists exactly this edit ("a title heading added to assets no one asked about") as harm. After the judge, the proposal mint also splices `type:` and `updated:` lines into the frontmatter when they are missing (next rules and Proposal outcome). *Why:* on 396 labelled reflect edits, frontmatter-only fixes were good 24 of 26 times, and edits that also rewrote the body were bad 175 of 224. (`reflect.ts:5-10,
|
|
302
|
-
- **A title becomes a `# title` heading only if the body has no level-1 heading.** Otherwise the title is ignored. The test is a multi-line regex that also matches a `# comment` line inside a fenced shell block, which suppresses the title. (`reflect.ts:
|
|
303
|
-
- **Only the changed keys' frontmatter lines are rewritten.** Every other line is kept, and an unchanged-equal field is dropped from the patch. The mint then adds `type:` and `updated:` lines (see Mint below). *Why:* a source whose YAML the parser reads only partly (a broken description beside a list) must lose nothing. (`reflect.ts:
|
|
304
|
-
- **A source whose frontmatter fence is opened but never closed gets no proposal.** It is reported as `no_change` ("empty diff"), recorded `unchanged` and counted by the plasticity counter; the person the comment says should fix it is never told. *Why:* it would leave the old block in the body under a new one, so a person fixes it. (`reflect.ts:
|
|
305
|
-
- **Missing required description is derived from the asset's own text.** If the source has frontmatter, its type requires a description (lesson, knowledge, memory, skill, command, agent, workflow or fact), and neither source nor patch has one, akm derives one from the title, first heading or first prose sentence. The candidate must pass `isValidDescription`; it is never free-form. That holds for the derived description only: a model patch on a source with no frontmatter gets a new block (`reflect.ts:
|
|
306
|
-
- **Allowed types.** Knowledge, memory, lesson, skill, agent, command and workflow are allowed. Another type is allowed only if its current content is already frontmatter + markdown. `secret` is always refused, and its content is never read or sent to an LLM. Refusals are `unsupported_type`, with the message "Use `akm proposal new` or edit the file directly". *Why:* the output is frontmatter + markdown, which would break a script or env file. (`reflect.ts:
|
|
307
|
-
- **A patch that changes nothing creates no proposal.** The `no_change` result applies when every field is `null` or equal to the source's. *Why:* reflect may answer "nothing to change"; the old prompt forbade it, and edits that fixed no defect were bad 86 of 93 times. (`reflect.ts:
|
|
308
|
-
- **Noise gate: noop and cosmetic edits are dropped before any judge call.** `noop` means identical up to trailing whitespace. `cosmetic` means identical after normalising frontmatter as parsed YAML (keys sorted) and prose as unwrapped text. Code fences and indented code compare verbatim. Anything uncertain is `substantive`. Subreasons are `reflect_skipped_noop` and `reflect_skipped_cosmetic`. *Why:* "Empty and cosmetic-only edits... each cost an LLM call and a review slot." It is conservative: "letting a cosmetic edit through costs a review, suppressing a real fix loses work" (#580). (`src/commands/improve/reflect-noise.ts:5-37,225-305`; `reflect.ts:
|
|
299
|
+
- **Only `description`, `when_to_use` and a title. The body text is the source's own, except that a title adds a `# title` heading.** The engine returns `confidence` plus `frontmatterPatch {description, when_to_use, title}`, each a non-empty single-line string or `null`. akm applies the patch itself. When the patch carries a title and the body has no level-1 heading, akm prepends `# <title>` and strips every leading blank line of the body, so the body is not byte for byte unchanged; #1147 lists exactly this edit ("a title heading added to assets no one asked about") as harm. After the judge, the proposal mint also splices `type:` and `updated:` lines into the frontmatter when they are missing (next rules and Proposal outcome). *Why:* on 396 labelled reflect edits, frontmatter-only fixes were good 24 of 26 times, and edits that also rewrote the body were bad 175 of 224. (`reflect.ts:5-10,229-235,237-259,313-322,428-446`; CHANGELOG 0.9.25-alpha.3)
|
|
300
|
+
- **A title becomes a `# title` heading only if the body has no level-1 heading.** Otherwise the title is ignored. The test is a multi-line regex that also matches a `# comment` line inside a fenced shell block, which suppresses the title. (`reflect.ts:257-259`)
|
|
301
|
+
- **Only the changed keys' frontmatter lines are rewritten.** Every other line is kept, and an unchanged-equal field is dropped from the patch. The mint then adds `type:` and `updated:` lines (see Mint below). *Why:* a source whose YAML the parser reads only partly (a broken description beside a list) must lose nothing. (`reflect.ts:243-245,274,289-304`)
|
|
302
|
+
- **A source whose frontmatter fence is opened but never closed gets no proposal.** It is reported as `no_change` ("empty diff"), recorded `unchanged` and counted by the plasticity counter; the person the comment says should fix it is never told. *Why:* it would leave the old block in the body under a new one, so a person fixes it. (`reflect.ts:253-255,1042-1059`)
|
|
303
|
+
- **Missing required description is derived from the asset's own text.** If the source has frontmatter, its type requires a description (lesson, knowledge, memory, skill, command, agent, workflow or fact), and neither source nor patch has one, akm derives one from the title, first heading or first prose sentence. The candidate must pass `isValidDescription`; it is never free-form. That holds for the derived description only: a model patch on a source with no frontmatter gets a new block (`reflect.ts:280-283`). *Why:* the promote-time validator (20-400 chars) rejected ~14 of 16 scraped-doc proposals (#636). Injecting a whole block or overwriting an authored one is out of scope. (`reflect.ts:196-218,261-273`; `src/core/authoring-rules.ts:32-37,71,90`; CHANGELOG ~7225)
|
|
304
|
+
- **Allowed types.** Knowledge, memory, lesson, skill, agent, command and workflow are allowed. Another type is allowed only if its current content is already frontmatter + markdown. `secret` is always refused, and its content is never read or sent to an LLM. Refusals are `unsupported_type`, with the message "Use `akm proposal new` or edit the file directly". *Why:* the output is frontmatter + markdown, which would break a script or env file. (`reflect.ts:158-173,744-777`)
|
|
305
|
+
- **A patch that changes nothing creates no proposal.** The `no_change` result applies when every field is `null` or equal to the source's. *Why:* reflect may answer "nothing to change"; the old prompt forbade it, and edits that fixed no defect were bad 86 of 93 times. (`reflect.ts:275,1046-1060`; CHANGELOG 0.9.25-alpha.3)
|
|
306
|
+
- **Noise gate: noop and cosmetic edits are dropped before any judge call.** `noop` means identical up to trailing whitespace. `cosmetic` means identical after normalising frontmatter as parsed YAML (keys sorted) and prose as unwrapped text. Code fences and indented code compare verbatim. Anything uncertain is `substantive`. Subreasons are `reflect_skipped_noop` and `reflect_skipped_cosmetic`. *Why:* "Empty and cosmetic-only edits... each cost an LLM call and a review slot." It is conservative: "letting a cosmetic edit through costs a review, suppressing a real fix loses work" (#580). (`src/commands/improve/reflect-noise.ts:5-37,225-305`; `reflect.ts:1020-1060`)
|
|
309
307
|
- **Prompt constraints on what to change.** The prompt says to change a field only if missing, broken, or claiming more than the body covers. Feedback is "a signal, not a fact to insert". Feedback about a task the asset never claims to cover needs no change. For stale or superseded notes, `when_to_use` names the version or date the body records. With no feedback the prompt says to fix only a missing or broken field. *Why:* models invented sections (incident dates, ports, disk layouts) when feedback asserted a claim (#952); most negative feedback is a retrieval miss, not a defect. (`src/assets/prompts/reflect-feedback-framing.md`; `src/integrations/agent/prompts.ts:333-362`; CHANGELOG 0.9.25-alpha.3, #952)
|
|
310
308
|
- **Prompt names the frontmatter problems akm can see.** These are a description split by a stray period, an escaped quote, no `when_to_use`, and no title; the repair keeps the description's wording, names, numbers and paths. *Why:* without the list the model fixed 51 of 122 broken descriptions and 93 of 193 missing titles; with it, 120 and 189. (`prompts.ts:243-289`; CHANGELOG 0.9.25-alpha.3)
|
|
311
|
-
- **Content cap in the prompt.** The asset is capped at 12,000 chars (`REFLECT_CONTENT_CAP`) with a truncation marker. A direct-LLM runner may use half its context window after prompt overhead, never below 12,000. *Why:* the flat cap keeps argv under OS ARG_MAX for opencode/claude (E2BIG). The marker is exported so validators can refuse a body that echoes it (#952). (`prompts.ts:94-111,383-398`; `reflect.ts:
|
|
309
|
+
- **Content cap in the prompt.** The asset is capped at 12,000 chars (`REFLECT_CONTENT_CAP`) with a truncation marker. A direct-LLM runner may use half its context window after prompt overhead, never below 12,000. *Why:* the flat cap keeps argv under OS ARG_MAX for opencode/claude (E2BIG). The marker is exported so validators can refuse a body that echoes it (#952). (`prompts.ts:94-111,383-398`; `reflect.ts:859-869`)
|
|
312
310
|
- **Reflexion context: the 3 newest content-based rejections for the ref are shown.** The prompt says not to propose the same change again, and to return `null` if no other change is justified. Each shows the reason and a 500-char preview. Procedural rejections (stale target, expiry, asset missing) are left out. *Why:* "proposals that have already been reviewed and refused". Procedural refusals "are not judgements on the content". (`src/commands/improve/stage.ts:209-236`; `src/commands/proposal/proposal-types.ts:346-357`; `prompts.ts:400-416`)
|
|
313
|
-
- **Recent feedback is shown, capped.** Up to 10 lines per target, or 20 across all assets without a ref. A line given on a different body hash is marked "(given on an earlier version of the text)". The lines are the last events of any age: the 30-day window and the acted-on filter apply only at planning, and unscoped runs prefix each line with the ref. *Why:* the feedback judged text the asset may no longer have. (`reflect.ts:
|
|
311
|
+
- **Recent feedback is shown, capped.** Up to 10 lines per target, or 20 across all assets without a ref. A line given on a different body hash is marked "(given on an earlier version of the text)". The lines are the last events of any age: the 30-day window and the acted-on filter apply only at planning, and unscoped runs prefix each line with the ref. *Why:* the feedback judged text the asset may no longer have. (`reflect.ts:124-156`)
|
|
314
312
|
- **Recent reflect errors become "Avoid These Patterns" (rolling window of 3).** The section is run-only guidance. Accept-time validation warns if the heading appears in the proposal. *Why:* the section is run diagnostics, not asset content. A person accepting can't edit the proposal (#963). (`preparation.ts:161-167`; `loop-stages.ts:213-226`; `src/core/content-safety.ts:8-19`; `proposal-quality-validators.ts:347-362`)
|
|
315
|
-
- **Lesson lint findings are supplied as schema hints.** (`reflect.ts:
|
|
316
|
-
- **Self-refine passes: default 1.** `maxRefineIters` is clamped to at least 1. Later passes critique the prior draft and stop early on an unchanged draft. The improve loop never sets it, so improve always runs 1 pass. *Why:* R-1 / #372 (Self-Refine). (`reflect.ts:114
|
|
313
|
+
- **Lesson lint findings are supplied as schema hints.** (`reflect.ts:175-179`)
|
|
314
|
+
- **Self-refine passes: default 1.** `maxRefineIters` is clamped to at least 1. Later passes critique the prior draft and stop early on an unchanged draft. The improve loop never sets it, so improve always runs 1 pass. *Why:* R-1 / #372 (Self-Refine). (`reflect.ts:113-114,950,962-1002`)
|
|
317
315
|
|
|
318
316
|
### Quality gate & judge
|
|
319
317
|
|
|
320
|
-
- **The judge is on by default; set `processes.reflect.qualityGate.enabled: false` to turn it off.** Each gate (reflect, distill) follows only its own switch. *Why:* 0.9.23 fixed a bug where reflect's gate ran if either gate was on. (`reflect.ts:
|
|
321
|
-
- **Judge engine resolution.** The order is: the gate's own `engine`/`model`/`timeoutMs`/`llm`, then the reflect runner if it is an LLM engine, then the `reflect_proposal_quality-judge` process engine. A gate naming settings that resolve to no engine throws before anything is generated, with no fallback. A gate that names nothing takes the reflect runner or the process engine. A judge whose credential is missing at run time throws for each planned ref, which the loop reports as `mode:"error"` with no ledger row and no `engine_unavailable` skip: plan resolution checks process runners only, never `qualityGate` engines. *Why:* #1011. "The gate never falls back to another judge." (`reflect.ts:
|
|
322
|
-
- **No judge available means the proposal is queued for review, not rejected.** The review reason is `no-judge-configured`, with a warning.
|
|
318
|
+
- **The judge is on by default; set `processes.reflect.qualityGate.enabled: false` to turn it off.** Each gate (reflect, distill) follows only its own switch. *Why:* 0.9.23 fixed a bug where reflect's gate ran if either gate was on. (`reflect.ts:1222`; `src/core/config/schema/improve-processes.ts:66-88`; CHANGELOG 0.9.23)
|
|
319
|
+
- **Judge engine resolution.** The order is: the gate's own `engine`/`model`/`timeoutMs`/`llm`, then the reflect runner if it is an LLM engine, then the `reflect_proposal_quality-judge` process engine. A gate naming settings that resolve to no engine throws before anything is generated, with no fallback. A gate that names nothing takes the reflect runner or the process engine. A judge whose credential is missing at run time throws for each planned ref, which the loop reports as `mode:"error"` with no ledger row and no `engine_unavailable` skip: plan resolution checks process runners only, never `qualityGate` engines. *Why:* #1011. "The gate never falls back to another judge." (`reflect.ts:1221-1235,1243-1244`; `stage.ts:311-345`; `improve-strategies.ts:434-454`; `loop-stages.ts:185-195`)
|
|
320
|
+
- **No judge available means the proposal is queued for review, not rejected.** The review reason is `no-judge-configured`, with a warning. (`reflect.ts:1236-1242,1118-1122`)
|
|
323
321
|
- **Three criteria on a 1-5 scale: NEED, PRESERVATION, QUALITY.**
|
|
324
322
|
- NEED: every changed field fixes a real problem. Score 1-2 if any changed field rewrites a sound field.
|
|
325
323
|
- PRESERVATION: the new description keeps every fact; all other frontmatter is unchanged. Score 1-3 if anything is dropped.
|
|
@@ -327,26 +325,21 @@ All paths are relative to the `rules-010` worktree. Citation form is `path:line`
|
|
|
327
325
|
|
|
328
326
|
*Why:* the rubric was tuned for frontmatter-only revisions on 113 reviewed proposals (rubric f07). The production model passed 78 of 95 good and 3 of 18 bad, against 53 and 7 under the earlier body-rewrite rubric. (`stage.ts:458-510,526`; CHANGELOG 0.9.25-alpha.4)
|
|
329
327
|
- **Pass band: every criterion must be at least 4.** Otherwise a mean of 2.5 or more is "review" and below 2.5 is "reject". A unanimous 4+ is required because judging on a mean (3.5) let rewrites that only reworded a correct asset through. *Why:* CHANGELOG 0.9.21. Note the following for reflect. (`stage.ts:582-592,643`)
|
|
330
|
-
- **A reflect non-pass with a mean of 3.5 or more goes to a person; a lower one is refused (changed in this PR, decision 4).** `judgeFailed` is true only for `reviewNeeded && score === -1`. `judgeUncertain` is `reviewNeeded` with a score at or above `REFLECT_REVIEW_MIN_SCORE` (3.5). A judge-uncertain revision is minted as a pending proposal, stamped `deferred`, gate `quality-gate`, reason `judge-uncertain`, with the judge's scores and reason attached, and records no `quality_rejected` window. A non-pass with a real score below 3.5 (the judge labels 2.5 to 4 as review) still goes to `refuse(...)`: no proposal, `quality_rejected`, 14-day hard window.
|
|
328
|
+
- **A reflect non-pass with a mean of 3.5 or more goes to a person; a lower one is refused (changed in this PR, decision 4).** `judgeFailed` is true only for `reviewNeeded && score === -1`. `judgeUncertain` is `reviewNeeded` with a score at or above `REFLECT_REVIEW_MIN_SCORE` (3.5). A judge-uncertain revision is minted as a pending proposal, stamped `deferred`, gate `quality-gate`, reason `judge-uncertain`, with the judge's scores and reason attached, and records no `quality_rejected` window. A non-pass with a real score below 3.5 (the judge labels 2.5 to 4 as review) still goes to `refuse(...)`: no proposal, `quality_rejected`, 14-day hard window. *Why:* on labelled judge runs (akm-eval judge-gate, lab 27B Q2) the shipped judge's non-passes with a mean of 3.5 or more were 17 good and 9 bad among 113 frontmatter-only proposals, while below 3.5, 396 labelled body rewrites were 6 good and 285 bad; so the band is worth a person's time and what is below it is not. (`reflect.ts:1004-1018,1091-1116,1118-1124`, `stage.ts:643`)
|
|
331
329
|
- **The lesson-only "reject if any criterion at or below 2" rule does not apply to reflect.** It triggers only when a `grounding` criterion is present. (`stage.ts:522-523,638-642`)
|
|
332
|
-
- **A judge failure routes to review rather than rejection.** A timeout, error or unparseable reply gives `score: -1` and `reviewNeeded: true`. Reflect creates the proposal pending, with a gate decision `deferred`, reason `judge-error`, gate `quality-gate`.
|
|
330
|
+
- **A judge failure routes to review rather than rejection.** A timeout, error or unparseable reply gives `score: -1` and `reviewNeeded: true`. Reflect creates the proposal pending, with a gate decision `deferred`, reason `judge-error`, gate `quality-gate`. *Why:* reflect used to record `quality_rejected`, blocking the asset for 14 days over a judge outage. (`stage.ts:631-635`; `reflect.ts:1102,1118-1148`; CHANGELOG 0.9.24)
|
|
333
331
|
- **The judge runs at temperature 0 with thinking off.** The judge only thinks if its own engine sets `enableThinking: true`. *Why:* 0.9.17 pinned temperature 0 because at the previous effective default of 0.3, 10 of 16 identical inputs had flipped verdict (CHANGELOG 0.9.17). A thinking judge passed more good and more bad edits, took ~20x the tokens, and 30-67 s median against ~5 s. (`stage.ts:589-592,619-623`; `docs/reference/configuration.md:123-125,545-552`; CHANGELOG 0.9.23)
|
|
334
332
|
- **The judge response schema is strict.** The shape is `{scores:{need,preservation,quality}, reason}`, each score an integer 1-5, all required. A missing or out-of-range criterion is a parse failure. A reply the stage can't parse gets one corrective retry, and a missing or out-of-range criterion makes `parseJudgeResponse` return `undefined`, which is exactly what `callStage` retries; only extra keys pass without a retry. (`stage.ts:534-580`; `stage.ts:123-131`; CHANGELOG 0.9.25-alpha.2)
|
|
335
|
-
- **Rubric text depends on what drove the revision.** If any feedback line starts `[negative]`, the rubric says the revision cannot change the body so need not resolve the feedback, and judges only the changed fields. It justifies a change only if the field claims more than the body covers, or the note is stale (then a `when_to_use` must name the version or date). Otherwise it is the "maintenance" paragraph: only a missing or broken field needs a change, and rewording a sound field is churn. The switch reads feedback of any age (see Recent feedback is shown), so a proactive pick whose history holds an old or already-fixed negative is judged under the permissive "negative" paragraph, and unscoped lines carry a ref prefix so `[negative]` never matches there. *Why:* most negative feedback is a retrieval miss. (`stage.ts:463-468,480`; `reflect.ts:
|
|
333
|
+
- **Rubric text depends on what drove the revision.** If any feedback line starts `[negative]`, the rubric says the revision cannot change the body so need not resolve the feedback, and judges only the changed fields. It justifies a change only if the field claims more than the body covers, or the note is stale (then a `when_to_use` must name the version or date). Otherwise it is the "maintenance" paragraph: only a missing or broken field needs a change, and rewording a sound field is churn. The switch reads feedback of any age (see Recent feedback is shown), so a proactive pick whose history holds an old or already-fixed negative is judged under the permissive "negative" paragraph, and unscoped lines carry a ref prefix so `[negative]` never matches there. *Why:* most negative feedback is a retrieval miss. (`stage.ts:463-468,480`; `reflect.ts:139-152`)
|
|
336
334
|
- **A judge on an agent engine gets a read-only verification paragraph.** It may read the asset, or one the changed region names, via `akm_show`. The purpose is to verify a fact the revision adds or alters. At most one or two reads, and never search. Each added statement must be found in the asset or feedback, and each source fact must be in the revision. *Why:* with the plain judge on qwen3.8-27b, the agent judge passed 38 of 40 good edits and 14 of 60 bad, against 34 and 12. A thinking engine is needed, since without thinking the model looped on tool calls. The plain judge's prompt is unchanged. (`stage.ts:448-456,507,669`; CHANGELOG 0.9.25-alpha.2)
|
|
337
|
-
- **The judge sees the patched content, not the persisted bytes.** It receives the source, the full candidate (the `applyReflectPatch` output), a "changed region" diff, and the feedback (truncated to 1,000 chars). `createProposal` then passes the `.md` payload through `ensureAkmMarkdownType`, which splices `type:` and `updated: <date>` into the frontmatter when missing (or adds a whole block), and the `staged` hash is computed on that post-stamp proposal, so the drain accepts bytes the judge never saw (#1147 names "`type` and provenance stamps"). Each document is bounded to 6,000 chars (2,960 head and 2,960 tail around a middle-omitted marker), narrower than the writer's 12,000 or more: a description grounded only in the omitted middle of an asset over 6,000 chars can only score "not supported" on QUALITY and is `quality_rejected` for 14 days; only an agent-engine judge can read the asset. (`stage.ts:281,424-446,471-510`; `reflect.ts:
|
|
335
|
+
- **The judge sees the patched content, not the persisted bytes.** It receives the source, the full candidate (the `applyReflectPatch` output), a "changed region" diff, and the feedback (truncated to 1,000 chars). `createProposal` then passes the `.md` payload through `ensureAkmMarkdownType`, which splices `type:` and `updated: <date>` into the frontmatter when missing (or adds a whole block), and the `staged` hash is computed on that post-stamp proposal, so the drain accepts bytes the judge never saw (#1147 names "`type` and provenance stamps"). Each document is bounded to 6,000 chars (2,960 head and 2,960 tail around a middle-omitted marker), narrower than the writer's 12,000 or more: a description grounded only in the omitted middle of an asset over 6,000 chars can only score "not supported" on QUALITY and is `quality_rejected` for 14 days; only an agent-engine judge can read the asset. (`stage.ts:281,424-446,471-510`; `reflect.ts:1025-1030,1044-1093`; `repository.ts:431-433`; `src/core/asset/akm-markdown.ts:39-77`; `prompts.ts:104`)
|
|
338
336
|
- **A judge pass stamps the proposal `staged`.** The gate decision is `outcome: "staged"`, `reason: "quality-judge"`, `gate: "quality-gate"`. It carries the judged content hash, per-criterion scores and the judge reason. The drain accepts it only while the content hash still matches. (`stage.ts:241-292`; `src/commands/proposal/drain.ts:597-603`)
|
|
339
337
|
- **`akm improve judge` runs the reflect judge on one stdin revision.** It reads `{source, candidate, feedback, ref}` and writes nothing. It uses the reflect gate's engine, for testing a judge engine. (`src/commands/improve/improve-cli.ts:255-301`)
|
|
340
338
|
|
|
341
|
-
###
|
|
339
|
+
### After the judge
|
|
342
340
|
|
|
343
|
-
- **
|
|
344
|
-
- **
|
|
345
|
-
- **No queries means it passes unjudged.** With 0 queries, the gate returns `pass: true`. Also, if state.db can't be read, the query list is empty and the gate passes. *Why:* "no retrieval queries to compare on". (`retrieval-gate.ts:61-67,129`)
|
|
346
|
-
- **Ungraded checks go to a person (changed in this PR, decision 4).** If a grade cannot be obtained for the old or new content (transport error, timeout or unparseable reply on any of up to ten grading calls), the gate returns `pass:false` with `ungraded: true`, and reflect mints the revision as a pending proposal stamped `deferred`, gate `quality-gate`, reason `retrieval-ungraded`, with the check's words as the judge reason, and records no ledger window. A measured regression is still refused as `quality_rejected` (14-day hard window), and so is a budget-watchdog abort mid-gate, which is not `ungraded`. *Why:* a check that could not grade measured nothing, so it is no regression; the gate used to fail closed, like the quality judge did until 0.9.24 (CHANGELOG 0.9.24). (`retrieval-gate.ts:104-117,150-157`; `reflect.ts:1119-1135,1148-1157,1193`; `improve-ledger-repository.ts:136-140`; `CHANGELOG.md:1489-1500`)
|
|
347
|
-
- **Grading input and settings.** The judge sees type, ref, name, description and the first 1,500 chars of the body, using the retrieval-eval judge prompt. Settings are temperature 0, thinking off and a strict schema. *Why:* same shape as the retrieval eval judge (kappa 0.83 vs human grades). (`retrieval-gate.ts:33-36,69-89,132-149`)
|
|
348
|
-
- **Refusal records `retrievalRegression` metadata.** It stores query count and the before/after means, writes a `quality_rejected` ledger row (14-day hard window), and returns `quality_rejected`. (`reflect.ts:1134-1145`)
|
|
349
|
-
- **No other continuity check exists for reflect.** Accept-time freshness checks the target's `beforeHash`: a target changed after mint is auto-rejected `stale-target`. (`reflect.ts:1119-1145`; `drain.ts:142-211`)
|
|
341
|
+
- **Nothing follows the judge (changed in this PR, decision 9).** A revision the judge passes is minted as a proposal at once: stamped `staged` when the judge ran, unstamped when it is off. Reflect no longer grades the old and the new text against the asset's own retrieval queries, so there is no `retrieval-ungraded` review reason, no `retrievalRegression` event metadata and no ledger row from such a check. The check used to refuse a revision whose mean grade dropped at all (3.00 to 2.60 was enough) with a 14-day `quality_rejected` window. *Why:* re-measured on 0.10 (akm-eval `evals/reflect` with retrieval queries and the judge on, `--repeat 3`), it refused 12% (35B, 11 of 90 public; 6 of 89 private) and 17% (27B Q4, 17 of 101 public; 11 of 96 private) of the revisions the judge had passed, mostly correct fixes, and lowered defects fixed from 84 to 77 (35B) and from 92 to 77 (27B) of 96. Reflect now edits frontmatter only, so an edit has little room to hurt retrieval. ([itlackey/akm#1171](https://github.com/itlackey/akm/issues/1171)) (`reflect.ts:1118-1148`)
|
|
342
|
+
- **Reflect has no accept-time continuity check of its own.** Accept-time freshness checks the target's `beforeHash`: a target changed after mint is auto-rejected `stale-target`. (`drain.ts:142-211`)
|
|
350
343
|
|
|
351
344
|
### Validation
|
|
352
345
|
|
|
@@ -355,7 +348,7 @@ All paths are relative to the `rules-010` worktree. Citation form is `path:line`
|
|
|
355
348
|
- `meta_commentary_added`
|
|
356
349
|
- `frontmatter_copied_into_body`
|
|
357
350
|
|
|
358
|
-
*Why:* on 396 labelled edits (83 good, 313 bad) the default lists hit 22 bad and 0 good ones, "so a hit is refused unjudged". (`reflect-noise.ts:39-134`; `reflect.ts:
|
|
351
|
+
*Why:* on 396 labelled edits (83 good, 313 bad) the default lists hit 22 bad and 0 good ones, "so a hit is refused unjudged". (`reflect-noise.ts:39-134`; `reflect.ts:1086-1088`)
|
|
359
352
|
- **Default placeholder phrases.** `please confirm`, `please verify`, `to be confirmed`, `to be determined`, `to be verified`. `TODO`, `TBD` and `FIXME` are deliberately not on the list. *Why:* "an asset may carry those on purpose, and the owner does not want them refused". (`reflect-noise.ts:55-62`)
|
|
360
353
|
- **Default meta-commentary phrases.** 23 phrases: `feedback signal(s)`, `feedback indicate(s)`, `feedback suggest(s)`, `feedback ask(s)`, `feedback says`, `feedback report(s)`, `feedback request(s)`, `this revision`, `the source asset`, `the source note`, `the source memory`, `the original asset`, `the original note`, `the original memory`, `the original version of this`, `quality gate rejected`, `proposal rejected`. *Why:* an asset talking about its own edit is a defect no judge needs to weigh. (`reflect-noise.ts:64-88`)
|
|
361
354
|
- **Frontmatter-copied-into-body.**
|
|
@@ -364,11 +357,11 @@ All paths are relative to the `rules-010` worktree. Citation form is `path:line`
|
|
|
364
357
|
- **Override.** `processes.reflect.defectFilter.{placeholders,metaCommentary,frontmatterKeys}` replaces a default list, and `[]` turns the rule off.
|
|
365
358
|
|
|
366
359
|
(`reflect-noise.ts:90-111,136-176`; `src/core/config/schema/improve-processes.ts:101-114`)
|
|
367
|
-
- **Generated content safety.** If the patch text contains the `[REDACTED]` marker, or equals a configured credential in the dispatch's sensitive values, it is rejected as `parse_error`. Nothing is persisted. *Why:* "refusing to persist already-redacted text" or an echoed credential (#962). (`reflect.ts:
|
|
368
|
-
- **Output contract.** The reply must be exactly `{confidence, frontmatterPatch}` (plus `ref` when unscoped). `confidence` is in [0,1]. `frontmatterPatch` has exactly three keys, each a non-empty single-line string or `null`. A fallback "framed" format applies when the endpoint rejects JSON Schema: `AKM_REFLECT_CONFIDENCE`/`AKM_REFLECT_REF`/`AKM_REFLECT_FRONTMATTER_PATCH` lines. *Why:* all engine kinds are asked for the same reply. Framing is for endpoints that reject JSON Schema. (`reflect.ts:
|
|
369
|
-
- **One repair turn, shared across passes.** An unparseable reply gets exactly one repair turn within the original deadline (`repairAttempts` budget is shared across refine passes); a second failure returns `parse_error`. *Why:* "within the original deadline, then fails". A stage retries only a reply it cannot read, not one that merely misses the schema. (`reflect.ts:
|
|
370
|
-
- **`enableThinking: false` is forced on every reflect dispatch.** *Why:* code comment: "Visible chain-of-thought can exhaust the output before the envelope." Docs add: improve's reflect always requests `enableThinking: false` "for a machine-readable payload". (`reflect.ts:556
|
|
371
|
-
- **Runs under the model-work tool policy.** Credentials are checked, and a transport that can't confine the policy fails, before any work. An agent can edit only its scratch directory, so it returns the proposal as its reply. (`reflect.ts:
|
|
360
|
+
- **Generated content safety.** If the patch text contains the `[REDACTED]` marker, or equals a configured credential in the dispatch's sensitive values, it is rejected as `parse_error`. Nothing is persisted. *Why:* "refusing to persist already-redacted text" or an echoed credential (#962). (`reflect.ts:1290-1295`; `src/core/content-safety.ts:21-34`)
|
|
361
|
+
- **Output contract.** The reply must be exactly `{confidence, frontmatterPatch}` (plus `ref` when unscoped). `confidence` is in [0,1]. `frontmatterPatch` has exactly three keys, each a non-empty single-line string or `null`. A fallback "framed" format applies when the endpoint rejects JSON Schema: `AKM_REFLECT_CONFIDENCE`/`AKM_REFLECT_REF`/`AKM_REFLECT_FRONTMATTER_PATCH` lines. *Why:* all engine kinds are asked for the same reply. Framing is for endpoints that reject JSON Schema. (`reflect.ts:313-351,357,448-494,909-910`)
|
|
362
|
+
- **One repair turn, shared across passes.** An unparseable reply gets exactly one repair turn within the original deadline (`repairAttempts` budget is shared across refine passes); a second failure returns `parse_error`. *Why:* "within the original deadline, then fails". A stage retries only a reply it cannot read, not one that merely misses the schema. (`reflect.ts:578-602,983`; CHANGELOG 0.9.25-alpha.2)
|
|
363
|
+
- **`enableThinking: false` is forced on every reflect dispatch.** *Why:* code comment: "Visible chain-of-thought can exhaust the output before the envelope." Docs add: improve's reflect always requests `enableThinking: false` "for a machine-readable payload". (`reflect.ts:555-556`; `docs/reference/configuration.md:121-125`)
|
|
364
|
+
- **Runs under the model-work tool policy.** Credentials are checked, and a transport that can't confine the policy fails, before any work. An agent can edit only its scratch directory, so it returns the proposal as its reply. (`reflect.ts:844-857,933-940`)
|
|
372
365
|
- **Mint-time check is structural only.** Lesson, task and workflow proposals must parse as their type, else `invalid_canonical_structure`. Quality validators run at accept. *Why:* running the blocking durable-content validators at mint would throw instead of minting for a person to review. (`src/commands/proposal/repository.ts:439-463`; `src/commands/proposal/validators/proposal-validators.ts:149-172`)
|
|
373
366
|
- **Accept-time blocking validators.** These apply to reflect proposals.
|
|
374
367
|
- **Truncation marker.** A body still containing `... [truncated — focus on the visible portion]` is blocked.
|
|
@@ -389,22 +382,21 @@ All paths are relative to the `rules-010` worktree. Citation form is `path:line`
|
|
|
389
382
|
|
|
390
383
|
### Proposal outcome
|
|
391
384
|
|
|
392
|
-
- **Mint with source `reflect`, a `sourceRun` of `reflect-<ms>`, and the model's self-reported confidence.** The proposal is stored with a FileChange envelope, before-hashes of the target, and eligibility source. Self-reported confidence is stored for reviewers and the triage judge. A `proposed` ledger row (7-day soft window) is written on mint. The mint rewrites the `.md` payload with `ensureAkmMarkdownType` for every source, not only reflect: a missing `type:` and a missing `updated:` are each added as one line before the closing `---`, or a new block is added when there is no frontmatter. (`reflect.ts:
|
|
385
|
+
- **Mint with source `reflect`, a `sourceRun` of `reflect-<ms>`, and the model's self-reported confidence.** The proposal is stored with a FileChange envelope, before-hashes of the target, and eligibility source. Self-reported confidence is stored for reviewers and the triage judge. A `proposed` ledger row (7-day soft window) is written on mint. The mint rewrites the `.md` payload with `ensureAkmMarkdownType` for every source, not only reflect: a missing `type:` and a missing `updated:` are each added as one line before the closing `---`, or a new block is added when there is no frontmatter. (`reflect.ts:1124-1148`; `repository.ts:350-510,431-433`; `reflect.ts:324-330`; `src/core/asset/akm-markdown.ts:39-77`)
|
|
393
386
|
- **Outcome by status.**
|
|
394
387
|
|
|
395
388
|
| Case | Result |
|
|
396
389
|
|---|---|
|
|
397
|
-
| Judge passed
|
|
398
|
-
| Judge passed, retrieval check could not grade (changed in this PR, decision 4) | Pending, `deferred`, gate `quality-gate`, reason `retrieval-ungraded`. Left for a person. |
|
|
390
|
+
| Judge passed | Pending, stamped `staged` (gate `quality-gate`, reason `quality-judge`, with content hash, scores and reason). The triage drain accepts it while the content hash still matches. |
|
|
399
391
|
| Judge failed (timeout/error/unparseable) | Pending, `deferred`, gate `quality-gate`, reason `judge-error`. Left for a person. |
|
|
400
392
|
| Judge scored below pass with a mean of 3.5 or more (changed in this PR, decision 4) | Pending, `deferred`, gate `quality-gate`, reason `judge-uncertain`, with scores and reason. Left for a person. |
|
|
401
393
|
| No judge configured | Pending, `deferred`, gate `reflect`, reason `no-judge-configured`. |
|
|
402
394
|
| Judge disabled (`qualityGate.enabled: false`) | Pending, unstamped; the drain decides. |
|
|
403
|
-
| Judge scored below pass (mean under 3.5), defect hit
|
|
395
|
+
| Judge scored below pass (mean under 3.5), defect hit | No proposal, `quality_rejected`, 14-day hard ledger window. |
|
|
404
396
|
| Noop or cosmetic | No proposal, `no_change`, 7-day soft window (`unchanged`). |
|
|
405
397
|
| Unsupported type or file outside layout | Skipped, `unsupported_type`, 7-day soft window (`unchanged`). |
|
|
406
398
|
|
|
407
|
-
Lines for these: `reflect.ts:1148
|
|
399
|
+
Lines for these: `reflect.ts:1118-1148,1063-1085`; `loop-stages.ts:241-268`. With the gate off, `staged` is never stamped (`reflect.ts:1137-1147`). *Why:* the review/queue semantics are the gate decision on the proposal.
|
|
408
400
|
- **Nothing prevents a second pending reflect proposal for the same ref.** `createProposal` leaves "may this ref be proposed again" to the stage's candidate selection, which reads only the ledger; the `proposed` row is a 7-day soft window that newer negative feedback lifts, and the proactive lane re-admits an asset after `dueDays` (30). So an unreviewed asset is re-reflected at about day 30 and day 60 before the first proposal expires at day 90 (Reflexion context shows rejected proposals only), and an explicit `--scope <ref>` always mints another. If one sibling is accepted, the others are auto-rejected `stale-target` and the ledger records `failed`. Only retire proposals have a one-pending-per-asset rule. *Why:* not stated. (`repository.ts:344-349`, `preparation.ts:858-863`, `proactive-maintenance.ts:67-86`, `stage.ts:218-229`, `pair-pass.ts:948-953`)
|
|
409
401
|
- **Auto-accept is the triage drain, with deterministic gates.** The `default` strategy runs triage `applyMode: "promote"`, `maxAcceptsPerRun: 200`, judgment off. Accepts are staged proposals (still matching hash), capped by `maxAccepts`. Empty diffs are rejected. Any `deferred` decision from another gate is left for a person. *Why:* only the judged drain caused harm (#1132; #1147 later found the deterministic drain accepting harmful proactive edits, hence the proactive rule below); a lower cap of 25 let the review queue grow nightly (200 default since 0.10.26101002). (`drain.ts:5-28,551-590`; `src/assets/improve-strategies/default.json:15`; CHANGELOG 0.10.26101001/2)
|
|
410
402
|
- **Proactive-lane proposals are never auto-accepted.** The drain defers any proposal with `eligibilitySource === "proactive"` as `proactive-needs-review`. This holds through a staged accept and the judgment tier. Feedback-driven (`signal-delta`, `scope`) proposals are unchanged. *Why:* with the lane on, reflect edits on assets no one had asked about were accepted unattended after the judge passed them (`when_to_use`, `type` and provenance stamps, a title heading added) (#1147). (`drain.ts:13-16,593-596`; CHANGELOG 0.10.26101001 "A proposal from the proactive lane waits for a person")
|
|
@@ -414,14 +406,14 @@ All paths are relative to the `rules-010` worktree. Citation form is `path:line`
|
|
|
414
406
|
- **Human review rejection feeds back as Reflexion context.** A person's `akm proposal reject` writes a `rejected` ledger row (14-day hard window for reflect) and the rejection reason appears as "Previously Rejected Proposals" in later prompts. The 14-day window is partly redundant with that Reflexion context and also blocks legitimately different new feedback (see Soft versus hard ledger windows). (`improve-ledger-repository.ts:74-77,149-152`; `stage.ts:209-236`)
|
|
415
407
|
- **Failure routing and classification.** `unsupported_type` and `no_change` are deterministic skips. They are tallied as `reflect-skipped`, recorded in the ledger as `unchanged` (7-day soft window), and do not feed "Avoid These Patterns". Real failures are `reflect-failed`, write no ledger row, and their error message (except a budget abort) joins the rolling window of 3 shown to later reflect prompts. A `quality_rejected` records itself, and its judge text is not fed back. `content_policy_reject` maps to `reflect-guard-rejected` and was the old size-rail bucket; nothing in the 0.10 reflect path emits it any more. Only a guard rejection records `failed`, so a transient failure never consumes the signal that planned the ref (fixed in this PR, #1). *Why:* "A refused type or an unchanged asset is a deterministic skip, not an LLM fault"; "the judge's text is no lesson for the next prompt". (`loop-stages.ts:240-266`; `src/integrations/agent/spawn.ts:30-60`)
|
|
416
408
|
- **Plasticity counter.** Each `no_change` result increments a per-asset consecutive-no-op count; a successful proposal resets it. Three consecutive no-ops halve the asset's selection rank. (`loop-stages.ts:149-160,277-278`; `salience.ts:129-134`)
|
|
417
|
-
- **Event trail.** Every invocation closes with one `reflect_completed` event, with `reflect_invoked` once credentials validate. Failures carry the reason and subreason (`unsupported_type`, `no_change`, `parse_error`, `agent_crash`, `spawn_failed`; also `reflect_skipped_noop`/`_cosmetic`, `enoent` and `file_outside_layout`). A `quality_rejected` `reflect_completed` event instead carries `qualityRejected: true` plus the score, criteria
|
|
409
|
+
- **Event trail.** Every invocation closes with one `reflect_completed` event, with `reflect_invoked` once credentials validate. Failures carry the reason and subreason (`unsupported_type`, `no_change`, `parse_error`, `agent_crash`, `spawn_failed`; also `reflect_skipped_noop`/`_cosmetic`, `enoent` and `file_outside_layout`). A `quality_rejected` `reflect_completed` event instead carries `qualityRejected: true` plus the score, criteria or defect metadata, and no `ok`, `reason` or `subreason`. The loop also emits `improve_reflect_outcome`. A judge-uncertain proposal's event carries `qualityReason` and `qualityScore` (changed in this PR, decision 4). (`reflect.ts:12-14,638-668,1149-1162,1257-1282`; `loop-stages.ts:270-279`)
|
|
418
410
|
|
|
419
411
|
### Code and doc mismatches
|
|
420
412
|
- `default.json` and `docs/guides/improve-the-library.md:106` describe reflect as "rewrites only from negative feedback". That is true of the signal-delta lane, but the proactive lane also plans reflect on assets with no feedback (then the "maintenance" judge paragraph applies). (`default.json:2`)
|
|
421
413
|
- `salience.ts:129-134` documents the no-op dampener as "Consolidation selection only", but `preparation.ts:956-969` applies it when ranking the reflect/distill loop.
|
|
422
414
|
- The `content_policy_reject` / `reflect-guard-rejected` classification still exists, but the reflect size rail it was named for was removed in 0.9.25-alpha.3.
|
|
423
|
-
- Reflect's judge labels a mean in [2.5, 4) as "review" (`runQualityJudge`), but reflect sends only a mean of 3.5 or more to a person and refuses 2.5 to 3.5; the cut is `REFLECT_REVIEW_MIN_SCORE`, not the judge's label (changed in this PR, decision 4). (`stage.ts:643`, `reflect.ts:
|
|
424
|
-
- The code comment at `reflect.ts:
|
|
415
|
+
- Reflect's judge labels a mean in [2.5, 4) as "review" (`runQualityJudge`), but reflect sends only a mean of 3.5 or more to a person and refuses 2.5 to 3.5; the cut is `REFLECT_REVIEW_MIN_SCORE`, not the judge's label (changed in this PR, decision 4). (`stage.ts:643`, `reflect.ts:1018,1104-1116`)
|
|
416
|
+
- The code comment at `reflect.ts:239-241` and the judge intro (`stage.ts:464`) say the body is kept byte for byte / unchanged apart from an added title heading; neither mentions that every leading blank line of the body is stripped when the heading is prepended (`reflect.ts:258-259`).
|
|
425
417
|
|
|
426
418
|
## Distill
|
|
427
419
|
|
|
@@ -470,11 +462,11 @@ All paths are relative to the `rules-010` worktree. Citation form is `path:line`
|
|
|
470
462
|
|
|
471
463
|
- **Heuristic quality findings go to a human** — If lesson quality findings remain (invalid description or `when_to_use`, identical description and `when_to_use`, pseudo-frontmatter in the body), the content is minted as a `review_needed` proposal at score 2.0. The gate decision is `deferred`/`quality-gate`, reason `quality-review`. *Why:* "heuristic quality findings go to a human, not the bin". (`src/commands/improve/distill.ts:688-698,991-1075`; `src/commands/proposal/validators/proposal-quality-validators.ts:228-267`)
|
|
472
464
|
|
|
473
|
-
- **Quality gate (judge) is on by default** — `processes.distill.qualityGate.enabled` defaults to true. The gate can name its own judge engine, model, timeout or llm (#1011). A gate that names judge settings resolving to no engine throws `ConfigError` before generation (the loop reports `mode:"error"` with no ledger row), and a named judge never falls back to another judge. A gate that names nothing falls back to `defaults.llmEngine`. A judge absent at judge time (the promotion path) returns `pass:false, score:-1, reviewNeeded:true` (changed in this PR, decision 4), so `rejectDistilled` records `review_needed` and queues the lesson as a pending proposal (gate `quality-gate`, reason `quality-review`) for a person, as reflect does with `no-judge-configured`; it no longer records a 30-day `quality_rejected`. Plan resolution never pre-checks a `qualityGate` engine. A configuration gap is not a verdict on the memory. *Why:* #1011. (`src/commands/improve/distill.ts:702-704,747-750,1008-1043`; `src/commands/improve/stage.ts:311-345,593-613`; `reflect.ts:
|
|
465
|
+
- **Quality gate (judge) is on by default** — `processes.distill.qualityGate.enabled` defaults to true. The gate can name its own judge engine, model, timeout or llm (#1011). A gate that names judge settings resolving to no engine throws `ConfigError` before generation (the loop reports `mode:"error"` with no ledger row), and a named judge never falls back to another judge. A gate that names nothing falls back to `defaults.llmEngine`. A judge absent at judge time (the promotion path) returns `pass:false, score:-1, reviewNeeded:true` (changed in this PR, decision 4), so `rejectDistilled` records `review_needed` and queues the lesson as a pending proposal (gate `quality-gate`, reason `quality-review`) for a person, as reflect does with `no-judge-configured`; it no longer records a 30-day `quality_rejected`. Plan resolution never pre-checks a `qualityGate` engine. A configuration gap is not a verdict on the memory. *Why:* #1011. (`src/commands/improve/distill.ts:702-704,747-750,1008-1043`; `src/commands/improve/stage.ts:311-345,593-613`; `reflect.ts:1137-1147,1236-1242`; `improve-strategies.ts:434-454`)
|
|
474
466
|
|
|
475
467
|
- **Judge rubric (reusable / nonRedundancy / grounding)** — Three 1-5 integer scores. REUSABLE: a rule with its reason, not a record of what was done. NON-REDUNDANCY: compared only to the related assets listed (it scores 5 if none are listed). GROUNDING: every cause, step, number and limit appears in the source or its feedback. The judge sees the same 3000-char source body the writer saw, the 1500-char feedback lines, and the 2000-char proposed lesson. `temperature: 0`. Thinking is off unless the judge's own engine enables it. *Why:* the old "information not already present in the source" criterion let invented claims score as novel. (`src/commands/improve/stage.ts:364-422,593-645`)
|
|
476
468
|
|
|
477
|
-
- **Judge routing: any criterion at 2 or below rejects** — `grounding` is scored but excluded from the mean. A pass needs every criterion in the mean at 4 or above. Otherwise a mean of at least 2.5 is `review_needed`, and a lower mean is rejected. Any lesson criterion at 2 or below (`LESSON_REJECT_MAX_SCORE = 2`, grounding included) is `quality_rejected` whatever the mean. A judge timeout, parse failure or missing judge engine routes to review (`score -1`; the missing engine changed in this PR, decision 4). Grounding is excluded from both the mean and the `lowest >= 4` pass test, so reusable 5, nonRedundancy 5 and grounding 3 ("one stretches what the source says") passes outright; only grounding 2 or less vetoes. The only recorded reason for excluding it (CHANGELOG 0.9.19: the judge was not shown the feedback) no longer holds, because the judge has been shown the feedback since 0.9.27-alpha.2 (`stage.ts:390-397`)
|
|
469
|
+
- **Judge routing: any criterion at 2 or below rejects** — `grounding` is scored but excluded from the mean. A pass needs every criterion in the mean at 4 or above. Otherwise a mean of at least 2.5 is `review_needed`, and a lower mean is rejected. Any lesson criterion at 2 or below (`LESSON_REJECT_MAX_SCORE = 2`, grounding included) is `quality_rejected` whatever the mean. A judge timeout, parse failure or missing judge engine routes to review (`score -1`; the missing engine changed in this PR, decision 4). Grounding is excluded from both the mean and the `lowest >= 4` pass test, so reusable 5, nonRedundancy 5 and grounding 3 ("one stretches what the source says") passes outright; only grounding 2 or less vetoes. The only recorded reason for excluding it (CHANGELOG 0.9.19: the judge was not shown the feedback) no longer holds, because the judge has been shown the feedback since 0.9.27-alpha.2 (`stage.ts:390-397`); adding grounding to the pass bar was measured and leaves the outcome unchanged, so it stays out (Policy decision 6). *Why:* "a mean hid a decisive score (4 and 1 average 2.5)"; 17 of 19 lessons that were not rejected were bad on 2026-10-05. (`src/commands/improve/stage.ts:512-523,552-557,583-643`)
|
|
478
470
|
|
|
479
471
|
- **A passing lesson goes to a person, never the drain (`distill-review`)** — A judged pass is minted `deferred` with gate `quality-gate`, reason `distill-review`, and carries the criterion scores and the judge reason. Judge confidence is `score/5`. With the gate off nothing is judged and the proposal is minted unstamped for the drain to decide. *Why:* staged precision was 2 of 12 on 2026-10-05; no judge score separated the two good lessons from the ten bad ones. (`src/commands/improve/distill.ts:769-796`; `src/commands/improve/stage.ts:240-260`)
|
|
480
472
|
|
|
@@ -633,6 +625,7 @@ Paths are relative to the repository root of this branch. `consolidate.ts` is `s
|
|
|
633
625
|
- **Knowledge is never replaced with a memory** — If the retired side would be a knowledge note and the successor a memory (a `.derived` memory counts as a memory), the pair is flipped to retire the memory instead, but only if the judge lists no claim held only by the memory. Otherwise nothing is minted. The flipped pair then runs every guard like any retirement. *Why:* owner rule 2026-10-08: knowledge is the reviewed form and a memory is raw capture, often the memory the note was promoted from. 21 such retirements had been accepted since 2026-10-01. (`pair-pass.ts:757-763`; #1092)
|
|
634
626
|
- **Retire guards** — Never retire a `.derived` memory whose parent still exists. Within a run, an asset used as a retired side or successor cannot be retired or reused again (same-run chain guard). The "never retire a `.derived` whose parent exists" reasoning is not applied to successors: `foo` can be retired in favour of `bar.derived`, and `bar` later retired against `X`, so `foo`'s claims leave the live tree with no judge having compared `bar.derived` to anything. A retire-worthy verdict dropped by the chain guard counts as failed, so its initiator gets no ledger row and is retried. A pair whose refs and both body hashes match a rejected or reverted retire proposal is skipped before the judge call, except one rejected only by expiry or as a stale target. *Why:* night 1 judged 177 duplicate verdicts into only 59 proposals, so silently abandoning the dropped ones was a bug. A reverted retirement is the same "no, not this" signal as a rejection. (`pair-pass.ts:388-411,713-726,765-789`)
|
|
635
627
|
- **Continuity-risk check (forgetting safety)** — Before minting, up to 5 of the retired asset's most recent cleaned past queries are replayed through the real search with limit 10. For every query where the retired asset ranked in the top 10, the successor must also rank in the top 10. Failures, and unverified queries (the search threw, or fell back to keyword-only, including every later query once an earlier one fell back), set `continuityRisk` on the proposal. It still mints, but is excluded from all bulk accept and from drain staging. No recorded queries, or normalized-identical bodies, means no check and no risk. A flagged proposal has no recovery path: an embedding outage during the pair pass flags every retire proposal minted that night, they can only be accepted by id, no command re-runs the check (`checkRetirementContinuity` is called only from the pair pass), and reopen preserves the flag, so one bad night means about 90 individual accepts. *Why:* this is rule R3, the forgetting-safety lane's purpose moved to a per-proposal search-rank replay. An unverified query must never read as "no risk found". A dead embedding endpoint would otherwise cost about an hour (S2). (`src/commands/improve/consolidate/continuity-check.ts:5-16,26-29,135-192`; `pair-pass.ts:797-809`; `src/commands/proposal/proposal.ts:536-543`)
|
|
628
|
+
- **Which past queries are replayed.** They come from the usage log: distinct text of user `search` and `curate` events that returned the retired ref, most recent first. They are whitespace-collapsed and deduplicated, capped at `MAX_QUERIES = 5`, and anything over 2,000 chars is dropped. Harness or tool envelopes and the stash README boilerplate are excluded. The cap and the 2,000-char drop are the helper's own constants (kept from the removed reflect retrieval gate): its comment says the retrieval suite treats a longer input as a paste, but `non-task-input.ts` says the opposite ("Length is not a signal": prompts over 2,000 characters found relevant assets at about the rate of shorter long prompts). (`src/commands/improve/retrieval-queries.ts:15-39`; `src/indexer/usage/usage-events.ts:180-198`; `src/core/non-task-input.ts:8-20`)
|
|
636
629
|
- **Second look (`confirmNothingLost`)** — One more call asks only what the retired note holds that the kept one lacks. It is true only on a clean, empty `missing` list. A failed call counts as not confirmed. *Why:* "it caught 2 of 4 duplicates the judge got wrong, and held back none of 109 right ones". (`pair-pass.ts:676-701`)
|
|
637
630
|
- **Gate staging (when a retire proposal is staged)** — The proposal is always minted `pending`. It is stamped `staged` (gate `consolidate-pair`, reason = the judge label, content hash) only when all three hold:
|
|
638
631
|
- for a `duplicate`, both claim lists are empty;
|
|
@@ -684,7 +677,7 @@ Paths are relative to the rules-010 checkout. `repository.ts` is `src/commands/p
|
|
|
684
677
|
- **Orphan purge** — Only pending `reflect` proposals whose target no longer exists in any configured source dir are auto-rejected, with gate `orphan-purge`, reason `asset-missing` and review reason "Asset no longer exists on disk". Lessons and other sources are skipped. *Why:* other sources, lessons above all, may legitimately target assets that do not exist yet. (`repository.ts:827-869`; `loop-stages.ts:671-696`)
|
|
685
678
|
- **Procedural rejections are not content verdicts** — Rejections from stale target, expiry and orphan purge (and old rejections whose review reason starts `expired:`) are excluded from the "previously rejected" prompt context and the accept-rate metric. (`proposal-types.ts:340-357`)
|
|
686
679
|
- **Rejected-proposal context for the model** — A stage is shown at most 3 (`MAX_REJECTED_PROPOSALS`) of the newest non-procedural rejections for the same ref, each with its reason and the first 500 characters of the rejected content, as "don't repeat this". It reads the archive. (`src/commands/improve/stage.ts:208-231`)
|
|
687
|
-
- **Gate decisions** — A gate stamps `staged`, `deferred`, `auto-accepted` or `auto-rejected` on a pending proposal without changing its status. `staged` means a judge passed this exact content hash. `deferred` writes a `review_needed` ledger row, and a proposal no longer pending is silently skipped. `akm feedback` stamps `auto-accepted` with reason `manual-fix` (gate `feedback`) on an exact fix it then accepts at once (changed in this PR, decision 3); reflect and distill stamp `deferred` with gate `quality-gate` for `judge-error`, `judge-uncertain
|
|
680
|
+
- **Gate decisions** — A gate stamps `staged`, `deferred`, `auto-accepted` or `auto-rejected` on a pending proposal without changing its status. `staged` means a judge passed this exact content hash. `deferred` writes a `review_needed` ledger row, and a proposal no longer pending is silently skipped. `akm feedback` stamps `auto-accepted` with reason `manual-fix` (gate `feedback`) on an exact fix it then accepts at once (changed in this PR, decision 3); reflect and distill stamp `deferred` with gate `quality-gate` for `judge-error`, `judge-uncertain` and `quality-review` (changed in this PR, decision 4), and the drain leaves a `deferred` proposal from a gate other than its own to a person (`drain.ts:586-590`). (`repository.ts:792-825`; `proposal-types.ts:111-151`)
|
|
688
681
|
- **`akm feedback` basics** — Exactly one of `--positive` or `--negative` is required. The signal is recorded as a usage event. For a user-attributed event the utility is recomputed right away with no reindex, by the same function and formula `akm index` uses (see Utility update). When a previously high-utility asset (0.5 or more) drops below 0.5, an `improve_review_needed` event is emitted (an event, not a proposal, so it cannot overwrite an asset); no code consumes it. The output's `rankingUpdate` field reports the ranking effect, not the utility write: `applied: true` exactly when the recorded event is user-attributed (`source='user'`), which `akm search` and `akm curate` count themselves, so the next search already reflects the signal (see Search: Feedback scales the fused score; changed in this PR, decision 1); a failed utility write only warns and does not change it. The utility score is read by improve and metrics only. (`src/commands/feedback-cli.ts:173-240,586-592,834-854,897-899`; #386)
|
|
689
682
|
- **Anti-self-reinforcement guard** — A feedback event whose usage-event source is not `user` (`improve`, `task`, `audit`, `unknown`) is recorded but does not count toward search ranking, and the utility is not recomputed for it. The output says so as `rankingUpdate: { applied: false, reason }`, the reason naming the source and saying it does not count toward search ranking. Set `AKM_EVENT_SOURCE=user` to record user demand. *Why:* it stops an agent or automated pipeline from boosting its own picks, and the search ranking and the utility recompute both apply the same rule by counting only `source = 'user'` events (`usage-events.ts:122-148`, `indexer.ts:1880`); the code says "must not be removed" (R-034 only asks that the skip not pass silently). (`feedback-cli.ts:224-229,897-899`; tests `tests/integration/feedback-command.test.ts`, "rankingUpdate")
|
|
690
683
|
- **Negative feedback requires `--reason`** — By default a negative with no reason is a `UsageError` (exit 2), telling you to say what is wrong and what should change. `feedback.requireReason: false` in akm.json downgrades this to a warning. The reason is shown to the next improve run, which may repair the asset's description, title or `when_to_use` from it, but not its text. *Why:* F-3 / #384. (`feedback-cli.ts:643-658`; `feedback-cli.ts:494-507`)
|
|
@@ -728,7 +721,7 @@ Paths are relative to the rules-010 checkout. `repository.ts` is `src/commands/p
|
|
|
728
721
|
- **Optional, non-blocking rebuild lock** — `akm index` takes a PID-liveness lock. With `--skip-if-locked` and the lock held, it exits 0 reporting `skipped: lock-held`. Without the flag it warns and contends. There is no age-based reclaim. `indexWrittenAssets` (the write path for `remember`, `import` and extract) skips while the lock is held, uses a 5,000 ms busy timeout, and fails open. *Why:* #872 (an age-based lease stranded an install in a half-day indexing outage, per `run-lock.ts:15`) and #956. (`src/indexer/index-rebuild-lock.ts:53`, `src/commands/sources/stash-cli.ts:90-112`, `src/indexer/index-written-assets.ts:47,86`, #872, #956)
|
|
729
722
|
- **Reads only bootstrap the index** — `ensureIndex` rebuilds inline only when the DB is missing, has no entries table, has 0 rows, or was built for a different stash. It does not check content freshness, and the read path never clones, pulls or fetches sources (`hydrateSources: false`). `mode: "blocking"` (used by improve) also rebuilds when an indexable file under the primary stash's akm placement roots (symlinks and dot-directories skipped) is newer than `builtAt` with a differing hash, or unindexed; other bundles and non-akm layouts never trigger it. So a hand-edited file is visible to search only after a reindex: reads never check freshness, `core/index-refresh` runs daily, and the only sub-daily reindex is that blocking `ensureIndex` in the improve consolidate task. *Why:* the old stale-walk plus detached reindex per read caused writer contention. (`src/indexer/ensure-index.ts:11-22,173-222,227-247`)
|
|
730
723
|
- **Finalize pass** — After embeddings the indexer rebuilds FTS BM25 totals if stale, relinks `usage_events` to entry ids, recomputes `utility_scores`, clears stale LLM-cache rows, purges usage events older than 90 days, and VACUUMs after a layout migration or when more than half the pages are free. *Why:* #1048, where stale totals made an updated index rank differently from a fresh one. (`src/indexer/indexer.ts:339-420,1839-1849`, #1048)
|
|
731
|
-
- **Utility has one formula and one implementation, and search reads neither (changed in this PR)** — `recomputeUtilityScores` is the only writer: `utility = prev × 0.7^days + max(selectRate, max(0, pos − neg) / total) × (1 − 0.7^days)` with `prev ?? 0`, counting only `source='user'` events. `akm index` finalize runs it for every entry on every run (and purges usage events past 90 days, and moves the `last_utility_computed_at` watermark); `akm feedback` runs it for the one entry it touched, without the purge or the watermark (see Proposals and feedback: Utility update). The value is a decayed base converging to a positive-only target that can never fall below the show/search rate; one positive plus one negative is rate 0. `akm search` never reads `utility_scores`; its readers are improve, metrics and this recompute, so search ranking, which takes its feedback signal from the usage events (see Search: Feedback scales the fused score), does not depend on it. (`src/indexer/indexer.ts:362-367,1839-
|
|
724
|
+
- **Utility has one formula and one implementation, and search reads neither (changed in this PR)** — `recomputeUtilityScores` is the only writer: `utility = prev × 0.7^days + max(selectRate, max(0, pos − neg) / total) × (1 − 0.7^days)` with `prev ?? 0`, counting only `source='user'` events. `akm index` finalize runs it for every entry on every run (and purges usage events past 90 days, and moves the `last_utility_computed_at` watermark); `akm feedback` runs it for the one entry it touched, without the purge or the watermark (see Proposals and feedback: Utility update). The value is a decayed base converging to a positive-only target that can never fall below the show/search rate; one positive plus one negative is rate 0. `akm search` never reads `utility_scores`; its readers are improve, metrics and this recompute, so search ranking, which takes its feedback signal from the usage events (see Search: Feedback scales the fused score), does not depend on it. (`src/indexer/indexer.ts:362-367,1839-1950`, `src/commands/feedback-cli.ts:232`)
|
|
732
725
|
- **No LLM work in `akm index`** — The metadata-enhance and LLM entity-graph passes are retired (0.9.17-alpha.9), and `timing.llmMs` stays at about 0. Declared links (`asset_links`, #935: `xrefs`, `supersededBy`, `derived_from`, workflow/task `uses` and similar) are derived from the stored document with no model. The memory-inference pass lives under `src/indexer/passes` but is driven from improve (`loop-stages.ts`), and is gated by `experimental.improveAutonomy`. *Why:* metadata enrichment moved nDCG@10 by -0.0092 at about 27 B70-hours cost (CHANGELOG 0.9.17). (`src/indexer/indexer.ts:739-742`, `src/indexer/links/declared-links.ts:1-30`, `src/commands/improve/loop-stages.ts:517-565`, `CHANGELOG.md:2027`, #935)
|
|
733
726
|
- **`--clean` and `--dry-run`** — `--dry-run` is only valid together with `--clean`, otherwise a `INVALID_FLAG_VALUE` error. `--clean` removes rows whose file is absent but keeps rows for unreadable files, with a warning. *Why:* #791, where EACCES was treated as absent and wiped an index. (`src/indexer/indexer.ts:429-470,607-620`, #791)
|
|
734
727
|
- **Derived memories** — A `*.derived` memory carries a `derived_from` link to its parent. Identical-content copies are not separate search results (see Search). Why: not stated beyond the link mechanics. (`src/indexer/links/declared-links.ts:23-35`)
|
|
@@ -751,11 +744,11 @@ Paths are relative to the rules-010 checkout. `repository.ts` is `src/commands/p
|
|
|
751
744
|
- **`--detail` levels** — Values are `brief|normal|full|agent`, and the default is `brief` from `output.detail`. `agent` is a projection, not a verbosity. Brief search hits keep `type, name, ref, action, estimatedTokens, keys`. Normal is `type, name, description (capped at 250 chars), action, score, estimatedTokens, warnings, quality` (and `keys`): it drops `ref`. Full adds the envelope and timing. Agent keeps `name, ref, type, path, editable, editHint (only if read-only), description, action, score, estimatedTokens, keys`. `akm show` ignores the config default and returns full unless `--detail brief` is passed explicitly. `--shape` no longer exists (#1091). *Why:* a bare `show` must always return the asset body. (`src/output/context.ts:49,115-128`, `src/output/shapes/helpers.ts:13,253-301,340-401`, `src/commands/read/search-cli.ts:155-182,246-262`, `tests/output-shapes-unit.test.ts:131-144`, #1091)
|
|
752
745
|
- **Curate = search + enrichment** — Curate takes the top `limit` of the same fused search, feedback factor included (it calls `akmSearch`, which calls `searchLocal`). Each hit gets a preview (160 chars), run and parameters, and up to 2 `supportRefs` from declared links (outgoing first, then incoming). Registry hits fill only the slots left unfilled. Input that is a harness envelope (starts with `<`, contains `</`) or the stash README line yields an empty abstention instead of unrelated items. `--pack N` greedily packs full hit content up to N tokens in rank order, truncating only a single first oversized hit. (`src/commands/read/curate.ts:129-134,203-225,228,255-268,312-345,495-520`, `src/commands/read/search.ts:195`, `src/core/non-task-input.ts`)
|
|
753
746
|
- **Optional curate rerank** — Off unless `search.curateRerank.enabled`. It reorders the top `topN` fused candidates (default 30, max 50) using name, description and 2,000 chars of indexed content. Any failure keeps the fused order. `search.curateRerank` is host-local and stripped from inherited config. *Why:* #951. (`src/commands/read/curate.ts:132-134,457-480`, `src/core/config/schema/search.ts`, #951)
|
|
754
|
-
- **Feedback adjusts utility, and only for user-sourced signals** — `akm feedback` records a `feedback` usage event. For `source=user` it then recomputes that entry's utility with the same `recomputeUtilityScores` formula `akm index` uses (see Proposals and feedback: Utility update), and if a score that was at least 0.5 falls below 0.5 it emits an `improve_review_needed` event. For improve, task or audit sources no recompute runs and the output reports `rankingUpdate: { applied: false, reason }`. This utility score is read only by improve and metrics; the search ranking uses the events (see Feedback scales the fused score). *Why:* anti-self-reinforcement guard; #862 (search impressions must not inflate scores); F-5/#386. (`src/commands/feedback-cli.ts:173-240,834-854`, `src/indexer/indexer.ts:1839-
|
|
747
|
+
- **Feedback adjusts utility, and only for user-sourced signals** — `akm feedback` records a `feedback` usage event. For `source=user` it then recomputes that entry's utility with the same `recomputeUtilityScores` formula `akm index` uses (see Proposals and feedback: Utility update), and if a score that was at least 0.5 falls below 0.5 it emits an `improve_review_needed` event. For improve, task or audit sources no recompute runs and the output reports `rankingUpdate: { applied: false, reason }`. This utility score is read only by improve and metrics; the search ranking uses the events (see Feedback scales the fused score). *Why:* anti-self-reinforcement guard; #862 (search impressions must not inflate scores); F-5/#386. (`src/commands/feedback-cli.ts:173-240,834-854`, `src/indexer/indexer.ts:1839-1950`)
|
|
755
748
|
- **Feedback scales the fused score (changed in this PR, decision 1)** — After fusion, `searchLocal` reads the user feedback counts of the fused candidates from state.db in one query (`countFeedbackSignalsByRef`: `feedback` events with `source = 'user'`, positives and negatives per `entry_ref`) and multiplies each score by `1 + 0.03 × (ln(1+pos) − ln(1+neg))` (`FEEDBACK_WEIGHT` 0.03), then reorders. The factor is the same for every asset and logarithmic: about 2%, 3%, 5% and 9% for 1, 2, 5 and 20 positives, so 5 positives lift a hit more than 2 and each further vote counts for less, and the effect reorders close matches without overriding relevance. A negative lowers the score the same way; an asset with no user feedback is unchanged. Feedback recorded by improve, task or audit does not count. Counts come straight from the events, so the next search reflects a new signal with no reindex. `utility_scores` is untouched and not read by search; its readers are still `improve` (eligibility, salience, retrieval counts), metrics and the index recompute. A search never creates state.db: if it cannot read the counts it ranks without feedback and adds the warning "Feedback was not applied to the ranking: <error>". *Why:* the owner decided that positive feedback should apply a consistent, scaling boost and negative feedback a consistent, scaling penalty. 0.03 was measured on the retrieval suite's fused lists: 5 positives lift a hit from rank 10 by a median of 3 places and 20 by 4 to 5, and at 0.05 two wrong votes cost about 1.6 times the nDCG@10 they cost at 0.03. (`src/indexer/search/ranking.ts:56-93`; `src/indexer/search/db-search.ts:336-338,364-385`; `src/indexer/usage/usage-events.ts:122-148`; consumers grep: `src/commands/improve/eligibility.ts`, `salience.ts`, `preparation.ts`, `src/commands/metrics/*`)
|
|
756
749
|
- **Negative feedback needs a reason** — `--reason` is required unless `feedback.requireReason: false`, which downgrades to a warning. `--replace/--with/--source`, `--superseded-by` and `--outdated` are negative-only fixes that need `--source` and `--reason`, and each `--replace` text must appear exactly once. A fix is applied at once (changed in this PR, decision 3). `--applied-to lessons/<n>` credits a lesson on positive feedback only. (`src/commands/feedback-cli.ts:557-688,742-768,873-889`)
|
|
757
750
|
- **Usage events recorded** — `search` logs one `search` event per returned hit (first 50 resolved hits) plus one summary event carrying `resultCount`, `mode`, and timings. `curate` logs per-item `curate` events plus a summary. `show` logs one `show` event per successful show, and also `events.jsonl` `show`/`select` traces. `source` comes from `AKM_EVENT_SOURCE` (`user` by default; `improve`, `task`, `audit`, `unknown`; an invalid value becomes `unknown`). Queries are credential-redacted, telemetry uses a short busy timeout and is skipped under contention, and `skipLogging` suppresses it. Curate's nested search and show do not log. (`src/commands/read/search.ts:350-440`, `src/commands/read/curate.ts:140-205`, `src/indexer/usage/show-usage.ts:89-111`, `src/indexer/usage/usage-events.ts:41-49`)
|
|
758
|
-
- **Improve-internal reads and the event source** — The continuity check (`searchLocal`, no logging)
|
|
751
|
+
- **Improve-internal reads and the event source** — The continuity check (`searchLocal`, no logging) and distill's related search (`skipLogging: true, eventSource: "improve"`) are clean. But an agent-engine judge's `akm_show` reads are dispatched with no `eventSource`, so the child inherits the ambient `AKM_EVENT_SOURCE` (`task` under the scheduler; unset, which means `user`, for a human-typed `akm improve`), and only reflect's writer run is stamped `improve`; such reads could count as user demand. Whether the opencode plugin's `akm_show` shells out to the CLI is outside this repository, so this is conditional. *Why:* not stated. (`stage.ts:453-455`, `src/llm/structured-call.ts:273-278`, `src/integrations/agent/runner-dispatch.ts:272-274`, `loop-stages.ts:227`, `distill.ts:1364`)
|
|
759
752
|
- **What counts as retrieval for the 90-day rule** — Usage events are purged after 90 days at each `akm index`. Improve's retrieval scope is the set of `entry_ref`s from `source='user'` events of type `search`, `curate`, `show` or `feedback` in the last 90 days. A `.derived` hit also counts for its parent. Machine sources (`improve`, `task`, `audit`, `unknown`) are excluded. An asset outside the scope that no improve stage has processed and that is older than the window is left out. If state.db is unreadable, every asset stays eligible. *Why:* #986. (`src/indexer/usage/usage-events.ts:167-203`, `src/commands/improve/retrieval-scope.ts:1-92`, #986)
|
|
760
753
|
- **Show loop-break** — If `show` of the same ref was recorded 2 or more times in the previous hour, the response gains `showLoopWarning` (3+ shows). (`src/commands/read/show.ts:142-150`, `src/indexer/usage/show-usage.ts:20-31`)
|
|
761
754
|
- **Did-you-mean** — Search itself has none. It exists for unknown command names (edit distance at most `floor(len/3)`), unknown flags (at most `max(2, ceil(len/3))`), and an `akm improve <scope>` that matches no asset type (a warning only). (`src/cli.ts:969`, `src/cli/unknown-flags.ts:155-185`, `src/commands/improve/eligibility.ts:318-319`)
|
|
@@ -264,36 +264,12 @@ person with reason `proactive-needs-review`, and the judgment tier does not see
|
|
|
264
264
|
it. An empty diff is still rejected. Proposals driven by feedback (`signal-delta`,
|
|
265
265
|
`scope`) follow the one rule unchanged (#1147).
|
|
266
266
|
|
|
267
|
-
### Retrieval regression gate
|
|
268
|
-
|
|
269
|
-
Reflect refuses a revision of an existing asset that grades lower on the
|
|
270
|
-
asset's own retrieval queries (#722, `src/commands/improve/retrieval-gate.ts`).
|
|
271
|
-
After the quality judge passes, up to five distinct user `search`/`curate`
|
|
272
|
-
queries that returned the asset (envelopes, the stash README line and inputs
|
|
273
|
-
over 2,000 characters dropped) are each graded 0–3 against the old and the new
|
|
274
|
-
content with the retrieval eval's prompt, `retrieval-relevance-judge.md`, and
|
|
275
|
-
its document shape: type, ref, name, description and the first 1,500 body
|
|
276
|
-
characters. A lower mean for the new content is a `quality_rejected` refusal
|
|
277
|
-
with the reflect rejection window. A grade that cannot be obtained (a transport
|
|
278
|
-
error, a timeout or a reply that is not a grade) measured no regression, so the
|
|
279
|
-
revision is queued for a person (reason `retrieval-ungraded`, gate
|
|
280
|
-
`quality-gate`) instead of refused; a call aborted by the run's budget is
|
|
281
|
-
neither. An asset with no such queries is not graded.
|
|
282
|
-
|
|
283
|
-
The gate exists because it was measured first. Of 60 accepted reflect rewrites
|
|
284
|
-
(since 2026-07-01, stratified by lane) judged this way, 14 graded lower (23%,
|
|
285
|
-
95% CI 14–35%) and 12 higher. Grading the same content twice flipped 11 of 174
|
|
286
|
-
query grades, which puts 3 of 60 rewrites in the "lower" bucket by noise
|
|
287
|
-
alone. The threshold for building it was fixed before judging: a lower bound of
|
|
288
|
-
at least 10%.
|
|
289
|
-
|
|
290
267
|
### Retirement continuity
|
|
291
268
|
|
|
292
269
|
Before the consolidate pair pass mints a `retire` proposal (rule R3,
|
|
293
270
|
`src/commands/improve/consolidate/continuity-check.ts`), it replays up to five
|
|
294
271
|
of the retired asset's own past `search`/`curate` queries
|
|
295
|
-
(`loadRetrievalQueries`,
|
|
296
|
-
for the retrieval regression gate — boilerplate, harness/tool envelopes,
|
|
272
|
+
(`loadRetrievalQueries`, `../retrieval-queries.ts` — boilerplate, harness/tool envelopes,
|
|
297
273
|
pastes and near-duplicates dropped before replay, S3a) through akm's own
|
|
298
274
|
search, in-process — the ranking a user actually gets, no LLM. For every
|
|
299
275
|
query where the retired asset ranked in the top 10, the successor must too
|
|
@@ -210,8 +210,7 @@ Reflect changes only an asset's `description`, `when_to_use` and title. The engi
|
|
|
210
210
|
6. Validate the reply with `parseSchemaReflectOutput` (`parseFramedReflectOutput` for the framed contract), which strips `<think>` blocks and code fences, then requires exactly the contract's fields: `confidence` and a `frontmatterPatch` of `description`, `when_to_use` and `title` (each a non-empty single-line string, or `null` for no change), plus `ref` when no target was given. A reply that fails gets one repair turn, shared across self-refine passes: the conversation so far, and a request to reformat the reply in the contract (`reflect-output-repair.md`). A reply still invalid fails with `parse_error` and queues nothing. A failed agent or SDK dispatch is reported as it ran, with its exit code and stderr.
|
|
211
211
|
7. Apply the patch (`applyReflectPatch`) to the source asset: its frontmatter with the non-null `description` and `when_to_use` set, and its body unchanged. A non-null `title` becomes a `# <title>` heading and one blank line at the top of the body, only when the body has no level-1 heading; otherwise it is ignored. A source that requires a `description` and has none gets one derived from its own text when the patch gave none (#636). A patch that changes nothing (every field `null`, or equal to the source's, and no `description` to derive) is the `no_change` outcome (`reflect_skipped_noop`), as is a source with no content to patch; a result that differs only cosmetically is `reflect_skipped_cosmetic`. Neither creates a proposal.
|
|
212
212
|
8. Quality gate (`processes.reflect.qualityGate`, on unless disabled): one judge call scores the revision against the source on **need**, **preservation** and **quality** ([Quality judge](../improvement.md#quality-judge)), and passes it only when every criterion scores 4 or more. A score that does not pass is refused (`quality_rejected`, no proposal) unless its mean is 3.5 or more: that is uncertain, not bad, and the proposal is deferred for a person (`judge-uncertain`, gate `quality-gate`, with the criterion `scores` and the judge's reason) and records no rejection window. A judge that times out, errors or replies unparseably gives no verdict, and the proposal is deferred for review too (`judge-error`, gate `quality-gate`). Before the judge, with the gate on or off, reflect refuses (`quality_rejected`, no proposal, no judge call; the rule is the event's `reflectDefect`) a revision that adds placeholder text (`placeholder_added`: "please confirm", "to be confirmed"; not `TODO`, `TBD` or `FIXME`), talks about its own edit (`meta_commentary_added`: "the feedback says", "this revision") or copies frontmatter into its body (`frontmatter_copied_into_body`: `sources:` or `updated:` lines outside code, a `sources` value). Each rule counts only what the revision adds to its source (`findReflectDefect`), and its wording is a list that `processes.reflect.defectFilter` replaces or, when empty, turns off ([Configuration](../../reference/configuration.md#strategies)).
|
|
213
|
-
9.
|
|
214
|
-
10. Write the proposal (`mintProposal`). A pass the judge passed is stamped `staged`, and the triage drain accepts it; with the gate off the proposal is minted unstamped for the drain to decide. One made with no judge configured (`no-judge-configured`) is deferred for review (gate `reflect`), and so is one whose judge gave no verdict (`judge-error`), one in the judge's uncertain band (`judge-uncertain`) and one whose retrieval check could not grade (`retrieval-ungraded`). `akm proposal list` shows the reason; the drain leaves every deferral for a person.
|
|
213
|
+
9. Write the proposal (`mintProposal`). A pass the judge passed is stamped `staged`, and the triage drain accepts it; with the gate off the proposal is minted unstamped for the drain to decide. One made with no judge configured (`no-judge-configured`) is deferred for review (gate `reflect`), and so is one whose judge gave no verdict (`judge-error`) and one in the judge's uncertain band (`judge-uncertain`). `akm proposal list` shows the reason; the drain leaves every deferral for a person.
|
|
215
214
|
|
|
216
215
|
**What it writes:** one durable proposal row in `state.db`. It never writes asset files directly.
|
|
217
216
|
|
|
@@ -485,8 +484,8 @@ proposal.
|
|
|
485
484
|
|
|
486
485
|
Before the pair pass mints a `retire` proposal, it replays up to five of the
|
|
487
486
|
retired asset's own past `search`/`curate` queries
|
|
488
|
-
(`loadRetrievalQueries`, `src/commands/improve/retrieval-
|
|
489
|
-
|
|
487
|
+
(`loadRetrievalQueries`, `src/commands/improve/retrieval-queries.ts`:
|
|
488
|
+
`usableRetrievalQueries`
|
|
490
489
|
drops stash-README boilerplate and harness/tool envelopes
|
|
491
490
|
(`nonTaskInput`), pastes over 2,000 characters, and queries that are
|
|
492
491
|
duplicates once whitespace is collapsed, S3a) through akm's own search,
|
|
@@ -218,7 +218,7 @@ Reasoning effort has one word in a request, `reasoningEffort`. `effort`, as a
|
|
|
218
218
|
Unattended model work is the work akm hands a model with no one watching:
|
|
219
219
|
- the improve processes (reflect, distill, consolidate, memory inference,
|
|
220
220
|
extract, and validation's repair);
|
|
221
|
-
- the quality
|
|
221
|
+
- the quality and triage judges;
|
|
222
222
|
- index passes;
|
|
223
223
|
- `akm remember --enrich`.
|
|
224
224
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "akm-cli",
|
|
3
|
-
"version": "0.10.
|
|
3
|
+
"version": "0.10.26101201-alpha",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "akm (Agent Knowledge Manager) — a portable, local-first capability library for AI agents. Discover, load, share, and improve reusable skills, scripts, workflows, and knowledge across any shell-capable coding agent, including Claude Code, OpenCode, and Cursor.",
|
|
6
6
|
"keywords": [
|
|
@@ -1,6 +0,0 @@
|
|
|
1
|
-
You are grading search-and-retrieval results for an AI coding agent's knowledge base (the akm tool). For the given query and ONE candidate asset, grade how useful loading this asset would be, on this scale:
|
|
2
|
-
3 = exactly the asset an agent should load for this query or task; it directly answers or performs the request.
|
|
3
|
-
2 = relevant and clearly useful, though not the single best asset for the query.
|
|
4
|
-
1 = same general topic as the query, but would not actually help complete this specific query or task.
|
|
5
|
-
0 = unrelated to the query.
|
|
6
|
-
Reply with ONLY a JSON object: {"grade": <integer 0-3>, "reason": "<=25 words"}.
|
|
@@ -1,134 +0,0 @@
|
|
|
1
|
-
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
2
|
-
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
3
|
-
// file, You can obtain one at https://mozilla.org/MPL/2.0/.
|
|
4
|
-
/**
|
|
5
|
-
* The retrieval regression gate (#722): a reflect rewrite of an existing asset
|
|
6
|
-
* must not grade lower on the queries that actually retrieved it.
|
|
7
|
-
*
|
|
8
|
-
* Measured before it was built: of 60 accepted reflect rewrites judged against
|
|
9
|
-
* their own queries, 14 graded lower (23%, 95% CI 14–35%) and 12 higher. The
|
|
10
|
-
* old and new content are graded one query at a time, blind, with the
|
|
11
|
-
* retrieval-eval judge's prompt (kappa 0.83 against human grades) and its
|
|
12
|
-
* document shape: type, ref, name, description and the first 1,500 characters
|
|
13
|
-
* of the body.
|
|
14
|
-
*/
|
|
15
|
-
import relevanceJudgePrompt from "../../assets/prompts/retrieval-relevance-judge.md" with { type: "text" };
|
|
16
|
-
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
17
|
-
import { parseRefInput } from "../../core/asset/resolve-ref.js";
|
|
18
|
-
import { nonTaskInput } from "../../core/non-task-input.js";
|
|
19
|
-
import { parseEmbeddedJsonResponse } from "../../core/parse.js";
|
|
20
|
-
import { listRetrievalQueries } from "../../indexer/usage/usage-events.js";
|
|
21
|
-
import { readLedgerDb, stripBundle } from "./ledger.js";
|
|
22
|
-
import { callStage } from "./stage.js";
|
|
23
|
-
/** Queries graded per rewrite, as measured. */
|
|
24
|
-
const MAX_QUERIES = 5;
|
|
25
|
-
/** The retrieval suite treats a longer input as a paste, not a query; the measurement did the same. */
|
|
26
|
-
const MAX_QUERY_CHARS = 2000;
|
|
27
|
-
/** The judge sees this much of the body, as in the retrieval eval. */
|
|
28
|
-
const MAX_DOC_CHARS = 1500;
|
|
29
|
-
export const GRADE_SCHEMA = {
|
|
30
|
-
type: "object",
|
|
31
|
-
required: ["grade", "reason"],
|
|
32
|
-
additionalProperties: false,
|
|
33
|
-
properties: { grade: { type: "integer", minimum: 0, maximum: 3 }, reason: { type: "string" } },
|
|
34
|
-
};
|
|
35
|
-
function parseGrade(raw) {
|
|
36
|
-
const grade = parseEmbeddedJsonResponse(raw)?.grade;
|
|
37
|
-
return typeof grade === "number" && Number.isInteger(grade) && grade >= 0 && grade <= 3 ? grade : undefined;
|
|
38
|
-
}
|
|
39
|
-
/** Up to five distinct task queries, in the given order, whitespace collapsed. */
|
|
40
|
-
export function usableRetrievalQueries(raw) {
|
|
41
|
-
const out = [];
|
|
42
|
-
for (const text of raw) {
|
|
43
|
-
const query = text.replace(/\s+/g, " ").trim();
|
|
44
|
-
if (!query || query.length > MAX_QUERY_CHARS || nonTaskInput(query) || out.includes(query))
|
|
45
|
-
continue;
|
|
46
|
-
out.push(query);
|
|
47
|
-
if (out.length === MAX_QUERIES)
|
|
48
|
-
break;
|
|
49
|
-
}
|
|
50
|
-
return out;
|
|
51
|
-
}
|
|
52
|
-
/** The asset's own retrieval queries from the usage log (none when state.db cannot be read). */
|
|
53
|
-
export function loadRetrievalQueries(access, ref) {
|
|
54
|
-
try {
|
|
55
|
-
return usableRetrievalQueries(readLedgerDb(access, (db) => listRetrievalQueries(db, stripBundle(ref))) ?? []);
|
|
56
|
-
}
|
|
57
|
-
catch {
|
|
58
|
-
return [];
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
function judgeDocument(ref, raw) {
|
|
62
|
-
const conceptId = stripBundle(ref);
|
|
63
|
-
const parsed = parseRefInput(conceptId);
|
|
64
|
-
const { data, content } = parseFrontmatter(raw);
|
|
65
|
-
const text = content
|
|
66
|
-
.replace(/\n{3,}/g, "\n\n")
|
|
67
|
-
.trim()
|
|
68
|
-
.slice(0, MAX_DOC_CHARS);
|
|
69
|
-
const name = typeof data.name === "string" && data.name ? data.name : parsed.name;
|
|
70
|
-
const description = typeof data.description === "string" ? data.description : "";
|
|
71
|
-
return [
|
|
72
|
-
"Candidate asset:",
|
|
73
|
-
`Type: ${parsed.type}`,
|
|
74
|
-
`Ref: ${conceptId}`,
|
|
75
|
-
`Name: ${name}`,
|
|
76
|
-
`Description: ${description}`,
|
|
77
|
-
"",
|
|
78
|
-
"Content:",
|
|
79
|
-
text,
|
|
80
|
-
].join("\n");
|
|
81
|
-
}
|
|
82
|
-
/**
|
|
83
|
-
* Grade `before` and `after` on each query; refuse the rewrite when the new
|
|
84
|
-
* content's mean grade is lower. A grade that cannot be obtained is no
|
|
85
|
-
* regression: the verdict is `ungraded` and a person decides, as when the
|
|
86
|
-
* quality judge gives no verdict. An aborted call is neither.
|
|
87
|
-
*/
|
|
88
|
-
export async function runRetrievalRegressionGate(args) {
|
|
89
|
-
if (args.queries.length === 0)
|
|
90
|
-
return { pass: true, queries: 0, reason: "no retrieval queries to compare on" };
|
|
91
|
-
const documents = { old: judgeDocument(args.ref, args.before), new: judgeDocument(args.ref, args.after) };
|
|
92
|
-
const totals = { old: 0, new: 0 };
|
|
93
|
-
for (const query of args.queries) {
|
|
94
|
-
for (const version of ["old", "new"]) {
|
|
95
|
-
const outcome = await callStage({
|
|
96
|
-
feature: "proposal_quality_gate",
|
|
97
|
-
runner: args.runner,
|
|
98
|
-
system: relevanceJudgePrompt.trim(),
|
|
99
|
-
prompt: `Query: ${query}\n\n${documents[version]}`,
|
|
100
|
-
request: {
|
|
101
|
-
enableThinking: false,
|
|
102
|
-
temperature: 0,
|
|
103
|
-
responseSchema: GRADE_SCHEMA,
|
|
104
|
-
...(args.timeoutMs !== undefined ? { timeoutMs: args.timeoutMs } : {}),
|
|
105
|
-
...(args.signal ? { signal: args.signal } : {}),
|
|
106
|
-
...(args.chat ? { chat: args.chat } : {}),
|
|
107
|
-
},
|
|
108
|
-
parse: parseGrade,
|
|
109
|
-
...(args.onNotices ? { onNotices: args.onNotices } : {}),
|
|
110
|
-
});
|
|
111
|
-
const grade = outcome.ok ? parseGrade(outcome.raw) : undefined;
|
|
112
|
-
if (grade === undefined) {
|
|
113
|
-
return {
|
|
114
|
-
pass: false,
|
|
115
|
-
queries: args.queries.length,
|
|
116
|
-
reason: `retrieval check could not grade the ${version} content${outcome.ok ? "" : ` (${outcome.reason})`}`,
|
|
117
|
-
...(outcome.ok || outcome.reason !== "aborted" ? { ungraded: true } : {}),
|
|
118
|
-
};
|
|
119
|
-
}
|
|
120
|
-
totals[version] += grade;
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
const oldMean = totals.old / args.queries.length;
|
|
124
|
-
const newMean = totals.new / args.queries.length;
|
|
125
|
-
const pass = newMean >= oldMean;
|
|
126
|
-
const grades = `${oldMean.toFixed(2)} -> ${newMean.toFixed(2)} over ${args.queries.length} retrieval ${args.queries.length === 1 ? "query" : "queries"}`;
|
|
127
|
-
return {
|
|
128
|
-
pass,
|
|
129
|
-
queries: args.queries.length,
|
|
130
|
-
oldMean,
|
|
131
|
-
newMean,
|
|
132
|
-
reason: pass ? `retrieval grade ${grades}` : `retrieval regression: grade ${grades}`,
|
|
133
|
-
};
|
|
134
|
-
}
|