@event4u/agent-config 14.13.0 → 14.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.augment-plugin/marketplace.json +3 -3
- package/.augment-plugin/plugin.json +1 -1
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +116 -387
- package/dist/agent-src/commands/analyze/inbox.md +36 -0
- package/dist/agent-src/rules/code-comment-discipline.md +6 -0
- package/dist/agent-src/scripts/archive_completed_roadmaps.ts +6 -6
- package/dist/agent-src/scripts/guarded_baseline.ts +22 -7
- package/dist/agent-src/scripts/stubs_due.ts +118 -2
- package/dist/agent-src/scripts/update_roadmap_progress.ts +8 -8
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/cli-delegate/cmd_conformance.js +38 -1
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +4 -4
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +1 -1
- package/dist/discovery/dormancy-report.md +1 -1
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +2 -2
- package/dist/discovery/trust-report.md +1 -1
- package/dist/discovery/workspaces.json +2 -2
- package/dist/hooks/dispatch.js +1438 -711
- package/dist/mcp/registry-manifest.json +2 -2
- package/dist/mcp/server.json +2 -2
- package/dist/mcp/server.mjs +112 -17
- package/docs/guidelines/agent-infra/guarded-baseline.md +1 -1
- package/docs/guidelines/code-clarity.md +18 -0
- package/package.json +1 -1
- package/src/agent-src/scripts/archive_completed_roadmaps.ts +6 -6
- package/src/agent-src/scripts/guarded_baseline.ts +22 -7
- package/src/agent-src/scripts/stubs_due.ts +118 -2
- package/src/agent-src/scripts/update_roadmap_progress.ts +8 -8
- package/src/config/estate-count-budget.json +20 -1
- package/src/config/gate-coverage.yml +150 -0
- package/src/config/gate-violation-baselines.json +8 -3
- package/src/config/preamble-payload-budget.json +19 -11
- package/src/config/rule-stub-ceilings.json +14 -7
- package/src/scripts/_dispatch.bash +0 -4
- package/src/scripts/_lib/activation_ladder.ts +47 -3
- package/src/scripts/_lib/activation_receipt_producer.ts +341 -0
- package/src/scripts/_lib/adr_frontmatter.ts +26 -0
- package/src/scripts/_lib/candidate_pair_delta.ts +349 -0
- package/src/scripts/_lib/carrier_status.ts +118 -0
- package/src/scripts/_lib/cascade_stage_enumeration.ts +92 -0
- package/src/scripts/_lib/corpus_manifest.ts +726 -0
- package/src/scripts/_lib/delivery_arm_experiment.ts +18 -2
- package/src/scripts/_lib/estate_carrier_cases.ts +72 -0
- package/src/scripts/_lib/evaluation_cascade.ts +452 -0
- package/src/scripts/_lib/evolution_roi.ts +562 -0
- package/src/scripts/_lib/harness_evolution_guards.ts +10 -0
- package/src/scripts/_lib/lexical_shortlist.ts +215 -0
- package/src/scripts/_lib/llm_candidate_proposer.ts +461 -0
- package/src/scripts/_lib/llm_proposer_transport.ts +153 -0
- package/src/scripts/_lib/pathology_archive.ts +393 -0
- package/src/scripts/_lib/promotion_capability.ts +315 -0
- package/src/scripts/_lib/promotion_evidence.ts +414 -0
- package/src/scripts/_lib/promotion_review.ts +294 -0
- package/src/scripts/_lib/prompt_shape.ts +92 -0
- package/src/scripts/_lib/release_highlights.ts +70 -1
- package/src/scripts/_lib/release_material.ts +125 -2
- package/src/scripts/_lib/routing_corpus.ts +307 -0
- package/src/scripts/_lib/routing_index_input.ts +104 -0
- package/src/scripts/_lib/rule_injection.ts +22 -1
- package/src/scripts/_lib/semantic_noop.ts +167 -0
- package/src/scripts/activation_receipt.ts +258 -0
- package/src/scripts/ai_council/cross_exam.ts +128 -0
- package/src/scripts/ai_council/explain_route.ts +157 -0
- package/src/scripts/ai_council/information_gain.ts +293 -0
- package/src/scripts/ai_council/leakage_corpus.ts +385 -0
- package/src/scripts/ai_council/leakage_patterns.ts +453 -0
- package/src/scripts/ai_council/minority_retention.ts +171 -0
- package/src/scripts/ai_council/recouncil_savings.ts +299 -0
- package/src/scripts/ai_council/replay_route.ts +223 -0
- package/src/scripts/ai_council/reviewer_assignment.ts +279 -0
- package/src/scripts/ai_council/routing_training_row.ts +197 -0
- package/src/scripts/ai_council/synthesis_strategy.ts +159 -0
- package/src/scripts/check_adr_frontmatter.ts +4 -9
- package/src/scripts/check_estate_count.ts +205 -24
- package/src/scripts/check_release_highlights.ts +96 -13
- package/src/scripts/check_roadmap_trackable.ts +17 -2
- package/src/scripts/corpus_manifest.ts +235 -0
- package/src/scripts/evolution_lab.ts +181 -8
- package/src/scripts/hook_manifest.json +1 -1
- package/src/scripts/hook_manifest.yaml +28 -6
- package/src/scripts/hooks/comment_discipline_hook.ts +146 -0
- package/src/scripts/hooks/concern_registry.ts +2 -0
- package/src/scripts/hooks/turn_end_gate_hook.ts +9 -1
- package/src/scripts/language_mirror_hook.ts +14 -2
- package/src/scripts/lint_carrier_integrity.ts +820 -0
- package/src/scripts/lint_code_comments.ts +643 -0
- package/src/scripts/lint_plan_risk_register.ts +39 -19
- package/src/scripts/lint_promotion_paths.ts +779 -0
- package/src/scripts/llm_propose.ts +256 -0
- package/src/scripts/measure_delivery_sets.ts +361 -0
- package/src/scripts/measure_routing_signal.ts +307 -0
- package/src/scripts/model_rule_injection.ts +19 -3
- package/src/scripts/probe_council_retention.ts +495 -0
- package/src/scripts/release.ts +21 -37
- package/src/scripts/release_drill.ts +143 -0
- package/src/scripts/release_env.ts +35 -0
- package/src/scripts/release_publication.ts +84 -0
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A lexical shortlist over the existing BM25 core — a SHORTLIST, never a decider.
|
|
3
|
+
*
|
|
4
|
+
* `road-to-governed-harness-evolution` Phase 6, step 6.3.
|
|
5
|
+
*
|
|
6
|
+
* > *6.3 Only then consider a lexical shortlist, and only as a shortlist. Over
|
|
7
|
+
* > the existing BM25 core. No embeddings […]. The skipped parent proposed the
|
|
8
|
+
* > shortlist and explicitly refused it as final truth.*
|
|
9
|
+
* > verify: **the shortlist feeds a later stage and never decides alone.**
|
|
10
|
+
*
|
|
11
|
+
* ## No embeddings, and the citation this step carried had moved
|
|
12
|
+
*
|
|
13
|
+
* The step's own text cites `docs/contracts/no-runtime-boundary.md:40` for the
|
|
14
|
+
* classification. That file is now a **superseded pointer** (ADR-249,
|
|
15
|
+
* 2026-08-27) and line 40 no longer says it. The substance survived the move
|
|
16
|
+
* intact and is quoted from its live home instead:
|
|
17
|
+
* `docs/contracts/resident-process-governance.md:82` — *"A code-graph cache
|
|
18
|
+
* passes; a vector index fails"*, under the P3 state-store test, with P4
|
|
19
|
+
* separately prohibiting any index build requiring network or model calls.
|
|
20
|
+
* So this module builds over `_lib/lexical_index.ts` — hand-rolled BM25, pure
|
|
21
|
+
* Node stdlib, the core ADR-061 already sanctions. A static scan in
|
|
22
|
+
* `tests/scripts/_lib/lexical_shortlist.test.ts` asserts this module and that
|
|
23
|
+
* core reach no embedding construct, no vector store and no model call. The
|
|
24
|
+
* construct list lives in the test rather than here for a mechanical reason:
|
|
25
|
+
* a scanner whose banned literals sit in the file it scans matches its own
|
|
26
|
+
* declaration and can never pass — the same reason
|
|
27
|
+
* `tests/scripts/governed_harness_no_live_harness.test.ts` keeps its banned set
|
|
28
|
+
* on the test side.
|
|
29
|
+
*
|
|
30
|
+
* ## "Never decides alone", made mechanical rather than promised
|
|
31
|
+
*
|
|
32
|
+
* A shortlist decides alone in exactly two ways, and both are closed here by
|
|
33
|
+
* construction rather than by a caller's discipline:
|
|
34
|
+
*
|
|
35
|
+
* 1. **By addition** — delivering something the authoritative matcher did not
|
|
36
|
+
* fire on. Impossible: {@link orderByShortlist} takes the matcher's output
|
|
37
|
+
* as its input domain and returns a PERMUTATION of it. Ids the shortlist
|
|
38
|
+
* ranks that the matcher did not return are dropped on the floor, and
|
|
39
|
+
* {@link selectForInjection}'s tie-break argument is consulted only to
|
|
40
|
+
* order ids that are already in `matches`.
|
|
41
|
+
* 2. **By subtraction** — silently removing a matcher hit, which is a
|
|
42
|
+
* decision dressed as a filter. Impossible: the same permutation
|
|
43
|
+
* invariant. {@link ShortlistDecidedAloneError} is thrown when the output
|
|
44
|
+
* id multiset differs from the input's in either direction, so a future
|
|
45
|
+
* edit that turns the reordering into a filter fails loudly instead of
|
|
46
|
+
* quietly losing recall.
|
|
47
|
+
*
|
|
48
|
+
* The shortlist's actual job is the third thing, and it is a real one: when the
|
|
49
|
+
* per-prompt byte cap BINDS, something has to choose which matched bodies
|
|
50
|
+
* survive it. Today that is the matcher's trigger-hit count and then router
|
|
51
|
+
* declaration order — and the hit count is a coarse integer on which most rules
|
|
52
|
+
* tie, so the surviving set is decided by declaration order, which is not a
|
|
53
|
+
* relevance signal at all. The shortlist supplies one, subordinate to the
|
|
54
|
+
* matcher score and never above it.
|
|
55
|
+
*
|
|
56
|
+
* ## Subordination is an ORDER of comparison keys, and the order is the contract
|
|
57
|
+
*
|
|
58
|
+
* `score desc → shortlist rank → router order`. The matcher's verdict is read
|
|
59
|
+
* first and the shortlist only breaks its ties. Promoting the shortlist above
|
|
60
|
+
* `score` would let a lexically-similar rule outrank one the matcher fired on
|
|
61
|
+
* twice, which is the shortlist deciding — a test pins the key order and is
|
|
62
|
+
* proved to go red when it is inverted.
|
|
63
|
+
*
|
|
64
|
+
* ## What this module does NOT claim
|
|
65
|
+
*
|
|
66
|
+
* That the shortlist improves anything. Whether a lexical tie-break beats
|
|
67
|
+
* declaration order under a binding cap is a MEASUREMENT, and step 6.4 is where
|
|
68
|
+
* the loss ceiling for it gets pre-registered. `model_rule_injection
|
|
69
|
+
* --three-arm --shortlist` runs the arm with it, default OFF, so 6.1's recorded
|
|
70
|
+
* figures are reproducible unchanged.
|
|
71
|
+
*/
|
|
72
|
+
import { LexicalIndex, type IndexDoc } from './lexical_index.js';
|
|
73
|
+
import { allTierRules, loadRuleBody, type Router, type TierRuleMatch } from './rule_injection.js';
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* How many ids a shortlist may carry.
|
|
77
|
+
*
|
|
78
|
+
* A STATED default, not a measured optimum, and deliberately generous: this
|
|
79
|
+
* shortlist narrows nothing (it reorders), so `k` bounds only how far down the
|
|
80
|
+
* BM25 ranking a tie-break signal is available before router order takes over.
|
|
81
|
+
* Revisit-if 6.4 measures a cap-drop the shortlist could have prevented at a
|
|
82
|
+
* rank beyond it.
|
|
83
|
+
*/
|
|
84
|
+
export const SHORTLIST_SIZE = 40;
|
|
85
|
+
|
|
86
|
+
export class ShortlistDecidedAloneError extends Error {
|
|
87
|
+
constructor(message: string) {
|
|
88
|
+
super(message);
|
|
89
|
+
this.name = 'ShortlistDecidedAloneError';
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Index every tier rule that has a projected body.
|
|
95
|
+
*
|
|
96
|
+
* Kernel rules are excluded for the same reason `_lib/rule_injection.ts`
|
|
97
|
+
* excludes them from injection: they are standing context by definition, so
|
|
98
|
+
* they are never candidates for a cap that only ever binds on injected bodies.
|
|
99
|
+
* `allTierRules` already returns the non-kernel set.
|
|
100
|
+
*/
|
|
101
|
+
export function buildRuleIndex(repoRoot: string, router: Router): LexicalIndex {
|
|
102
|
+
const docs: IndexDoc[] = [];
|
|
103
|
+
for (const r of allTierRules(router)) {
|
|
104
|
+
const body = loadRuleBody(repoRoot, r.id);
|
|
105
|
+
if (body === null) continue;
|
|
106
|
+
// The id is joined into the text on purpose: a rule's own name is the
|
|
107
|
+
// single most reliable lexical signal it carries, and hyphenated ids
|
|
108
|
+
// tokenise into exactly the words a prompt about that rule would use.
|
|
109
|
+
docs.push({ id: r.id, text: `${r.id.split('-').join(' ')}\n${body}` });
|
|
110
|
+
}
|
|
111
|
+
return new LexicalIndex(docs);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The ranked shortlist for one prompt. BM25 only; no model call, no embedding.
|
|
116
|
+
*
|
|
117
|
+
* Returns ids in descending relevance, capped at `k`. An empty result is a real
|
|
118
|
+
* answer — it means no indexed body shares a scoring term with the prompt, and
|
|
119
|
+
* the caller then falls back to router order exactly as it does today.
|
|
120
|
+
*/
|
|
121
|
+
export function shortlistIds(
|
|
122
|
+
index: LexicalIndex,
|
|
123
|
+
prompt: string,
|
|
124
|
+
k: number = SHORTLIST_SIZE,
|
|
125
|
+
): string[] {
|
|
126
|
+
if (k <= 0) return [];
|
|
127
|
+
return index.rank([prompt]).slice(0, k).map((r) => r.id);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Rank lookup: id → position in the shortlist, or `Infinity` for absent.
|
|
132
|
+
*
|
|
133
|
+
* `Infinity` rather than a large integer so an unshortlisted id can never sort
|
|
134
|
+
* above a shortlisted one by arithmetic accident.
|
|
135
|
+
*/
|
|
136
|
+
export function shortlistRanks(ranked: readonly string[]): Map<string, number> {
|
|
137
|
+
const m = new Map<string, number>();
|
|
138
|
+
for (const [i, id] of ranked.entries()) {
|
|
139
|
+
if (!m.has(id)) m.set(id, i);
|
|
140
|
+
}
|
|
141
|
+
return m;
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Reorder the matcher's output by the shortlist — a PERMUTATION, always.
|
|
146
|
+
*
|
|
147
|
+
* This is the function that makes "never decides alone" checkable. It takes the
|
|
148
|
+
* matcher's verdict as its input domain, applies the same key order
|
|
149
|
+
* {@link selectForInjection} applies (`score desc → shortlist rank → router
|
|
150
|
+
* order`), and REFUSES to return anything whose id multiset differs from what
|
|
151
|
+
* it was given.
|
|
152
|
+
*
|
|
153
|
+
* @throws {ShortlistDecidedAloneError} when the result would add or drop an id.
|
|
154
|
+
*/
|
|
155
|
+
export function orderByShortlist(
|
|
156
|
+
matches: readonly TierRuleMatch[],
|
|
157
|
+
ranked: readonly string[],
|
|
158
|
+
): TierRuleMatch[] {
|
|
159
|
+
const rank = shortlistRanks(ranked);
|
|
160
|
+
const out = [...matches].sort(
|
|
161
|
+
(a, b) =>
|
|
162
|
+
b.score - a.score ||
|
|
163
|
+
(rank.get(a.id) ?? Infinity) - (rank.get(b.id) ?? Infinity) ||
|
|
164
|
+
a.order - b.order,
|
|
165
|
+
);
|
|
166
|
+
assertPermutation(matches, out);
|
|
167
|
+
return out;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/** Same-multiset assertion, in both directions. Exported so a test can drive it. */
|
|
171
|
+
export function assertPermutation(
|
|
172
|
+
before: readonly TierRuleMatch[],
|
|
173
|
+
after: readonly TierRuleMatch[],
|
|
174
|
+
): void {
|
|
175
|
+
const count = (xs: readonly TierRuleMatch[]): Map<string, number> => {
|
|
176
|
+
const m = new Map<string, number>();
|
|
177
|
+
for (const x of xs) m.set(x.id, (m.get(x.id) ?? 0) + 1);
|
|
178
|
+
return m;
|
|
179
|
+
};
|
|
180
|
+
const a = count(before);
|
|
181
|
+
const b = count(after);
|
|
182
|
+
const added: string[] = [];
|
|
183
|
+
const dropped: string[] = [];
|
|
184
|
+
for (const [id, n] of b) {
|
|
185
|
+
if ((a.get(id) ?? 0) < n) added.push(id);
|
|
186
|
+
}
|
|
187
|
+
for (const [id, n] of a) {
|
|
188
|
+
if ((b.get(id) ?? 0) < n) dropped.push(id);
|
|
189
|
+
}
|
|
190
|
+
if (added.length > 0 || dropped.length > 0) {
|
|
191
|
+
throw new ShortlistDecidedAloneError(
|
|
192
|
+
'the shortlist changed the matcher`s set instead of ordering it' +
|
|
193
|
+
(added.length > 0 ? ` — added ${added.sort().join(', ')}` : '') +
|
|
194
|
+
(dropped.length > 0 ? ` — dropped ${dropped.sort().join(', ')}` : '') +
|
|
195
|
+
'. A shortlist that adds delivers what the matcher never fired on; one that ' +
|
|
196
|
+
'drops decides by subtraction. Both are the shortlist deciding alone.',
|
|
197
|
+
);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* A per-prompt shortlist function, bound to one repo root and router.
|
|
203
|
+
*
|
|
204
|
+
* The index is built ONCE and reused for every prompt — building it per prompt
|
|
205
|
+
* would make the cheap stage the expensive one, which is the opposite of what a
|
|
206
|
+
* shortlist is for.
|
|
207
|
+
*/
|
|
208
|
+
export function makePromptShortlist(
|
|
209
|
+
repoRoot: string,
|
|
210
|
+
router: Router,
|
|
211
|
+
k: number = SHORTLIST_SIZE,
|
|
212
|
+
): (prompt: string) => readonly string[] {
|
|
213
|
+
const index = buildRuleIndex(repoRoot, router);
|
|
214
|
+
return (prompt: string) => shortlistIds(index, prompt, k);
|
|
215
|
+
}
|
|
@@ -0,0 +1,461 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The METERED candidate proposer — the second arm, and the one that costs money.
|
|
3
|
+
*
|
|
4
|
+
* `road-to-governed-evidence-production` step 2.1, built under the
|
|
5
|
+
* **NARROWED 2026-09-01** disposition of `blocker: metered-backend-park`: a
|
|
6
|
+
* metered PROPOSER is admitted, a metered EVALUATOR stays forbidden.
|
|
7
|
+
*
|
|
8
|
+
* `_lib/candidate_proposer.ts` is the deterministic arm and the surface this
|
|
9
|
+
* mirrors — same {@link DefectObservation} input, same {@link CandidateRecord}
|
|
10
|
+
* output, same id scheme — so the two are comparable pair-wise over one input
|
|
11
|
+
* rather than being two different experiments wearing one name.
|
|
12
|
+
*
|
|
13
|
+
* ## The role constraint, and where each half of it is enforced
|
|
14
|
+
*
|
|
15
|
+
* > *A metered call may generate candidate text. It may not score, rank,
|
|
16
|
+
* > filter, select between, or supply any input to the verdict for the arms
|
|
17
|
+
* > being compared — whatever the module is called.*
|
|
18
|
+
*
|
|
19
|
+
* Held by construction, not by intention. Six places:
|
|
20
|
+
*
|
|
21
|
+
* | Forbidden role | What makes it unavailable |
|
|
22
|
+
* |---|---|
|
|
23
|
+
* | supply a score | {@link GenerationResult} carries `text` and `model` and nothing else, and {@link NoDecisionField} makes adding a scoring key a BUILD ERROR |
|
|
24
|
+
* | rank | no function here accepts more than one generation for one observation; there is no code path that holds two |
|
|
25
|
+
* | select between candidates | exactly one record per observation, asserted at the return by {@link CandidateCountError} |
|
|
26
|
+
* | filter | an observation that cannot be satisfied THROWS; it is never dropped, so the output can never be a subset |
|
|
27
|
+
* | reorder | the output order is `byteCompare` over the INPUT, reusing the deterministic arm's own comparator — model output is never an ordering key |
|
|
28
|
+
* | reach the verdict | this module imports no verdict module, asserted by its test |
|
|
29
|
+
*
|
|
30
|
+
* The retry loop is the one place a reader should look twice. It stops at the
|
|
31
|
+
* FIRST output that satisfies the record contract; it never holds two valid
|
|
32
|
+
* generations and never compares them. First-valid is a retry policy. Choosing
|
|
33
|
+
* the better of two valid generations would be selection, and there is no code
|
|
34
|
+
* path here that could express it.
|
|
35
|
+
*
|
|
36
|
+
* ## Who decides escalation
|
|
37
|
+
*
|
|
38
|
+
* The **pathology of a deterministic refusal**, never the model's opinion of its
|
|
39
|
+
* own output. The first attempt runs on `reason_unknown`, whose ladder is
|
|
40
|
+
* exactly `['lite']` — *"escalating on a reason nobody established is spending
|
|
41
|
+
* on a guess"* (`_lib/evolution_roi.ts:136-138`). A refusal is then classified
|
|
42
|
+
* into a {@link PathologyWhy} by {@link classifyRefusal}, and the walk continues
|
|
43
|
+
* on THAT class's ladder from its cheapest untried rung. A class with an empty
|
|
44
|
+
* ladder stops the walk.
|
|
45
|
+
*
|
|
46
|
+
* Every attempt is recorded and the whole list is checked by
|
|
47
|
+
* `assertCheapestFirst` (`_lib/evolution_roi.ts:215`) before this module
|
|
48
|
+
* returns — the production caller AC-3 asks for.
|
|
49
|
+
*
|
|
50
|
+
* ## No transport lives here
|
|
51
|
+
*
|
|
52
|
+
* The metered call arrives as an injected {@link TextGenerator} port. The
|
|
53
|
+
* shipped binding is `_lib/llm_proposer_transport.ts`; this module imports it
|
|
54
|
+
* nowhere, so the arm is testable with a stub and the transport is one small
|
|
55
|
+
* file a reviewer can read whole.
|
|
56
|
+
*/
|
|
57
|
+
|
|
58
|
+
import {
|
|
59
|
+
RECIPES,
|
|
60
|
+
byteCompare,
|
|
61
|
+
candidateId,
|
|
62
|
+
type DefectObservation,
|
|
63
|
+
type SubjectReader,
|
|
64
|
+
} from './candidate_proposer.js';
|
|
65
|
+
import {
|
|
66
|
+
CANDIDATE_RECORD_VERSION,
|
|
67
|
+
CandidateSchemaError,
|
|
68
|
+
assertMutationPathsOwned,
|
|
69
|
+
type CandidateRecord,
|
|
70
|
+
type Mutation,
|
|
71
|
+
} from './candidate_record.js';
|
|
72
|
+
import {
|
|
73
|
+
assertCheapestFirst,
|
|
74
|
+
ladderFor,
|
|
75
|
+
nextTier,
|
|
76
|
+
type LadderAttempt,
|
|
77
|
+
type ModelTier,
|
|
78
|
+
} from './evolution_roi.js';
|
|
79
|
+
import type { PathologyWhy } from './pathology_archive.js';
|
|
80
|
+
|
|
81
|
+
type Assert<T extends true> = T;
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Keys that would turn a generation into a decision.
|
|
85
|
+
*
|
|
86
|
+
* The same mechanism `_lib/runtime_journal.ts`'s `FREE_FORM_KEYS` uses for the
|
|
87
|
+
* privacy floor, pointed at a different failure: there, a field that could hold
|
|
88
|
+
* content; here, a field that could hold a judgement. A closed deny-list is
|
|
89
|
+
* blunt on purpose — a guard that only rejects the keys someone thought of is a
|
|
90
|
+
* guard that has to be re-argued at every edit.
|
|
91
|
+
*/
|
|
92
|
+
export const DECISION_FIELD_KEYS = [
|
|
93
|
+
'score',
|
|
94
|
+
'scores',
|
|
95
|
+
'rank',
|
|
96
|
+
'ranking',
|
|
97
|
+
'rating',
|
|
98
|
+
'grade',
|
|
99
|
+
'confidence',
|
|
100
|
+
'probability',
|
|
101
|
+
'logprob',
|
|
102
|
+
'logprobs',
|
|
103
|
+
'best',
|
|
104
|
+
'chosen',
|
|
105
|
+
'selected',
|
|
106
|
+
'preferred',
|
|
107
|
+
'winner',
|
|
108
|
+
'verdict',
|
|
109
|
+
'order',
|
|
110
|
+
'priority',
|
|
111
|
+
'weight',
|
|
112
|
+
'quality',
|
|
113
|
+
] as const;
|
|
114
|
+
export type DecisionFieldKey = (typeof DECISION_FIELD_KEYS)[number];
|
|
115
|
+
|
|
116
|
+
/** `T` when `T` carries no {@link DecisionFieldKey}, and `never` when it does. */
|
|
117
|
+
export type NoDecisionField<T> = Extract<keyof T, DecisionFieldKey> extends never ? T : never;
|
|
118
|
+
|
|
119
|
+
// --- the port ----------------------------------------------------------------
|
|
120
|
+
|
|
121
|
+
/** One metered request. The tier is the ladder's, never the model's choice. */
|
|
122
|
+
export interface GenerationRequest {
|
|
123
|
+
readonly tier: ModelTier;
|
|
124
|
+
readonly system: string;
|
|
125
|
+
readonly prompt: string;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* What a metered call may return: text, and which model produced it.
|
|
130
|
+
*
|
|
131
|
+
* `model` is provenance and not a judgement — the protocol document needs it to
|
|
132
|
+
* record what was run. Nothing in this module reads it.
|
|
133
|
+
*/
|
|
134
|
+
export interface GenerationResult {
|
|
135
|
+
readonly text: string;
|
|
136
|
+
readonly model: string;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* The metered port. One request in, one text out.
|
|
141
|
+
*
|
|
142
|
+
* It cannot express a batch, so it cannot express a comparison: a signature
|
|
143
|
+
* taking `readonly GenerationRequest[]` and returning a sorted list would be a
|
|
144
|
+
* ranking surface even if today's implementation ignored the order.
|
|
145
|
+
*/
|
|
146
|
+
export type TextGenerator = (req: GenerationRequest) => Promise<GenerationResult>;
|
|
147
|
+
|
|
148
|
+
/** A generation the record contract refused. Carries WHY, as a pathology. */
|
|
149
|
+
export class GenerationRefusedError extends Error {
|
|
150
|
+
readonly why: PathologyWhy;
|
|
151
|
+
constructor(why: PathologyWhy, message: string) {
|
|
152
|
+
super(message);
|
|
153
|
+
this.name = 'GenerationRefusedError';
|
|
154
|
+
this.why = why;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** One observation produced no record and was not dropped. */
|
|
159
|
+
export class CandidateCountError extends Error {
|
|
160
|
+
constructor(message: string) {
|
|
161
|
+
super(message);
|
|
162
|
+
this.name = 'CandidateCountError';
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
// --- prompts -----------------------------------------------------------------
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* The system prompt. FROZEN — its bytes are part of the execution protocol
|
|
170
|
+
* (`docs/contracts/metered-proposer-protocol.md`), so an edit here is an edit to
|
|
171
|
+
* the protocol and invalidates a comparison already captured against it.
|
|
172
|
+
*/
|
|
173
|
+
export const SYSTEM_PROMPT =
|
|
174
|
+
'You rewrite one Markdown artifact to remove one named defect.\n' +
|
|
175
|
+
'Return ONLY the complete rewritten artifact body. No preamble, no fences,\n' +
|
|
176
|
+
'no commentary, no explanation of what you changed.\n' +
|
|
177
|
+
'Do not judge, score, rank, or compare anything. Produce one body.';
|
|
178
|
+
|
|
179
|
+
/** The user prompt for one observation. Pure — same inputs, same bytes. */
|
|
180
|
+
export function buildPrompt(obs: DefectObservation, body: string): string {
|
|
181
|
+
const recipe = RECIPES[obs.defectClass];
|
|
182
|
+
const route = obs.routeTo === undefined ? '' : `\nThe obligation must be routed to: ${obs.routeTo}\n`;
|
|
183
|
+
return (
|
|
184
|
+
`Defect class: ${obs.defectClass}\n` +
|
|
185
|
+
`What that means: ${recipe.summary}\n` +
|
|
186
|
+
`Artifact path: ${obs.subject}\n` +
|
|
187
|
+
route +
|
|
188
|
+
'\n--- BEGIN CURRENT BODY ---\n' +
|
|
189
|
+
body +
|
|
190
|
+
'\n--- END CURRENT BODY ---\n'
|
|
191
|
+
);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// --- the output contract ------------------------------------------------------
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* The largest generated body accepted, in bytes.
|
|
198
|
+
*
|
|
199
|
+
* A STATED CEILING, not a measured one, and it is a runaway guard rather than a
|
|
200
|
+
* quality bar: 256 KiB is far above any artifact in the candidate surface and
|
|
201
|
+
* far below a generation that has stopped producing an artifact. *Revisit-if* a
|
|
202
|
+
* legitimate subject approaches it.
|
|
203
|
+
*/
|
|
204
|
+
export const MAX_BODY_BYTES = 256 * 1024;
|
|
205
|
+
|
|
206
|
+
/** The NUL code unit, built rather than typed so no source file carries one. */
|
|
207
|
+
const NUL = String.fromCharCode(0);
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* Refuse a generation the record contract cannot accept.
|
|
211
|
+
*
|
|
212
|
+
* Every check here is a SHAPE check with an objective predicate, and none of
|
|
213
|
+
* them compares two generations or asks whether one is good. That distinction
|
|
214
|
+
* is the role constraint: `output_contract_violated` is a fact about one string,
|
|
215
|
+
* `better` is a judgement about two.
|
|
216
|
+
*
|
|
217
|
+
* @throws {GenerationRefusedError} carrying the pathology of the refusal.
|
|
218
|
+
*/
|
|
219
|
+
export function assertGenerationAcceptable(
|
|
220
|
+
obs: DefectObservation,
|
|
221
|
+
before: string,
|
|
222
|
+
after: string,
|
|
223
|
+
): void {
|
|
224
|
+
if (after.trim() === '') {
|
|
225
|
+
throw new GenerationRefusedError('output_contract_violated', 'the generation is empty');
|
|
226
|
+
}
|
|
227
|
+
const bytes = Buffer.byteLength(after, 'utf8');
|
|
228
|
+
if (bytes > MAX_BODY_BYTES) {
|
|
229
|
+
throw new GenerationRefusedError(
|
|
230
|
+
'output_contract_violated',
|
|
231
|
+
`the generation is ${String(bytes)} bytes, over the ${String(MAX_BODY_BYTES)}-byte ceiling`,
|
|
232
|
+
);
|
|
233
|
+
}
|
|
234
|
+
if (after.includes(NUL)) {
|
|
235
|
+
throw new GenerationRefusedError('output_contract_violated', 'the generation contains a NUL byte');
|
|
236
|
+
}
|
|
237
|
+
if (after === before) {
|
|
238
|
+
throw new GenerationRefusedError(
|
|
239
|
+
'output_contract_violated',
|
|
240
|
+
'the generation is byte-identical to the input — a proposal that changes nothing is not a proposal',
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
const recipe = RECIPES[obs.defectClass];
|
|
244
|
+
if (recipe.needsRouteTo && obs.routeTo !== undefined && !after.includes(obs.routeTo)) {
|
|
245
|
+
throw new GenerationRefusedError(
|
|
246
|
+
'output_contract_violated',
|
|
247
|
+
`the generation does not name the required route target '${obs.routeTo}'`,
|
|
248
|
+
);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* The pathology of a thrown failure.
|
|
254
|
+
*
|
|
255
|
+
* Deterministic and total. A refusal carries its own class; anything else is
|
|
256
|
+
* `execution_failed`, which is the honest reading of a transport or runtime
|
|
257
|
+
* error and licenses the full ladder because capability plausibly helps there.
|
|
258
|
+
*/
|
|
259
|
+
export function classifyRefusal(e: unknown): PathologyWhy {
|
|
260
|
+
if (e instanceof GenerationRefusedError) return e.why;
|
|
261
|
+
if (e instanceof CandidateSchemaError) return 'output_contract_violated';
|
|
262
|
+
return 'execution_failed';
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// --- the ladder walk -----------------------------------------------------------
|
|
266
|
+
|
|
267
|
+
/** The class the FIRST attempt runs on, before any refusal has been observed. */
|
|
268
|
+
export const INITIAL_CLASS: PathologyWhy = 'reason_unknown';
|
|
269
|
+
|
|
270
|
+
export interface MeteredProposal {
|
|
271
|
+
readonly records: readonly CandidateRecord[];
|
|
272
|
+
/** Every attempt, in the order it was made. The audit trail for AC-3. */
|
|
273
|
+
readonly attempts: readonly LadderAttempt[];
|
|
274
|
+
/** Which model produced each accepted record, by candidate id. Provenance only. */
|
|
275
|
+
readonly models: Readonly<Record<string, string>>;
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Walk the ladder for one observation until a generation is accepted.
|
|
280
|
+
*
|
|
281
|
+
* Throws when the ladder runs out — it never returns "no record", because a
|
|
282
|
+
* silently absent record is the filtering the role constraint forbids.
|
|
283
|
+
*/
|
|
284
|
+
async function proposeOne(
|
|
285
|
+
obs: DefectObservation,
|
|
286
|
+
before: string,
|
|
287
|
+
generate: TextGenerator,
|
|
288
|
+
attempts: LadderAttempt[],
|
|
289
|
+
seq: { n: number },
|
|
290
|
+
): Promise<{ record: CandidateRecord; model: string }> {
|
|
291
|
+
let cls: PathologyWhy = INITIAL_CLASS;
|
|
292
|
+
// Per OBSERVATION, deliberately. A new observation is a new proposal and
|
|
293
|
+
// starts at the cheapest rung again; the guard allows that explicitly — a
|
|
294
|
+
// repeat of an already-spent rung "is not an escalation"
|
|
295
|
+
// (`_lib/evolution_roi.ts:201-202`). Sharing one map across observations would
|
|
296
|
+
// exhaust the ladder on the second subject, which is what the first version
|
|
297
|
+
// of this function did and what its test caught.
|
|
298
|
+
const spentByClass = new Map<PathologyWhy, ModelTier[]>();
|
|
299
|
+
let lastError: unknown = null;
|
|
300
|
+
|
|
301
|
+
for (;;) {
|
|
302
|
+
const spent = spentByClass.get(cls) ?? [];
|
|
303
|
+
const tier = nextTier(cls, spent);
|
|
304
|
+
if (tier === null) {
|
|
305
|
+
// STOP. Either the ladder is empty for this class or every licensed
|
|
306
|
+
// rung is spent. `null` means stop in both cases — a caller reading
|
|
307
|
+
// it as "escalate" has inverted the policy.
|
|
308
|
+
throw new GenerationRefusedError(
|
|
309
|
+
cls,
|
|
310
|
+
`no tier left for '${obs.defectClass}' on ${obs.subject}: class '${cls}' licenses ` +
|
|
311
|
+
`${ladderFor(cls).join(' < ') || 'nothing'}` +
|
|
312
|
+
(lastError === null ? '' : ` — last refusal: ${(lastError as Error).message}`),
|
|
313
|
+
);
|
|
314
|
+
}
|
|
315
|
+
spent.push(tier);
|
|
316
|
+
spentByClass.set(cls, spent);
|
|
317
|
+
seq.n += 1;
|
|
318
|
+
attempts.push({ defect_class: cls, tier, sequence: seq.n });
|
|
319
|
+
|
|
320
|
+
try {
|
|
321
|
+
const gen = await generate({ tier, system: SYSTEM_PROMPT, prompt: buildPrompt(obs, before) });
|
|
322
|
+
assertGenerationAcceptable(obs, before, gen.text);
|
|
323
|
+
const mutations: Mutation[] = [{ path: obs.subject, content: gen.text }];
|
|
324
|
+
assertMutationPathsOwned(mutations);
|
|
325
|
+
return {
|
|
326
|
+
record: {
|
|
327
|
+
kind: 'candidate',
|
|
328
|
+
version: CANDIDATE_RECORD_VERSION,
|
|
329
|
+
// The SAME id function as the deterministic arm: a content
|
|
330
|
+
// hash over class, subject and mutation bytes. Two arms
|
|
331
|
+
// producing the same bytes therefore produce the same id,
|
|
332
|
+
// which is what makes a pair-wise comparison legible.
|
|
333
|
+
id: candidateId(obs.defectClass, obs.subject, mutations),
|
|
334
|
+
// The dimension comes from the RECIPE, never from the model.
|
|
335
|
+
// A model that could pick the dimension would be choosing
|
|
336
|
+
// which arm of the alphabet its own candidate sits on, and
|
|
337
|
+
// that is a decision.
|
|
338
|
+
dimension: RECIPES[obs.defectClass].dimension,
|
|
339
|
+
lifecycle: 'proposed',
|
|
340
|
+
mutations,
|
|
341
|
+
},
|
|
342
|
+
model: gen.model,
|
|
343
|
+
};
|
|
344
|
+
} catch (e) {
|
|
345
|
+
lastError = e;
|
|
346
|
+
cls = classifyRefusal(e);
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/**
|
|
352
|
+
* Propose one candidate per observation, using a metered generator.
|
|
353
|
+
*
|
|
354
|
+
* The metered twin of `proposeCandidates`. Ordering is fixed EXACTLY ONCE, on
|
|
355
|
+
* the input, with the deterministic arm's own comparator — so the two arms walk
|
|
356
|
+
* the same observations in the same order and a pair is a pair.
|
|
357
|
+
*
|
|
358
|
+
* `priorAttempts` carries what an earlier, budget-aborted run of the same corpus
|
|
359
|
+
* already spent, so the ordering guard sees the whole run rather than its tail.
|
|
360
|
+
* It is UNTRUSTED input and is validated against the ladder by the same
|
|
361
|
+
* `assertCheapestFirst` call the fresh path uses — which is what makes that
|
|
362
|
+
* guard's red producible from this function's own arguments.
|
|
363
|
+
*
|
|
364
|
+
* @throws {CandidateCountError} if the output would not be one record per
|
|
365
|
+
* observation. It is not reachable through this function's own logic, which is
|
|
366
|
+
* why it is asserted rather than trusted: the invariant is what makes "no
|
|
367
|
+
* filtering" checkable by a reader who does not want to re-derive the loop.
|
|
368
|
+
*/
|
|
369
|
+
export async function proposeCandidatesWithModel(
|
|
370
|
+
observations: readonly DefectObservation[],
|
|
371
|
+
read: SubjectReader,
|
|
372
|
+
generate: TextGenerator,
|
|
373
|
+
priorAttempts: readonly LadderAttempt[] = [],
|
|
374
|
+
): Promise<MeteredProposal> {
|
|
375
|
+
const ordered = [...observations].sort(
|
|
376
|
+
(a, b) => byteCompare(a.defectClass, b.defectClass) || byteCompare(a.subject, b.subject),
|
|
377
|
+
);
|
|
378
|
+
const records: CandidateRecord[] = [];
|
|
379
|
+
// A resumed run's history LEADS the attempt list. The budget guard ABORTS
|
|
380
|
+
// rather than truncating (`_lib/harness_evolution_guards.ts:140`), so a run
|
|
381
|
+
// that hits `max_spend_cents` stops mid-corpus; the resume passes the
|
|
382
|
+
// observations it has left plus what the aborted run already spent, and the
|
|
383
|
+
// ordering guard below then sees the whole run rather than its tail.
|
|
384
|
+
//
|
|
385
|
+
// It does NOT seed the per-observation ladder. Resume happens at the
|
|
386
|
+
// observation level — a completed observation is simply absent from the new
|
|
387
|
+
// input — and a fresh observation legitimately starts at the cheapest rung.
|
|
388
|
+
const attempts: LadderAttempt[] = [...priorAttempts];
|
|
389
|
+
const models: Record<string, string> = {};
|
|
390
|
+
const seq = { n: attempts.reduce((m, a) => Math.max(m, a.sequence), 0) };
|
|
391
|
+
|
|
392
|
+
for (const obs of ordered) {
|
|
393
|
+
const { record, model } = await proposeOne(obs, read(obs.subject), generate, attempts, seq);
|
|
394
|
+
records.push(record);
|
|
395
|
+
models[record.id] = model;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
if (records.length !== ordered.length) {
|
|
399
|
+
throw new CandidateCountError(
|
|
400
|
+
`${String(ordered.length)} observation(s) produced ${String(records.length)} record(s) — a ` +
|
|
401
|
+
'proposer that returns fewer records than observations has filtered, which is an evaluator role',
|
|
402
|
+
);
|
|
403
|
+
}
|
|
404
|
+
// AC-3's production caller, over a REAL population: the history a caller
|
|
405
|
+
// supplied plus what the walk above actually did.
|
|
406
|
+
//
|
|
407
|
+
// Its red is producible from this function's own inputs, which is the
|
|
408
|
+
// property that distinguishes a guard from a comment. Over an empty
|
|
409
|
+
// `priorAttempts` it cannot fire — `nextTier` per class makes an
|
|
410
|
+
// out-of-order sequence unconstructible — so on its own the walk would be
|
|
411
|
+
// exactly the un-provable guard this repository has caught before
|
|
412
|
+
// (`_lib/candidate_proposer.ts:343-347`, one ordering site, one guard, one
|
|
413
|
+
// red). What makes it falsifiable is the resumed-run path: a caller that
|
|
414
|
+
// hands back a history in which a costlier tier was spent before a cheaper
|
|
415
|
+
// one for the same class is refused here, and that is a real failure of a
|
|
416
|
+
// real resume rather than a hypothetical.
|
|
417
|
+
assertCheapestFirst(attempts);
|
|
418
|
+
return { records, attempts, models };
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
/**
|
|
422
|
+
* The attempt sequence a run WOULD make if every generation were accepted first
|
|
423
|
+
* try — one `lite` attempt per observation, on {@link INITIAL_CLASS}.
|
|
424
|
+
*
|
|
425
|
+
* The dry-run population, and it is not a fixture: it is derived from the real
|
|
426
|
+
* observations by the same `nextTier` call the live walk uses, so a dry run puts
|
|
427
|
+
* the guard over a real, non-empty attempt list without spending anything.
|
|
428
|
+
*/
|
|
429
|
+
export function plannedAttempts(observations: readonly DefectObservation[]): LadderAttempt[] {
|
|
430
|
+
const ordered = [...observations].sort(
|
|
431
|
+
(a, b) => byteCompare(a.defectClass, b.defectClass) || byteCompare(a.subject, b.subject),
|
|
432
|
+
);
|
|
433
|
+
const attempts: LadderAttempt[] = [];
|
|
434
|
+
let n = 0;
|
|
435
|
+
for (let i = 0; i < ordered.length; i += 1) {
|
|
436
|
+
const tier = nextTier(INITIAL_CLASS, []);
|
|
437
|
+
if (tier === null) {
|
|
438
|
+
throw new GenerationRefusedError(
|
|
439
|
+
INITIAL_CLASS,
|
|
440
|
+
`class '${INITIAL_CLASS}' licenses no tier, so no metered attempt is planned`,
|
|
441
|
+
);
|
|
442
|
+
}
|
|
443
|
+
n += 1;
|
|
444
|
+
attempts.push({ defect_class: INITIAL_CLASS, tier, sequence: n });
|
|
445
|
+
}
|
|
446
|
+
assertCheapestFirst(attempts);
|
|
447
|
+
return attempts;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* The role constraint, at compile time.
|
|
452
|
+
*
|
|
453
|
+
* Adding `score`, `rank`, `confidence` or any other {@link DecisionFieldKey} to
|
|
454
|
+
* the metered port's result type makes this `Assert<false>` and stops the build.
|
|
455
|
+
* A metered call that returns a judgement is an evaluator whatever the module is
|
|
456
|
+
* named, and this is the one place that can be made unavailable rather than
|
|
457
|
+
* merely discouraged.
|
|
458
|
+
*/
|
|
459
|
+
type _GenerationResultCarriesNoDecisionField = Assert<
|
|
460
|
+
[NoDecisionField<GenerationResult>] extends [never] ? false : true
|
|
461
|
+
>;
|