instar 1.3.708 → 1.3.709

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,34 @@
1
+ /**
2
+ * LLM Benchmark Coverage Map — INSTAR-Bench v2 ratchet #2.
3
+ *
4
+ * Every LLM-driven component (a key of COMPONENT_CATEGORY in
5
+ * src/core/componentCategories.ts) must map to exactly one of:
6
+ * - { task: '<bench-task-id>' } — covered by an INSTAR-Bench v2 task
7
+ * (task definitions live in the bench harness:
8
+ * research/llm-pathway-bench/instar-bench-v2/tasks/ on the benching
9
+ * agent; the id here is the durable contract),
10
+ * - { pending: '<wave>' } — authoring queued; the pending set is PINNED
11
+ * shrink-only in tests/unit/llm-bench-coverage-ratchet.test.ts (you can
12
+ * graduate an entry to covered, you cannot add one without editing the
13
+ * pinned test — a visible, reviewed act),
14
+ * - { exempt: '<argued reason>' } — a real argument why benching this
15
+ * component is not meaningful (also pinned shrink-only).
16
+ *
17
+ * WHY (operator directive 2026-07-02, topic 29723 + INSTAR-BENCH-V2-SPEC §6):
18
+ * adding an LLM callsite without benchmark coverage must fail the build —
19
+ * an unbenched decision-maker is an unmeasured one, and routing defaults are
20
+ * benchmark-derived. Structure > Willpower.
21
+ *
22
+ * Companion ratchet: the wiring test (componentCategories-evaluate-coverage)
23
+ * keeps COMPONENT_CATEGORY exhaustive over .evaluate() callsites; THIS map +
24
+ * its test keep bench coverage exhaustive over COMPONENT_CATEGORY.
25
+ */
26
+ export type BenchCoverage = {
27
+ task: string;
28
+ } | {
29
+ pending: 'wave-2' | 'wave-3';
30
+ } | {
31
+ exempt: string;
32
+ };
33
+ export declare const LLM_BENCH_COVERAGE: Readonly<Record<string, BenchCoverage>>;
34
+ //# sourceMappingURL=llmBenchCoverage.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"llmBenchCoverage.d.ts","sourceRoot":"","sources":["../../src/data/llmBenchCoverage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,OAAO,EAAE,QAAQ,GAAG,QAAQ,CAAA;CAAE,GAChC;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvB,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAiEtE,CAAC"}
@@ -0,0 +1,88 @@
1
+ /**
2
+ * LLM Benchmark Coverage Map — INSTAR-Bench v2 ratchet #2.
3
+ *
4
+ * Every LLM-driven component (a key of COMPONENT_CATEGORY in
5
+ * src/core/componentCategories.ts) must map to exactly one of:
6
+ * - { task: '<bench-task-id>' } — covered by an INSTAR-Bench v2 task
7
+ * (task definitions live in the bench harness:
8
+ * research/llm-pathway-bench/instar-bench-v2/tasks/ on the benching
9
+ * agent; the id here is the durable contract),
10
+ * - { pending: '<wave>' } — authoring queued; the pending set is PINNED
11
+ * shrink-only in tests/unit/llm-bench-coverage-ratchet.test.ts (you can
12
+ * graduate an entry to covered, you cannot add one without editing the
13
+ * pinned test — a visible, reviewed act),
14
+ * - { exempt: '<argued reason>' } — a real argument why benching this
15
+ * component is not meaningful (also pinned shrink-only).
16
+ *
17
+ * WHY (operator directive 2026-07-02, topic 29723 + INSTAR-BENCH-V2-SPEC §6):
18
+ * adding an LLM callsite without benchmark coverage must fail the build —
19
+ * an unbenched decision-maker is an unmeasured one, and routing defaults are
20
+ * benchmark-derived. Structure > Willpower.
21
+ *
22
+ * Companion ratchet: the wiring test (componentCategories-evaluate-coverage)
23
+ * keeps COMPONENT_CATEGORY exhaustive over .evaluate() callsites; THIS map +
24
+ * its test keep bench coverage exhaustive over COMPONENT_CATEGORY.
25
+ */
26
+ export const LLM_BENCH_COVERAGE = {
27
+ // ── Covered by the critical set (Wave 1 — authored 2026-07-02) ──
28
+ MessageSentinel: { task: 'sentinel-classify' },
29
+ MessagingToneGate: { task: 'tone-gate' },
30
+ // CompletionEvaluator has TWO judged surfaces; both benched.
31
+ CompletionEvaluator: { task: 'completion-judge+p13-stop-judge' },
32
+ ExternalOperationGate: { task: 'external-op-gate' },
33
+ LLMSanitizer: { task: 'injection-sanitizer' },
34
+ WarrantsReplyGate: { task: 'warrants-reply' },
35
+ InputClassifier: { task: 'input-classifier' },
36
+ Usher: { task: 'usher' },
37
+ 'correction-learning': { task: 'correction-distiller' },
38
+ CoherenceReviewer: { task: 'gate-triage' },
39
+ // ── Argued exemptions (pinned; each must carry a real reason) ──
40
+ InteractivePoolCanaryJudge: {
41
+ exempt: 'judges a FIXED known-answer canary probe — the canary is its own benchmark; a bench task would re-test the same constant',
42
+ },
43
+ // ── Wave 2 (sentinels/gates/extractors still to author) ──
44
+ InputDetector: { pending: 'wave-2' },
45
+ InputGuard: { pending: 'wave-2' },
46
+ SessionActivitySentinel: { pending: 'wave-2' },
47
+ StallTriageNurse: { pending: 'wave-2' },
48
+ CommitmentSentinel: { pending: 'wave-2' },
49
+ PresenceProxy: { pending: 'wave-2' },
50
+ PromiseBeacon: { pending: 'wave-2' },
51
+ ProjectDriftChecker: { pending: 'wave-2' },
52
+ TemporalCoherenceChecker: { pending: 'wave-2' },
53
+ SessionWatchdog: { pending: 'wave-2' },
54
+ ResumeQueueDrainer: { pending: 'wave-2' },
55
+ TopicIntentArcCheck: { pending: 'wave-2' },
56
+ SlackAdapter: { pending: 'wave-2' },
57
+ TelegramAdapter: { pending: 'wave-2' },
58
+ PromptGate: { pending: 'wave-2' },
59
+ AutoApprover: { pending: 'wave-2' },
60
+ IntegrationGate: { pending: 'wave-2' },
61
+ UnjustifiedStopGate: { pending: 'wave-2' },
62
+ CoherenceGate: { pending: 'wave-2' },
63
+ OverrideDetector: { pending: 'wave-2' },
64
+ TaskClassifier: { pending: 'wave-2' },
65
+ ResumeValidator: { pending: 'wave-2' },
66
+ SessionSummarySentinel: { pending: 'wave-2' },
67
+ TopicIntentExtractor: { pending: 'wave-2' },
68
+ // ── Wave 3 (reflectors + background/job tasks) ──
69
+ JobReflector: { pending: 'wave-3' },
70
+ crossModelReviewer: { pending: 'wave-3' },
71
+ SelfKnowledgeTree: { pending: 'wave-3' },
72
+ TreeTriage: { pending: 'wave-3' },
73
+ TopicSummarizer: { pending: 'wave-3' },
74
+ ContextualEvaluator: { pending: 'wave-3' },
75
+ RelationshipManager: { pending: 'wave-3' },
76
+ StandardsConformanceReviewer: { pending: 'wave-3' },
77
+ DiscoveryEvaluator: { pending: 'wave-3' },
78
+ PipeSessionSpawner: { pending: 'wave-3' },
79
+ CartographerSweep: { pending: 'wave-3' },
80
+ StandardsCoverageEnrichment: { pending: 'wave-3' },
81
+ PreCompactionFlush: { pending: 'wave-3' },
82
+ TreeSynthesis: { pending: 'wave-3' },
83
+ LLMConflictResolver: { pending: 'wave-3' },
84
+ openConversationBrief: { pending: 'wave-3' },
85
+ 'a2a-checkin': { pending: 'wave-3' },
86
+ 'mentor-stage-b': { pending: 'wave-3' },
87
+ };
88
+ //# sourceMappingURL=llmBenchCoverage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"llmBenchCoverage.js","sourceRoot":"","sources":["../../src/data/llmBenchCoverage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAOH,MAAM,CAAC,MAAM,kBAAkB,GAA4C;IACzE,mEAAmE;IACnE,eAAe,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE;IAC9C,iBAAiB,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;IACxC,6DAA6D;IAC7D,mBAAmB,EAAE,EAAE,IAAI,EAAE,iCAAiC,EAAE;IAChE,qBAAqB,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACnD,YAAY,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE;IAC7C,iBAAiB,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;IAC7C,eAAe,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE;IAC7C,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;IACxB,qBAAqB,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;IACvD,iBAAiB,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;IAE1C,kEAAkE;IAClE,0BAA0B,EAAE;QAC1B,MAAM,EACJ,0HAA0H;KAC7H;IAED,4DAA4D;IAC5D,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,UAAU,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACjC,uBAAuB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC9C,gBAAgB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACvC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,wBAAwB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC/C,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,YAAY,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnC,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,UAAU,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACjC,YAAY,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnC,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,gBAAgB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACvC,cAAc,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACrC,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,sBAAsB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC7C,oBAAoB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAE3C,mDAAmD;IACnD,YAAY,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,iBAAiB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACxC,UAAU,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACjC,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,4BAA4B,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnD,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,iBAAiB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACxC,2BAA2B,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAClD,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,qBAAqB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC5C,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,gBAAgB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;CACxC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "instar",
3
- "version": "1.3.708",
3
+ "version": "1.3.709",
4
4
  "description": "Coherence infrastructure for self-evolving AI agents — on the Claude Code or Codex subscription you already have.",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "$schema": "./builtin-manifest.schema.json",
3
3
  "schemaVersion": 1,
4
- "generatedAt": "2026-07-02T08:43:26.009Z",
5
- "instarVersion": "1.3.708",
4
+ "generatedAt": "2026-07-02T08:48:48.771Z",
5
+ "instarVersion": "1.3.709",
6
6
  "entryCount": 202,
7
7
  "entries": {
8
8
  "hook:session-start": {
@@ -0,0 +1,97 @@
1
+ /**
2
+ * LLM Benchmark Coverage Map — INSTAR-Bench v2 ratchet #2.
3
+ *
4
+ * Every LLM-driven component (a key of COMPONENT_CATEGORY in
5
+ * src/core/componentCategories.ts) must map to exactly one of:
6
+ * - { task: '<bench-task-id>' } — covered by an INSTAR-Bench v2 task
7
+ * (task definitions live in the bench harness:
8
+ * research/llm-pathway-bench/instar-bench-v2/tasks/ on the benching
9
+ * agent; the id here is the durable contract),
10
+ * - { pending: '<wave>' } — authoring queued; the pending set is PINNED
11
+ * shrink-only in tests/unit/llm-bench-coverage-ratchet.test.ts (you can
12
+ * graduate an entry to covered, you cannot add one without editing the
13
+ * pinned test — a visible, reviewed act),
14
+ * - { exempt: '<argued reason>' } — a real argument why benching this
15
+ * component is not meaningful (also pinned shrink-only).
16
+ *
17
+ * WHY (operator directive 2026-07-02, topic 29723 + INSTAR-BENCH-V2-SPEC §6):
18
+ * adding an LLM callsite without benchmark coverage must fail the build —
19
+ * an unbenched decision-maker is an unmeasured one, and routing defaults are
20
+ * benchmark-derived. Structure > Willpower.
21
+ *
22
+ * Companion ratchet: the wiring test (componentCategories-evaluate-coverage)
23
+ * keeps COMPONENT_CATEGORY exhaustive over .evaluate() callsites; THIS map +
24
+ * its test keep bench coverage exhaustive over COMPONENT_CATEGORY.
25
+ */
26
+
27
+ export type BenchCoverage =
28
+ | { task: string }
29
+ | { pending: 'wave-2' | 'wave-3' }
30
+ | { exempt: string };
31
+
32
+ export const LLM_BENCH_COVERAGE: Readonly<Record<string, BenchCoverage>> = {
33
+ // ── Covered by the critical set (Wave 1 — authored 2026-07-02) ──
34
+ MessageSentinel: { task: 'sentinel-classify' },
35
+ MessagingToneGate: { task: 'tone-gate' },
36
+ // CompletionEvaluator has TWO judged surfaces; both benched.
37
+ CompletionEvaluator: { task: 'completion-judge+p13-stop-judge' },
38
+ ExternalOperationGate: { task: 'external-op-gate' },
39
+ LLMSanitizer: { task: 'injection-sanitizer' },
40
+ WarrantsReplyGate: { task: 'warrants-reply' },
41
+ InputClassifier: { task: 'input-classifier' },
42
+ Usher: { task: 'usher' },
43
+ 'correction-learning': { task: 'correction-distiller' },
44
+ CoherenceReviewer: { task: 'gate-triage' },
45
+
46
+ // ── Argued exemptions (pinned; each must carry a real reason) ──
47
+ InteractivePoolCanaryJudge: {
48
+ exempt:
49
+ 'judges a FIXED known-answer canary probe — the canary is its own benchmark; a bench task would re-test the same constant',
50
+ },
51
+
52
+ // ── Wave 2 (sentinels/gates/extractors still to author) ──
53
+ InputDetector: { pending: 'wave-2' },
54
+ InputGuard: { pending: 'wave-2' },
55
+ SessionActivitySentinel: { pending: 'wave-2' },
56
+ StallTriageNurse: { pending: 'wave-2' },
57
+ CommitmentSentinel: { pending: 'wave-2' },
58
+ PresenceProxy: { pending: 'wave-2' },
59
+ PromiseBeacon: { pending: 'wave-2' },
60
+ ProjectDriftChecker: { pending: 'wave-2' },
61
+ TemporalCoherenceChecker: { pending: 'wave-2' },
62
+ SessionWatchdog: { pending: 'wave-2' },
63
+ ResumeQueueDrainer: { pending: 'wave-2' },
64
+ TopicIntentArcCheck: { pending: 'wave-2' },
65
+ SlackAdapter: { pending: 'wave-2' },
66
+ TelegramAdapter: { pending: 'wave-2' },
67
+ PromptGate: { pending: 'wave-2' },
68
+ AutoApprover: { pending: 'wave-2' },
69
+ IntegrationGate: { pending: 'wave-2' },
70
+ UnjustifiedStopGate: { pending: 'wave-2' },
71
+ CoherenceGate: { pending: 'wave-2' },
72
+ OverrideDetector: { pending: 'wave-2' },
73
+ TaskClassifier: { pending: 'wave-2' },
74
+ ResumeValidator: { pending: 'wave-2' },
75
+ SessionSummarySentinel: { pending: 'wave-2' },
76
+ TopicIntentExtractor: { pending: 'wave-2' },
77
+
78
+ // ── Wave 3 (reflectors + background/job tasks) ──
79
+ JobReflector: { pending: 'wave-3' },
80
+ crossModelReviewer: { pending: 'wave-3' },
81
+ SelfKnowledgeTree: { pending: 'wave-3' },
82
+ TreeTriage: { pending: 'wave-3' },
83
+ TopicSummarizer: { pending: 'wave-3' },
84
+ ContextualEvaluator: { pending: 'wave-3' },
85
+ RelationshipManager: { pending: 'wave-3' },
86
+ StandardsConformanceReviewer: { pending: 'wave-3' },
87
+ DiscoveryEvaluator: { pending: 'wave-3' },
88
+ PipeSessionSpawner: { pending: 'wave-3' },
89
+ CartographerSweep: { pending: 'wave-3' },
90
+ StandardsCoverageEnrichment: { pending: 'wave-3' },
91
+ PreCompactionFlush: { pending: 'wave-3' },
92
+ TreeSynthesis: { pending: 'wave-3' },
93
+ LLMConflictResolver: { pending: 'wave-3' },
94
+ openConversationBrief: { pending: 'wave-3' },
95
+ 'a2a-checkin': { pending: 'wave-3' },
96
+ 'mentor-stage-b': { pending: 'wave-3' },
97
+ };
@@ -0,0 +1,42 @@
1
+ # Upgrade Guide — vNEXT
2
+
3
+ <!-- assembled-by: assemble-next-md -->
4
+ <!-- bump: patch -->
5
+
6
+ ## What Changed
7
+
8
+ INSTAR-Bench v2 ratchet #2 (spec §6; operator directive 2026-07-02, topic
9
+ 29723): a new coverage ledger (`src/data/llmBenchCoverage.ts`) maps every LLM
10
+ component in `COMPONENT_CATEGORY` to its benchmark status — covered by a
11
+ named bench task (the 10 Wave-1 critical gates/sentinels), pending (42,
12
+ pinned shrink-only), or exempt (1, argued). A CI test
13
+ (`llm-bench-coverage-ratchet.test.ts`) fails the build when a new LLM
14
+ component ships without a coverage decision, when the pending/exempt sets
15
+ GROW (graduating shrinks them; adding requires editing the pinned baseline in
16
+ the test — a visible, reviewed act), when an exemption's argument is lazy, or
17
+ when a Wave-1 critical component slides back to uncovered.
18
+
19
+ Together with the existing attribution ratchet + category-map wiring test,
20
+ the chain is closed: every LLM callsite must be attributed → categorized/
21
+ routed deliberately → benchmark-covered. No runtime surface; build-time only.
22
+
23
+ ## What to Tell Your User
24
+
25
+ Nothing proactively — no behavior changes. If asked: every one of my internal
26
+ AI helpers is now tracked against a benchmark coverage ledger, so any new
27
+ helper added to my infrastructure must ship with quality measurements (or a
28
+ written argument why not) before the build goes green. It keeps my model
29
+ choices grounded in measured results instead of guesses.
30
+
31
+ ## Summary of New Capabilities
32
+
33
+ None user-facing. New build-time guarantee: `src/data/llmBenchCoverage.ts`
34
+ (the coverage ledger) + `llm-bench-coverage-ratchet.test.ts` (the CI ratchet).
35
+ The ledger is importable by future tooling (e.g. the bench runner's coverage
36
+ report) but nothing at runtime consumes it yet.
37
+
38
+ ## Evidence
39
+
40
+ 6 new tests green (completeness, no-dangling, shrink-only pins ×2, argued
41
+ exemptions, Wave-1 regression pin); `tsc --noEmit` clean; full unit suite
42
+ green at push.
@@ -0,0 +1,11 @@
1
+ # New AI decision-makers now have to bring a test report card
2
+
3
+ Instar has about fifty little AI helpers inside it — one classifies your messages, one reviews outgoing replies, one judges whether an autonomous work session really finished, and so on. We just started benchmarking these helpers properly: for each one, a battery of hard test cases that tells us which AI model does that specific job well, fast, and cheap. Those measurements are what our model-routing choices will be based on going forward.
4
+
5
+ But a benchmark only stays trustworthy if it stays complete. The failure mode is silent: someone adds a NEW AI helper next month, nobody writes test cases for it, and suddenly there's an unmeasured decision-maker running in production whose model was picked by vibes. Six months later there are ten of those and the benchmark is a museum piece.
6
+
7
+ This change makes that impossible — not by policy, by build failure. There's now a coverage ledger listing every AI helper and its benchmark status: covered (with the name of its test battery), queued for authoring (an explicitly pinned to-do list), or exempt (with a written argument why testing it makes no sense — and "n/a" is literally rejected as too short). A test in the build system cross-checks that ledger against the master list of AI helpers. Add a new helper without deciding its benchmark story, and the build goes red with instructions telling you exactly what to do.
8
+
9
+ The teeth are in the details: the "queued" list can only shrink. You can graduate an entry to covered, but adding a new name to the queue requires editing the pinned list inside the test itself — a change any reviewer will see and question. Same for exemptions. So the lazy path (quietly park it in the queue forever) is exactly as visible as the honest path (write the test cases). Ten of the most critical helpers — the message classifier, the outbound-message gate, the completion judge, the injection screener, and their peers — are additionally pinned as "may never slide back": once covered, always covered.
10
+
11
+ This is the second ratchet of its kind. The first (already live) forces every new AI call site to register in the routing map so we know which provider runs it. Together they close the loop: every AI decision-maker in instar must be routed deliberately AND measured before it ships. Nothing changes at runtime — this is purely a build-time guarantee, and it costs nothing unless someone tries to skip the homework.
@@ -0,0 +1,41 @@
1
+ # Side-Effects Review — LLM benchmark-coverage ratchet (INSTAR-Bench v2 §6, ratchet #2)
2
+
3
+ **Version / slug:** `llm-bench-coverage-ratchet`
4
+ **Date:** `2026-07-02`
5
+ **Author:** `Instar Agent (echo)`
6
+ **Second-pass reviewer:** `not required` (build-time test + data map only; zero runtime surface)
7
+
8
+ ## Summary of the change
9
+
10
+ Adds `src/data/llmBenchCoverage.ts` — a coverage ledger mapping every LLM component (every key of `COMPONENT_CATEGORY`) to its INSTAR-Bench v2 status: `{task}` covered (10 Wave-1 critical components), `{pending}` queued (42, pinned), or `{exempt}` argued (1) — and `tests/unit/llm-bench-coverage-ratchet.test.ts`, which fails CI when a new LLM component ships without a coverage decision, when the pending/exempt sets grow (shrink-only pins), when an exemption's argument is lazy (<40 chars), or when a Wave-1 critical component slides back to uncovered. Operator directive 2026-07-02 (topic 29723) + INSTAR-BENCH-V2-SPEC §6. No runtime code is touched.
11
+
12
+ ## Decision-point inventory
13
+
14
+ None at runtime. The only "decision" is a CI pass/fail on repository content — a build-time ratchet identical in class to the existing `llm-attribution-ratchet`.
15
+
16
+ ## 1. Over-block
17
+ The ratchet can block a COMMIT (not a message/action): a developer adding an LLM component who genuinely cannot author bench cases yet must add the component to the pending baseline inside the test — a deliberate, visible act. That is the intended cost, not an over-block; there is no path it rejects that it shouldn't (an argued exemption is always available).
18
+
19
+ ## 2. Under-block
20
+ Honest limits: (a) the ratchet keys off `COMPONENT_CATEGORY` — an LLM callsite that dodges the category map would also dodge this map, but the companion evaluate-coverage wiring test + attribution lint close that hole upstream; (b) it cannot verify the bench task file CONTENT (those live in the bench harness on the benching agent) — the task id is a contract, and a wrong/empty id would surface at the next bench run, not in CI. Accepted for v1 of the ratchet; noted in the map docblock.
21
+
22
+ ## 3. Level-of-abstraction fit
23
+ Right layer: repository data + unit test, the established ratchet pattern (`llm-attribution-ratchet`, `WIRING_EXCLUSIONS` pins). Extends the existing chain (callsite → category map → THIS → bench coverage) rather than inventing a parallel mechanism.
24
+
25
+ ## 4. Signal vs authority compliance
26
+ Compliant. No runtime authority; a CI test is deterministic, cheap, and its "block" is a build failure with instructions. (docs/signal-vs-authority.md concerns runtime decision points; none added.)
27
+
28
+ ## 5. Interactions
29
+ Complements `llm-attribution-ratchet` (attribution presence), `componentCategories` wiring test (map exhaustiveness), and the LLM Routing Registry doc. No double-fire: each guards a different link of the chain. No runtime interaction possible.
30
+
31
+ ## 6. External surfaces
32
+ None. Not user-visible; no API, no message, no config.
33
+
34
+ ## 7. Multi-machine posture (Cross-Machine Coherence)
35
+ Not applicable — repository content evaluated in CI; identical on every machine by construction (git).
36
+
37
+ ## 8. Rollback cost
38
+ Delete the test + map (one revert). No state, no migration, no fleet impact — the ratchet only ever constrained future commits.
39
+
40
+ ## Test evidence
41
+ `llm-bench-coverage-ratchet.test.ts`: 6 tests — completeness over COMPONENT_CATEGORY, no dangling entries, pending pinned shrink-only, exemptions pinned + argued, non-empty task ids, Wave-1 regression pin. All green; `tsc --noEmit` clean.