instar 1.3.708 → 1.3.710

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,34 @@
1
+ /**
2
+ * LLM Benchmark Coverage Map — INSTAR-Bench v2 ratchet #2.
3
+ *
4
+ * Every LLM-driven component (a key of COMPONENT_CATEGORY in
5
+ * src/core/componentCategories.ts) must map to exactly one of:
6
+ * - { task: '<bench-task-id>' } — covered by an INSTAR-Bench v2 task
7
+ * (task definitions live in the bench harness:
8
+ * research/llm-pathway-bench/instar-bench-v2/tasks/ on the benching
9
+ * agent; the id here is the durable contract),
10
+ * - { pending: '<wave>' } — authoring queued; the pending set is PINNED
11
+ * shrink-only in tests/unit/llm-bench-coverage-ratchet.test.ts (you can
12
+ * graduate an entry to covered, you cannot add one without editing the
13
+ * pinned test — a visible, reviewed act),
14
+ * - { exempt: '<argued reason>' } — a real argument why benching this
15
+ * component is not meaningful (also pinned shrink-only).
16
+ *
17
+ * WHY (operator directive 2026-07-02, topic 29723 + INSTAR-BENCH-V2-SPEC §6):
18
+ * adding an LLM callsite without benchmark coverage must fail the build —
19
+ * an unbenched decision-maker is an unmeasured one, and routing defaults are
20
+ * benchmark-derived. Structure > Willpower.
21
+ *
22
+ * Companion ratchet: the wiring test (componentCategories-evaluate-coverage)
23
+ * keeps COMPONENT_CATEGORY exhaustive over .evaluate() callsites; THIS map +
24
+ * its test keep bench coverage exhaustive over COMPONENT_CATEGORY.
25
+ */
26
+ export type BenchCoverage = {
27
+ task: string;
28
+ } | {
29
+ pending: 'wave-2' | 'wave-3';
30
+ } | {
31
+ exempt: string;
32
+ };
33
+ export declare const LLM_BENCH_COVERAGE: Readonly<Record<string, BenchCoverage>>;
34
+ //# sourceMappingURL=llmBenchCoverage.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"llmBenchCoverage.d.ts","sourceRoot":"","sources":["../../src/data/llmBenchCoverage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,OAAO,EAAE,QAAQ,GAAG,QAAQ,CAAA;CAAE,GAChC;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvB,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAiEtE,CAAC"}
@@ -0,0 +1,88 @@
1
+ /**
2
+ * LLM Benchmark Coverage Map — INSTAR-Bench v2 ratchet #2.
3
+ *
4
+ * Every LLM-driven component (a key of COMPONENT_CATEGORY in
5
+ * src/core/componentCategories.ts) must map to exactly one of:
6
+ * - { task: '<bench-task-id>' } — covered by an INSTAR-Bench v2 task
7
+ * (task definitions live in the bench harness:
8
+ * research/llm-pathway-bench/instar-bench-v2/tasks/ on the benching
9
+ * agent; the id here is the durable contract),
10
+ * - { pending: '<wave>' } — authoring queued; the pending set is PINNED
11
+ * shrink-only in tests/unit/llm-bench-coverage-ratchet.test.ts (you can
12
+ * graduate an entry to covered, you cannot add one without editing the
13
+ * pinned test — a visible, reviewed act),
14
+ * - { exempt: '<argued reason>' } — a real argument why benching this
15
+ * component is not meaningful (also pinned shrink-only).
16
+ *
17
+ * WHY (operator directive 2026-07-02, topic 29723 + INSTAR-BENCH-V2-SPEC §6):
18
+ * adding an LLM callsite without benchmark coverage must fail the build —
19
+ * an unbenched decision-maker is an unmeasured one, and routing defaults are
20
+ * benchmark-derived. Structure > Willpower.
21
+ *
22
+ * Companion ratchet: the wiring test (componentCategories-evaluate-coverage)
23
+ * keeps COMPONENT_CATEGORY exhaustive over .evaluate() callsites; THIS map +
24
+ * its test keep bench coverage exhaustive over COMPONENT_CATEGORY.
25
+ */
26
+ export const LLM_BENCH_COVERAGE = {
27
+ // ── Covered by the critical set (Wave 1 — authored 2026-07-02) ──
28
+ MessageSentinel: { task: 'sentinel-classify' },
29
+ MessagingToneGate: { task: 'tone-gate' },
30
+ // CompletionEvaluator has TWO judged surfaces; both benched.
31
+ CompletionEvaluator: { task: 'completion-judge+p13-stop-judge' },
32
+ ExternalOperationGate: { task: 'external-op-gate' },
33
+ LLMSanitizer: { task: 'injection-sanitizer' },
34
+ WarrantsReplyGate: { task: 'warrants-reply' },
35
+ InputClassifier: { task: 'input-classifier' },
36
+ Usher: { task: 'usher' },
37
+ 'correction-learning': { task: 'correction-distiller' },
38
+ CoherenceReviewer: { task: 'gate-triage' },
39
+ // ── Argued exemptions (pinned; each must carry a real reason) ──
40
+ InteractivePoolCanaryJudge: {
41
+ exempt: 'judges a FIXED known-answer canary probe — the canary is its own benchmark; a bench task would re-test the same constant',
42
+ },
43
+ // ── Wave 2 (sentinels/gates/extractors still to author) ──
44
+ InputDetector: { pending: 'wave-2' },
45
+ InputGuard: { pending: 'wave-2' },
46
+ SessionActivitySentinel: { pending: 'wave-2' },
47
+ StallTriageNurse: { pending: 'wave-2' },
48
+ CommitmentSentinel: { pending: 'wave-2' },
49
+ PresenceProxy: { pending: 'wave-2' },
50
+ PromiseBeacon: { pending: 'wave-2' },
51
+ ProjectDriftChecker: { pending: 'wave-2' },
52
+ TemporalCoherenceChecker: { pending: 'wave-2' },
53
+ SessionWatchdog: { pending: 'wave-2' },
54
+ ResumeQueueDrainer: { pending: 'wave-2' },
55
+ TopicIntentArcCheck: { pending: 'wave-2' },
56
+ SlackAdapter: { pending: 'wave-2' },
57
+ TelegramAdapter: { pending: 'wave-2' },
58
+ PromptGate: { pending: 'wave-2' },
59
+ AutoApprover: { pending: 'wave-2' },
60
+ IntegrationGate: { pending: 'wave-2' },
61
+ UnjustifiedStopGate: { pending: 'wave-2' },
62
+ CoherenceGate: { pending: 'wave-2' },
63
+ OverrideDetector: { pending: 'wave-2' },
64
+ TaskClassifier: { pending: 'wave-2' },
65
+ ResumeValidator: { pending: 'wave-2' },
66
+ SessionSummarySentinel: { pending: 'wave-2' },
67
+ TopicIntentExtractor: { pending: 'wave-2' },
68
+ // ── Wave 3 (reflectors + background/job tasks) ──
69
+ JobReflector: { pending: 'wave-3' },
70
+ crossModelReviewer: { pending: 'wave-3' },
71
+ SelfKnowledgeTree: { pending: 'wave-3' },
72
+ TreeTriage: { pending: 'wave-3' },
73
+ TopicSummarizer: { pending: 'wave-3' },
74
+ ContextualEvaluator: { pending: 'wave-3' },
75
+ RelationshipManager: { pending: 'wave-3' },
76
+ StandardsConformanceReviewer: { pending: 'wave-3' },
77
+ DiscoveryEvaluator: { pending: 'wave-3' },
78
+ PipeSessionSpawner: { pending: 'wave-3' },
79
+ CartographerSweep: { pending: 'wave-3' },
80
+ StandardsCoverageEnrichment: { pending: 'wave-3' },
81
+ PreCompactionFlush: { pending: 'wave-3' },
82
+ TreeSynthesis: { pending: 'wave-3' },
83
+ LLMConflictResolver: { pending: 'wave-3' },
84
+ openConversationBrief: { pending: 'wave-3' },
85
+ 'a2a-checkin': { pending: 'wave-3' },
86
+ 'mentor-stage-b': { pending: 'wave-3' },
87
+ };
88
+ //# sourceMappingURL=llmBenchCoverage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"llmBenchCoverage.js","sourceRoot":"","sources":["../../src/data/llmBenchCoverage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAOH,MAAM,CAAC,MAAM,kBAAkB,GAA4C;IACzE,mEAAmE;IACnE,eAAe,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE;IAC9C,iBAAiB,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;IACxC,6DAA6D;IAC7D,mBAAmB,EAAE,EAAE,IAAI,EAAE,iCAAiC,EAAE;IAChE,qBAAqB,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACnD,YAAY,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE;IAC7C,iBAAiB,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;IAC7C,eAAe,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE;IAC7C,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;IACxB,qBAAqB,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;IACvD,iBAAiB,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;IAE1C,kEAAkE;IAClE,0BAA0B,EAAE;QAC1B,MAAM,EACJ,0HAA0H;KAC7H;IAED,4DAA4D;IAC5D,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,UAAU,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACjC,uBAAuB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC9C,gBAAgB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACvC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,wBAAwB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC/C,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,YAAY,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnC,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,UAAU,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACjC,YAAY,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnC,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,gBAAgB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACvC,cAAc,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACrC,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,sBAAsB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC7C,oBAAoB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAE3C,mDAAmD;IACnD,YAAY,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,iBAAiB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACxC,UAAU,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACjC,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,4BAA4B,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnD,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,iBAAiB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACxC,2BAA2B,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAClD,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,qBAAqB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC5C,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,gBAAgB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;CACxC,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "instar",
3
- "version": "1.3.708",
3
+ "version": "1.3.710",
4
4
  "description": "Coherence infrastructure for self-evolving AI agents — on the Claude Code or Codex subscription you already have.",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "$schema": "./builtin-manifest.schema.json",
3
3
  "schemaVersion": 1,
4
- "generatedAt": "2026-07-02T08:43:26.009Z",
5
- "instarVersion": "1.3.708",
4
+ "generatedAt": "2026-07-02T09:05:31.088Z",
5
+ "instarVersion": "1.3.710",
6
6
  "entryCount": 202,
7
7
  "entries": {
8
8
  "hook:session-start": {
@@ -0,0 +1,97 @@
1
+ /**
2
+ * LLM Benchmark Coverage Map — INSTAR-Bench v2 ratchet #2.
3
+ *
4
+ * Every LLM-driven component (a key of COMPONENT_CATEGORY in
5
+ * src/core/componentCategories.ts) must map to exactly one of:
6
+ * - { task: '<bench-task-id>' } — covered by an INSTAR-Bench v2 task
7
+ * (task definitions live in the bench harness:
8
+ * research/llm-pathway-bench/instar-bench-v2/tasks/ on the benching
9
+ * agent; the id here is the durable contract),
10
+ * - { pending: '<wave>' } — authoring queued; the pending set is PINNED
11
+ * shrink-only in tests/unit/llm-bench-coverage-ratchet.test.ts (you can
12
+ * graduate an entry to covered, you cannot add one without editing the
13
+ * pinned test — a visible, reviewed act),
14
+ * - { exempt: '<argued reason>' } — a real argument why benching this
15
+ * component is not meaningful (also pinned shrink-only).
16
+ *
17
+ * WHY (operator directive 2026-07-02, topic 29723 + INSTAR-BENCH-V2-SPEC §6):
18
+ * adding an LLM callsite without benchmark coverage must fail the build —
19
+ * an unbenched decision-maker is an unmeasured one, and routing defaults are
20
+ * benchmark-derived. Structure > Willpower.
21
+ *
22
+ * Companion ratchet: the wiring test (componentCategories-evaluate-coverage)
23
+ * keeps COMPONENT_CATEGORY exhaustive over .evaluate() callsites; THIS map +
24
+ * its test keep bench coverage exhaustive over COMPONENT_CATEGORY.
25
+ */
26
+
27
+ export type BenchCoverage =
28
+ | { task: string }
29
+ | { pending: 'wave-2' | 'wave-3' }
30
+ | { exempt: string };
31
+
32
+ export const LLM_BENCH_COVERAGE: Readonly<Record<string, BenchCoverage>> = {
33
+ // ── Covered by the critical set (Wave 1 — authored 2026-07-02) ──
34
+ MessageSentinel: { task: 'sentinel-classify' },
35
+ MessagingToneGate: { task: 'tone-gate' },
36
+ // CompletionEvaluator has TWO judged surfaces; both benched.
37
+ CompletionEvaluator: { task: 'completion-judge+p13-stop-judge' },
38
+ ExternalOperationGate: { task: 'external-op-gate' },
39
+ LLMSanitizer: { task: 'injection-sanitizer' },
40
+ WarrantsReplyGate: { task: 'warrants-reply' },
41
+ InputClassifier: { task: 'input-classifier' },
42
+ Usher: { task: 'usher' },
43
+ 'correction-learning': { task: 'correction-distiller' },
44
+ CoherenceReviewer: { task: 'gate-triage' },
45
+
46
+ // ── Argued exemptions (pinned; each must carry a real reason) ──
47
+ InteractivePoolCanaryJudge: {
48
+ exempt:
49
+ 'judges a FIXED known-answer canary probe — the canary is its own benchmark; a bench task would re-test the same constant',
50
+ },
51
+
52
+ // ── Wave 2 (sentinels/gates/extractors still to author) ──
53
+ InputDetector: { pending: 'wave-2' },
54
+ InputGuard: { pending: 'wave-2' },
55
+ SessionActivitySentinel: { pending: 'wave-2' },
56
+ StallTriageNurse: { pending: 'wave-2' },
57
+ CommitmentSentinel: { pending: 'wave-2' },
58
+ PresenceProxy: { pending: 'wave-2' },
59
+ PromiseBeacon: { pending: 'wave-2' },
60
+ ProjectDriftChecker: { pending: 'wave-2' },
61
+ TemporalCoherenceChecker: { pending: 'wave-2' },
62
+ SessionWatchdog: { pending: 'wave-2' },
63
+ ResumeQueueDrainer: { pending: 'wave-2' },
64
+ TopicIntentArcCheck: { pending: 'wave-2' },
65
+ SlackAdapter: { pending: 'wave-2' },
66
+ TelegramAdapter: { pending: 'wave-2' },
67
+ PromptGate: { pending: 'wave-2' },
68
+ AutoApprover: { pending: 'wave-2' },
69
+ IntegrationGate: { pending: 'wave-2' },
70
+ UnjustifiedStopGate: { pending: 'wave-2' },
71
+ CoherenceGate: { pending: 'wave-2' },
72
+ OverrideDetector: { pending: 'wave-2' },
73
+ TaskClassifier: { pending: 'wave-2' },
74
+ ResumeValidator: { pending: 'wave-2' },
75
+ SessionSummarySentinel: { pending: 'wave-2' },
76
+ TopicIntentExtractor: { pending: 'wave-2' },
77
+
78
+ // ── Wave 3 (reflectors + background/job tasks) ──
79
+ JobReflector: { pending: 'wave-3' },
80
+ crossModelReviewer: { pending: 'wave-3' },
81
+ SelfKnowledgeTree: { pending: 'wave-3' },
82
+ TreeTriage: { pending: 'wave-3' },
83
+ TopicSummarizer: { pending: 'wave-3' },
84
+ ContextualEvaluator: { pending: 'wave-3' },
85
+ RelationshipManager: { pending: 'wave-3' },
86
+ StandardsConformanceReviewer: { pending: 'wave-3' },
87
+ DiscoveryEvaluator: { pending: 'wave-3' },
88
+ PipeSessionSpawner: { pending: 'wave-3' },
89
+ CartographerSweep: { pending: 'wave-3' },
90
+ StandardsCoverageEnrichment: { pending: 'wave-3' },
91
+ PreCompactionFlush: { pending: 'wave-3' },
92
+ TreeSynthesis: { pending: 'wave-3' },
93
+ LLMConflictResolver: { pending: 'wave-3' },
94
+ openConversationBrief: { pending: 'wave-3' },
95
+ 'a2a-checkin': { pending: 'wave-3' },
96
+ 'mentor-stage-b': { pending: 'wave-3' },
97
+ };
@@ -0,0 +1,42 @@
1
+ # Upgrade Guide — vNEXT
2
+
3
+ <!-- assembled-by: assemble-next-md -->
4
+ <!-- bump: patch -->
5
+
6
+ ## What Changed
7
+
8
+ INSTAR-Bench v2 ratchet #2 (spec §6; operator directive 2026-07-02, topic
9
+ 29723): a new coverage ledger (`src/data/llmBenchCoverage.ts`) maps every LLM
10
+ component in `COMPONENT_CATEGORY` to its benchmark status — covered by a
11
+ named bench task (the 10 Wave-1 critical gates/sentinels), pending (42,
12
+ pinned shrink-only), or exempt (1, argued). A CI test
13
+ (`llm-bench-coverage-ratchet.test.ts`) fails the build when a new LLM
14
+ component ships without a coverage decision, when the pending/exempt sets
15
+ GROW (graduating shrinks them; adding requires editing the pinned baseline in
16
+ the test — a visible, reviewed act), when an exemption's argument is lazy, or
17
+ when a Wave-1 critical component slides back to uncovered.
18
+
19
+ Together with the existing attribution ratchet + category-map wiring test,
20
+ the chain is closed: every LLM callsite must be attributed → categorized/
21
+ routed deliberately → benchmark-covered. No runtime surface; build-time only.
22
+
23
+ ## What to Tell Your User
24
+
25
+ Nothing proactively — no behavior changes. If asked: every one of my internal
26
+ AI helpers is now tracked against a benchmark coverage ledger, so any new
27
+ helper added to my infrastructure must ship with quality measurements (or a
28
+ written argument why not) before the build goes green. It keeps my model
29
+ choices grounded in measured results instead of guesses.
30
+
31
+ ## Summary of New Capabilities
32
+
33
+ None user-facing. New build-time guarantee: `src/data/llmBenchCoverage.ts`
34
+ (the coverage ledger) + `llm-bench-coverage-ratchet.test.ts` (the CI ratchet).
35
+ The ledger is importable by future tooling (e.g. the bench runner's coverage
36
+ report) but nothing at runtime consumes it yet.
37
+
38
+ ## Evidence
39
+
40
+ 6 new tests green (completeness, no-dangling, shrink-only pins ×2, argued
41
+ exemptions, Wave-1 regression pin); `tsc --noEmit` clean; full unit suite
42
+ green at push.
@@ -0,0 +1,22 @@
1
+ # Upgrade Guide — vNEXT
2
+
3
+ <!-- assembled-by: assemble-next-md -->
4
+ <!-- bump: patch -->
5
+
6
+ ## What Changed
7
+
8
+ The five circuit-breaker tests in `tests/unit/TopicProfileOrchestrator.test.ts` synchronized on `parkedFor(...) !== null` — an intermediate state that `parkAndRevert` makes visible before its internal `flushDurably` await yields control. Under CI load, the 10ms poll could observe the park and assert on trip effects (un-park, `breaker-revert` audit, disclosure) that had not executed yet, failing the shard intermittently (twice on PR #1320's CI tonight; reproduced locally on iteration 1 of a 15-run loop). The tests now synchronize on the `breaker-revert` audit — emitted after every other synchronous trip side-effect — via a shared `waitForBreakerTrip(h)` helper. Test-only; no runtime source touched.
9
+
10
+ ## What to Tell Your User
11
+
12
+ None — internal change (no user-facing surface).
13
+
14
+ ## Summary of New Capabilities
15
+
16
+ None — internal change (no user-facing surface).
17
+
18
+ ## Evidence
19
+
20
+ - Pre-fix: `tests/unit/TopicProfileOrchestrator.test.ts` failed iteration 1/15 of a local repetition loop with the exact CI assertion (`expected [] to include '7'` at the `unparks` check), matching both CI shard 2/4 failures on PR #1320 (runs 28573712457 and its predecessor).
21
+ - Post-fix: 25 consecutive local runs green under concurrent benchmark load on the same machine.
22
+ - Full unit suite green at push (husky pre-push).
@@ -0,0 +1,11 @@
1
+ # New AI decision-makers now have to bring a test report card
2
+
3
+ Instar has about fifty little AI helpers inside it — one classifies your messages, one reviews outgoing replies, one judges whether an autonomous work session really finished, and so on. We just started benchmarking these helpers properly: for each one, a battery of hard test cases that tells us which AI model does that specific job well, fast, and cheap. Those measurements are what our model-routing choices will be based on going forward.
4
+
5
+ But a benchmark only stays trustworthy if it stays complete. The failure mode is silent: someone adds a NEW AI helper next month, nobody writes test cases for it, and suddenly there's an unmeasured decision-maker running in production whose model was picked by vibes. Six months later there are ten of those and the benchmark is a museum piece.
6
+
7
+ This change makes that impossible — not by policy, by build failure. There's now a coverage ledger listing every AI helper and its benchmark status: covered (with the name of its test battery), queued for authoring (an explicitly pinned to-do list), or exempt (with a written argument why testing it makes no sense — and "n/a" is literally rejected as too short). A test in the build system cross-checks that ledger against the master list of AI helpers. Add a new helper without deciding its benchmark story, and the build goes red with instructions telling you exactly what to do.
8
+
9
+ The teeth are in the details: the "queued" list can only shrink. You can graduate an entry to covered, but adding a new name to the queue requires editing the pinned list inside the test itself — a change any reviewer will see and question. Same for exemptions. So the lazy path (quietly park it in the queue forever) is exactly as visible as the honest path (write the test cases). Ten of the most critical helpers — the message classifier, the outbound-message gate, the completion judge, the injection screener, and their peers — are additionally pinned as "may never slide back": once covered, always covered.
10
+
11
+ This is the second ratchet of its kind. The first (already live) forces every new AI call site to register in the routing map so we know which provider runs it. Together they close the loop: every AI decision-maker in instar must be routed deliberately AND measured before it ships. Nothing changes at runtime — this is purely a build-time guarantee, and it costs nothing unless someone tries to skip the homework.
@@ -0,0 +1,41 @@
1
+ # Side-Effects Review — LLM benchmark-coverage ratchet (INSTAR-Bench v2 §6, ratchet #2)
2
+
3
+ **Version / slug:** `llm-bench-coverage-ratchet`
4
+ **Date:** `2026-07-02`
5
+ **Author:** `Instar Agent (echo)`
6
+ **Second-pass reviewer:** `not required` (build-time test + data map only; zero runtime surface)
7
+
8
+ ## Summary of the change
9
+
10
+ Adds `src/data/llmBenchCoverage.ts` — a coverage ledger mapping every LLM component (every key of `COMPONENT_CATEGORY`) to its INSTAR-Bench v2 status: `{task}` covered (10 Wave-1 critical components), `{pending}` queued (42, pinned), or `{exempt}` argued (1) — and `tests/unit/llm-bench-coverage-ratchet.test.ts`, which fails CI when a new LLM component ships without a coverage decision, when the pending/exempt sets grow (shrink-only pins), when an exemption's argument is lazy (<40 chars), or when a Wave-1 critical component slides back to uncovered. Operator directive 2026-07-02 (topic 29723) + INSTAR-BENCH-V2-SPEC §6. No runtime code is touched.
11
+
12
+ ## Decision-point inventory
13
+
14
+ None at runtime. The only "decision" is a CI pass/fail on repository content — a build-time ratchet identical in class to the existing `llm-attribution-ratchet`.
15
+
16
+ ## 1. Over-block
17
+ The ratchet can block a COMMIT (not a message/action): a developer adding an LLM component who genuinely cannot author bench cases yet must add the component to the pending baseline inside the test — a deliberate, visible act. That is the intended cost, not an over-block; there is no path it rejects that it shouldn't (an argued exemption is always available).
18
+
19
+ ## 2. Under-block
20
+ Honest limits: (a) the ratchet keys off `COMPONENT_CATEGORY` — an LLM callsite that dodges the category map would also dodge this map, but the companion evaluate-coverage wiring test + attribution lint close that hole upstream; (b) it cannot verify the bench task file CONTENT (those live in the bench harness on the benching agent) — the task id is a contract, and a wrong/empty id would surface at the next bench run, not in CI. Accepted for v1 of the ratchet; noted in the map docblock.
21
+
22
+ ## 3. Level-of-abstraction fit
23
+ Right layer: repository data + unit test, the established ratchet pattern (`llm-attribution-ratchet`, `WIRING_EXCLUSIONS` pins). Extends the existing chain (callsite → category map → THIS → bench coverage) rather than inventing a parallel mechanism.
24
+
25
+ ## 4. Signal vs authority compliance
26
+ Compliant. No runtime authority; a CI test is deterministic, cheap, and its "block" is a build failure with instructions. (docs/signal-vs-authority.md concerns runtime decision points; none added.)
27
+
28
+ ## 5. Interactions
29
+ Complements `llm-attribution-ratchet` (attribution presence), `componentCategories` wiring test (map exhaustiveness), and the LLM Routing Registry doc. No double-fire: each guards a different link of the chain. No runtime interaction possible.
30
+
31
+ ## 6. External surfaces
32
+ None. Not user-visible; no API, no message, no config.
33
+
34
+ ## 7. Multi-machine posture (Cross-Machine Coherence)
35
+ Not applicable — repository content evaluated in CI; identical on every machine by construction (git).
36
+
37
+ ## 8. Rollback cost
38
+ Delete the test + map (one revert). No state, no migration, no fleet impact — the ratchet only ever constrained future commits.
39
+
40
+ ## Test evidence
41
+ `llm-bench-coverage-ratchet.test.ts`: 6 tests — completeness over COMPONENT_CATEGORY, no dangling entries, pending pinned shrink-only, exemptions pinned + argued, non-empty task ids, Wave-1 regression pin. All green; `tsc --noEmit` clean.
@@ -0,0 +1,16 @@
1
+ # Side-Effects Review — tpo-breaker-flake-fix
2
+
3
+ **Change:** test-only. `tests/unit/TopicProfileOrchestrator.test.ts` — the five §10.4 circuit-breaker tests waited on `parkedFor('7') !== null`, an intermediate state `parkAndRevert` makes visible before its internal `flushDurably` await yields; under CI load the 10ms poll observed the park and asserted on trip effects (unpark / audit / disclosure) that had not run yet. They now wait on the `breaker-revert` audit — the trip's terminal signal, emitted after every other synchronous trip side-effect — via a shared `waitForBreakerTrip(h)` helper. No `src/` file touched.
4
+
5
+ 1. **Over-block** — none. No runtime decision surface; the tests assert the same behavior with a later, race-free synchronization point.
6
+ 2. **Under-block** — none. Coverage is unchanged: every assertion that existed still runs (including `parkedFor` non-null, asserted explicitly after the wait). The wait is stricter, not looser — it requires the full trip, not just the park.
7
+ 3. **Level-of-abstraction fit** — considered fixing the production ordering instead (assign `entry.parked` after `flushDurably`). Rejected: the mid-trip visibility is harmless in production (no production consumer polls `parkedFor` during a trip; the store lock serializes real readers), and reordering durable-write vs in-memory state has its own failure-mode tradeoffs (a crash between write and assign). The test was the thing making a timing assumption; the test is the right layer to fix.
8
+ 4. **Signal vs authority compliance** — n/a; no decision point, no gate, no blocking authority. Test-only.
9
+ 5. **Interactions** — the helper waits on `h.audits`, which each test's fresh harness resets, so no cross-test bleed. The two tests that subsequently call `requestRecoveryWrite` also benefit (they previously raced `durable.breakerTrips` being set mid-trip).
10
+ 6. **External surfaces** — none. Nothing visible to agents, users, or other systems changes.
11
+ 7. **Multi-machine posture** — n/a (test-only; no state, no replication surface).
12
+ 8. **Rollback cost** — `git revert` of one commit restores the old waits; zero data or fleet impact.
13
+
14
+ **Evidence:** old code failed iteration 1 of a 15-run local loop (same assertion as CI shard 2/4 failures on PR #1320 tonight, twice); fixed code passed 25 consecutive runs under benchmark load on the same machine.
15
+
16
+ **Second-pass review:** not required — no sentinel/gate/lifecycle runtime code touched (test file only).
@@ -0,0 +1,11 @@
1
+ # A flaky safety-net test now waits for the finish line, not the halfway mark
2
+
3
+ Instar has a "circuit breaker" for conversation settings: if an operator pins a topic to a model that keeps failing to launch, the breaker parks the bad pin, reverts to the last working settings, and restarts the session — so a typo'd model name can never brick a conversation.
4
+
5
+ The breaker itself works fine. Its TESTS, though, were flaky — they'd pass most of the time and occasionally fail on the build servers (twice tonight alone, blocking an unrelated PR from merging). Flaky tests are poison: every red build that ISN'T real teaches people to ignore red builds.
6
+
7
+ The root cause is a classic race. When the breaker trips, it does several things in a row: park the bad pin → revert → un-park the resume entry → write an audit record → tell the user → restart the session. The tests waited for the FIRST visible step ("is the pin parked yet?") and then immediately checked ALL the later steps. There's a tiny window where the park is visible but the rest hasn't happened yet — normally sub-millisecond, but on a loaded build machine the test's polling could land exactly inside it and fail on assertions about steps that were still in flight.
8
+
9
+ The fix: the tests now wait for the audit record — which the breaker writes only AFTER everything else in the trip is done — before checking any of the trip's effects. Same coverage, zero timing dependence. Verified by running the test file 25 times in a row under load (the old version failed on the very first try of a 15-run loop).
10
+
11
+ No production code changed at all — this is a test-only fix. The breaker behaves exactly as before; its report card just stopped lying about it.