instar 1.3.738 → 1.3.739
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/data/llmBenchCoverage.d.ts +18 -0
- package/dist/data/llmBenchCoverage.d.ts.map +1 -1
- package/dist/data/llmBenchCoverage.js +68 -0
- package/dist/data/llmBenchCoverage.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +2 -2
- package/src/data/llmBenchCoverage.ts +134 -0
- package/upgrades/1.3.739.md +77 -0
- package/upgrades/side-effects/evidence-bar-judge-extension.md +66 -0
|
@@ -35,4 +35,22 @@ export type UntrustedInputFlag = true | {
|
|
|
35
35
|
false: string;
|
|
36
36
|
};
|
|
37
37
|
export declare const LLM_UNTRUSTED_INPUT: Readonly<Record<string, UntrustedInputFlag>>;
|
|
38
|
+
/** The kind of claim a judge callsite credits/refuses (spec §2). Different kinds
|
|
39
|
+
* take different evidence, so each `judgesClaims: true` entry declares its kind:
|
|
40
|
+
* - completionClaim: proof of asserted work ("the task is done / tests pass");
|
|
41
|
+
* - healthClaim: behavioral-signal sufficiency (stall/stuck/health classifiers);
|
|
42
|
+
* - scoredCredit: rubric evaluators that award credit for claimed work. */
|
|
43
|
+
export type ClaimKind = 'completionClaim' | 'healthClaim' | 'scoredCredit';
|
|
44
|
+
export declare const CLAIM_KINDS: ReadonlyArray<ClaimKind>;
|
|
45
|
+
/** The `judgesClaims` classification of one LLM callsite:
|
|
46
|
+
* - `{ claimKind }` → judges a claim (true), of the declared kind;
|
|
47
|
+
* - `false` → does not judge a completion/progress/health claim;
|
|
48
|
+
* - `{ false: '<reason>' }` → a judge-SHAPED callsite argued OUT of scope
|
|
49
|
+
* (pinned shrink-only + reviewed, like the untrustedInput argued-false set). */
|
|
50
|
+
export type JudgesClaimsFlag = {
|
|
51
|
+
claimKind: ClaimKind;
|
|
52
|
+
} | false | {
|
|
53
|
+
false: string;
|
|
54
|
+
};
|
|
55
|
+
export declare const LLM_JUDGES_CLAIMS: Readonly<Record<string, JudgesClaimsFlag>>;
|
|
38
56
|
//# sourceMappingURL=llmBenchCoverage.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llmBenchCoverage.d.ts","sourceRoot":"","sources":["../../src/data/llmBenchCoverage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,OAAO,EAAE,QAAQ,GAAG,QAAQ,CAAA;CAAE,GAChC;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvB,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAqFtE,CAAC;AA4BF,MAAM,MAAM,kBAAkB,GAAG,IAAI,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1D,eAAO,MAAM,mBAAmB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAiF5E,CAAC"}
|
|
1
|
+
{"version":3,"file":"llmBenchCoverage.d.ts","sourceRoot":"","sources":["../../src/data/llmBenchCoverage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,MAAM,MAAM,aAAa,GACrB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,OAAO,EAAE,QAAQ,GAAG,QAAQ,CAAA;CAAE,GAChC;IAAE,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvB,eAAO,MAAM,kBAAkB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAqFtE,CAAC;AA4BF,MAAM,MAAM,kBAAkB,GAAG,IAAI,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAE1D,eAAO,MAAM,mBAAmB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAiF5E,CAAC;AA+CF;;;;6EAI6E;AAC7E,MAAM,MAAM,SAAS,GAAG,iBAAiB,GAAG,aAAa,GAAG,cAAc,CAAC;AAE3E,eAAO,MAAM,WAAW,EAAE,aAAa,CAAC,SAAS,CAIhD,CAAC;AAEF;;;;oFAIoF;AACpF,MAAM,MAAM,gBAAgB,GAAG;IAAE,SAAS,EAAE,SAAS,CAAA;CAAE,GAAG,KAAK,GAAG;IAAE,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpF,eAAO,MAAM,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAmExE,CAAC"}
|
|
@@ -171,4 +171,72 @@ export const LLM_UNTRUSTED_INPUT = {
|
|
|
171
171
|
false: 'attribution-manifest alias only (a legacy prompt-pattern matcher); the live matcher calls with attribution PromptGate, classified true there',
|
|
172
172
|
},
|
|
173
173
|
};
|
|
174
|
+
export const CLAIM_KINDS = [
|
|
175
|
+
'completionClaim',
|
|
176
|
+
'healthClaim',
|
|
177
|
+
'scoredCredit',
|
|
178
|
+
];
|
|
179
|
+
export const LLM_JUDGES_CLAIMS = {
|
|
180
|
+
// ── Judges (judgesClaims: true) ────────────────────────────────────────────
|
|
181
|
+
// Completion evaluators — credit/refuse a claim of asserted work.
|
|
182
|
+
CompletionEvaluator: { claimKind: 'completionClaim' }, // THE motivating callsite: credited a bare "tests pass"
|
|
183
|
+
UnjustifiedStopGate: { claimKind: 'completionClaim' }, // judges whether a stop is justified = is the claimed work actually done/blocked
|
|
184
|
+
JobReflector: { claimKind: 'completionClaim' }, // reflects on job success/completion (spec §2 pending-wave judge; classified now so the asymmetry can't silently reopen)
|
|
185
|
+
// Stall/stuck/health classifiers — credit/refuse a health claim about a session.
|
|
186
|
+
SessionWatchdog: { claimKind: 'healthClaim' }, // stuck-judge
|
|
187
|
+
PresenceProxy: { claimKind: 'healthClaim' }, // tier-3 stall judge
|
|
188
|
+
StallTriageNurse: { claimKind: 'healthClaim' }, // stall-triage diagnosis
|
|
189
|
+
TelegramAdapter: { claimKind: 'healthClaim' }, // confirmStallAlert — judges whether a session is genuinely stalled before alerting (inclusion criteria: stall/health classifier)
|
|
190
|
+
SlackAdapter: { claimKind: 'healthClaim' }, // confirmStallAlert (byte-identical prompt to Telegram's; parity)
|
|
191
|
+
// Scored evaluators — award credit for claimed work.
|
|
192
|
+
'mentor-stage-b': { claimKind: 'scoredCredit' }, // classifies mentor signals over mentee output (spec §2 pending-wave judge)
|
|
193
|
+
// ── Not a completion/progress/health judge (judgesClaims: false) ────────────
|
|
194
|
+
// Sentinels that classify/summarize but do not credit a completion/health claim.
|
|
195
|
+
InputDetector: false, // legacy prompt-pattern matcher alias
|
|
196
|
+
InputGuard: false, // input-coherence, not a completion/health claim
|
|
197
|
+
SessionActivitySentinel: false, // activity DIGEST — summarizes activity, does not credit a claim (exclusion: pure summarizer)
|
|
198
|
+
CommitmentSentinel: false, // detects commitments in text
|
|
199
|
+
PromiseBeacon: false, // no live LLM prompt (hooks unwired at construction — see its bench exemption)
|
|
200
|
+
MessageSentinel: false, // classifies message intent (pause/emergency), not a completion claim
|
|
201
|
+
TopicIntentArcCheck: false, // arc-check classification of a topic's intent, not a completion/health claim
|
|
202
|
+
InputClassifier: false, // input auto-approve vs relay classification
|
|
203
|
+
SessionSummarySentinel: false, // summarizes tmux output → task/phase/files (exclusion: pure summarizer)
|
|
204
|
+
ProjectDriftChecker: false, // is-work-on-project coherence, not a completion/health claim
|
|
205
|
+
TemporalCoherenceChecker: false, // temporal-coherence classification
|
|
206
|
+
ResumeQueueDrainer: false, // inspects session state before a resume — judges STATE validity, not an agent's claim
|
|
207
|
+
InteractivePoolCanaryJudge: false, // judges a FIXED known-answer canary constant, not an agent claim
|
|
208
|
+
// Gates that classify an action/message but do not credit a completion/health claim.
|
|
209
|
+
PromptGate: false, // detects prompt-injection in content
|
|
210
|
+
AutoApprover: false, // mechanical key injection, no LLM prompt of its own
|
|
211
|
+
IntegrationGate: false, // delegates to JobReflector.reflect(), no own callsite
|
|
212
|
+
ExternalOperationGate: false, // classifies operation mutability/reversibility, not a completion claim
|
|
213
|
+
WarrantsReplyGate: false, // "should I reply?" — not a completion/health claim
|
|
214
|
+
CoherenceGate: false, // no own callsite — flows through CoherenceReviewer
|
|
215
|
+
MessagingToneGate: false, // reviews outbound tone/leaks
|
|
216
|
+
CoherenceReviewer: false, // reviews outbound coherence
|
|
217
|
+
LLMSanitizer: false, // sanitizes untrusted inbound content
|
|
218
|
+
OverrideDetector: false, // detects override intent in a turn
|
|
219
|
+
TaskClassifier: false, // classifies task type
|
|
220
|
+
ResumeValidator: false, // matches a resume UUID against a topic — a state match, not a claim
|
|
221
|
+
// Reflectors/jobs that extract/summarize/route but do not credit a completion/health claim.
|
|
222
|
+
crossModelReviewer: false, // reviews a SPEC document, not a session's completion claim
|
|
223
|
+
SelfKnowledgeTree: false, // extracts self-knowledge
|
|
224
|
+
TreeTriage: false, // triages tree fragments
|
|
225
|
+
TopicSummarizer: false, // summarizes a topic
|
|
226
|
+
ContextualEvaluator: false, // evaluates context relevance
|
|
227
|
+
RelationshipManager: false, // extracts relationship facts
|
|
228
|
+
StandardsConformanceReviewer: false, // reviews artifact-vs-standard conformance, not a session completion claim
|
|
229
|
+
DiscoveryEvaluator: false, // evaluates serendipity discoveries
|
|
230
|
+
Usher: false, // routes a turn to candidate topics
|
|
231
|
+
TopicIntentExtractor: false, // extracts topic intent from a turn
|
|
232
|
+
PreCompactionFlush: false, // extracts durable facts before compaction
|
|
233
|
+
TreeSynthesis: false, // synthesizes knowledge fragments → answer
|
|
234
|
+
LLMConflictResolver: false, // resolves divergent multi-machine state
|
|
235
|
+
openConversationBrief: false, // generates an A2A conversation brief
|
|
236
|
+
'a2a-checkin': false, // summarizes A2A check-in threads
|
|
237
|
+
'correction-learning': false, // distills recurring corrections → preference
|
|
238
|
+
PipeSessionSpawner: false, // spawns from task descriptions
|
|
239
|
+
CartographerSweep: false, // authors doc-tree summaries over code
|
|
240
|
+
StandardsCoverageEnrichment: false, // enriches standards-coverage rows
|
|
241
|
+
};
|
|
174
242
|
//# sourceMappingURL=llmBenchCoverage.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"llmBenchCoverage.js","sourceRoot":"","sources":["../../src/data/llmBenchCoverage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAOH,MAAM,CAAC,MAAM,kBAAkB,GAA4C;IACzE,mEAAmE;IACnE,eAAe,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE;IAC9C,iBAAiB,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;IACxC,6DAA6D;IAC7D,mBAAmB,EAAE,EAAE,IAAI,EAAE,iCAAiC,EAAE;IAChE,qBAAqB,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACnD,YAAY,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE;IAC7C,iBAAiB,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;IAC7C,eAAe,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE;IAC7C,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;IACxB,qBAAqB,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;IACvD,iBAAiB,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;IAE1C,kEAAkE;IAClE,0BAA0B,EAAE;QAC1B,MAAM,EACJ,0HAA0H;KAC7H;IAED,mFAAmF;IACnF,UAAU,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;IAC7C,uBAAuB,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE;IACpD,gBAAgB,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE;IACpD,kBAAkB,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE;IACnD,aAAa,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;IAC/C,mBAAmB,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE;IACpD,wBAAwB,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE;IACxD,eAAe,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;IACjD,kBAAkB,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE;IACnD,mBAAmB,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE;IACnD,eAAe,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE;IACnD,mEAAmE;IACnE,sEAAsE;IACtE,iEAAiE;IACjE,YAAY,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE;IAChD,UAAU,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE;IAC1C,mBAAmB,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;IACtD,gBAAgB,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE;IAC/C,cAAc,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE;IAC3C,eAAe,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE;IAC7C,sBAAsB,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE;IAC5D,oBAAoB,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE;IAExD,yFAAyF;IACzF,eAAe,EAAE;QACf,MAAM,EACJ,yKAAyK;KAC5K;IACD,aAAa,EAAE;QACb,MAAM,EACJ,qKAAqK;KACxK;IACD,YAAY,EAAE;QACZ,MAAM,EACJ,6IAA6I;KAChJ;IACD,aAAa,EAAE;QACb,MAAM,EACJ,mLAAmL;KACtL;IACD,aAAa,EAAE;QACb,MAAM,EACJ,+LAA+L;KAClM;IAED,mDAAmD;IACnD,YAAY,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,iBAAiB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACxC,UAAU,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACjC,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,4BAA4B,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnD,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,iBAAiB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACxC,2BAA2B,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAClD,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,qBAAqB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC5C,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,gBAAgB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;CACxC,CAAC;AA8BF,MAAM,CAAC,MAAM,mBAAmB,GAAiD;IAC/E,yEAAyE;IACzE,UAAU,EAAE,IAAI;IAChB,uBAAuB,EAAE,IAAI;IAC7B,gBAAgB,EAAE,IAAI;IACtB,kBAAkB,EAAE,IAAI;IACxB,aAAa,EAAE,IAAI;IACnB,eAAe,EAAE,IAAI;IACrB,mBAAmB,EAAE,IAAI;IACzB,wBAAwB,EAAE,IAAI;IAC9B,mBAAmB,EAAE,IAAI;IACzB,eAAe,EAAE,IAAI;IACrB,kBAAkB,EAAE,IAAI;IACxB,mBAAmB,EAAE,IAAI;IACzB,YAAY,EAAE,IAAI;IAClB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,IAAI;IAC5B,eAAe,EAAE,IAAI;IAErB,4DAA4D;IAC5D,UAAU,EAAE,IAAI;IAChB,qBAAqB,EAAE,IAAI,EAAE,uEAAuE;IACpG,iBAAiB,EAAE,IAAI;IACvB,mBAAmB,EAAE,IAAI;IACzB,iBAAiB,EAAE,IAAI,EAAE,oEAAoE;IAC7F,iBAAiB,EAAE,IAAI;IACvB,YAAY,EAAE,IAAI,EAAE,kDAAkD;IACtE,gBAAgB,EAAE,IAAI;IACtB,cAAc,EAAE,IAAI;IACpB,eAAe,EAAE,IAAI,EAAE,wEAAwE;IAE/F,oFAAoF;IACpF,YAAY,EAAE,IAAI;IAClB,kBAAkB,EAAE,IAAI;IACxB,iBAAiB,EAAE,IAAI;IACvB,UAAU,EAAE,IAAI;IAChB,eAAe,EAAE,IAAI;IACrB,mBAAmB,EAAE,IAAI;IACzB,mBAAmB,EAAE,IAAI;IACzB,4BAA4B,EAAE,IAAI;IAClC,kBAAkB,EAAE,IAAI;IACxB,KAAK,EAAE,IAAI;IACX,oBAAoB,EAAE,IAAI;IAC1B,kBAAkB,EAAE,IAAI;IACxB,aAAa,EAAE,IAAI;IACnB,mBAAmB,EAAE,IAAI,EAAE,sDAAsD;IACjF,qBAAqB,EAAE,IAAI;IAC3B,aAAa,EAAE,IAAI,EAAE,4BAA4B;IACjD,qBAAqB,EAAE,IAAI;IAC3B,gBAAgB,EAAE,IAAI;IAEtB,+DAA+D;IAC/D,kBAAkB,EAAE,IAAI,EAAE,0DAA0D;IACpF,iBAAiB,EAAE,IAAI,EAAE,gGAAgG;IACzH,2BAA2B,EAAE,IAAI;IAEjC,0FAA0F;IAC1F,aAAa,EAAE;QACb,KAAK,EACH,8KAA8K;KACjL;IACD,0BAA0B,EAAE;QAC1B,KAAK,EACH,iIAAiI;KACpI;IACD,YAAY,EAAE;QACZ,KAAK,EACH,2IAA2I;KAC9I;IACD,eAAe,EAAE;QACf,KAAK,EACH,mIAAmI;KACtI;IACD,aAAa,EAAE;QACb,KAAK,EACH,+HAA+H;KAClI;IACD,aAAa,EAAE;QACb,KAAK,EACH,8IAA8I;KACjJ;CACF,CAAC"}
|
|
1
|
+
{"version":3,"file":"llmBenchCoverage.js","sourceRoot":"","sources":["../../src/data/llmBenchCoverage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAOH,MAAM,CAAC,MAAM,kBAAkB,GAA4C;IACzE,mEAAmE;IACnE,eAAe,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE;IAC9C,iBAAiB,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE;IACxC,6DAA6D;IAC7D,mBAAmB,EAAE,EAAE,IAAI,EAAE,iCAAiC,EAAE;IAChE,qBAAqB,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACnD,YAAY,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE;IAC7C,iBAAiB,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;IAC7C,eAAe,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE;IAC7C,KAAK,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE;IACxB,qBAAqB,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;IACvD,iBAAiB,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;IAE1C,kEAAkE;IAClE,0BAA0B,EAAE;QAC1B,MAAM,EACJ,0HAA0H;KAC7H;IAED,mFAAmF;IACnF,UAAU,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;IAC7C,uBAAuB,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE;IACpD,gBAAgB,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE;IACpD,kBAAkB,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE;IACnD,aAAa,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;IAC/C,mBAAmB,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE;IACpD,wBAAwB,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE;IACxD,eAAe,EAAE,EAAE,IAAI,EAAE,sBAAsB,EAAE;IACjD,kBAAkB,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE;IACnD,mBAAmB,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE;IACnD,eAAe,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE;IACnD,mEAAmE;IACnE,sEAAsE;IACtE,iEAAiE;IACjE,YAAY,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE;IAChD,UAAU,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE;IAC1C,mBAAmB,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;IACtD,gBAAgB,EAAE,EAAE,IAAI,EAAE,mBAAmB,EAAE;IAC/C,cAAc,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE;IAC3C,eAAe,EAAE,EAAE,IAAI,EAAE,kBAAkB,EAAE;IAC7C,sBAAsB,EAAE,EAAE,IAAI,EAAE,0BAA0B,EAAE;IAC5D,oBAAoB,EAAE,EAAE,IAAI,EAAE,wBAAwB,EAAE;IAExD,yFAAyF;IACzF,eAAe,EAAE;QACf,MAAM,EACJ,yKAAyK;KAC5K;IACD,aAAa,EAAE;QACb,MAAM,EACJ,qKAAqK;KACxK;IACD,YAAY,EAAE;QACZ,MAAM,EACJ,6IAA6I;KAChJ;IACD,aAAa,EAAE;QACb,MAAM,EACJ,mLAAmL;KACtL;IACD,aAAa,EAAE;QACb,MAAM,EACJ,+LAA+L;KAClM;IAED,mDAAmD;IACnD,YAAY,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,iBAAiB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACxC,UAAU,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACjC,eAAe,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACtC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,4BAA4B,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACnD,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,iBAAiB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACxC,2BAA2B,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAClD,kBAAkB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACzC,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,mBAAmB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC1C,qBAAqB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IAC5C,aAAa,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;IACpC,gBAAgB,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE;CACxC,CAAC;AA8BF,MAAM,CAAC,MAAM,mBAAmB,GAAiD;IAC/E,yEAAyE;IACzE,UAAU,EAAE,IAAI;IAChB,uBAAuB,EAAE,IAAI;IAC7B,gBAAgB,EAAE,IAAI;IACtB,kBAAkB,EAAE,IAAI;IACxB,aAAa,EAAE,IAAI;IACnB,eAAe,EAAE,IAAI;IACrB,mBAAmB,EAAE,IAAI;IACzB,wBAAwB,EAAE,IAAI;IAC9B,mBAAmB,EAAE,IAAI;IACzB,eAAe,EAAE,IAAI;IACrB,kBAAkB,EAAE,IAAI;IACxB,mBAAmB,EAAE,IAAI;IACzB,YAAY,EAAE,IAAI;IAClB,eAAe,EAAE,IAAI;IACrB,sBAAsB,EAAE,IAAI;IAC5B,eAAe,EAAE,IAAI;IAErB,4DAA4D;IAC5D,UAAU,EAAE,IAAI;IAChB,qBAAqB,EAAE,IAAI,EAAE,uEAAuE;IACpG,iBAAiB,EAAE,IAAI;IACvB,mBAAmB,EAAE,IAAI;IACzB,iBAAiB,EAAE,IAAI,EAAE,oEAAoE;IAC7F,iBAAiB,EAAE,IAAI;IACvB,YAAY,EAAE,IAAI,EAAE,kDAAkD;IACtE,gBAAgB,EAAE,IAAI;IACtB,cAAc,EAAE,IAAI;IACpB,eAAe,EAAE,IAAI,EAAE,wEAAwE;IAE/F,oFAAoF;IACpF,YAAY,EAAE,IAAI;IAClB,kBAAkB,EAAE,IAAI;IACxB,iBAAiB,EAAE,IAAI;IACvB,UAAU,EAAE,IAAI;IAChB,eAAe,EAAE,IAAI;IACrB,mBAAmB,EAAE,IAAI;IACzB,mBAAmB,EAAE,IAAI;IACzB,4BAA4B,EAAE,IAAI;IAClC,kBAAkB,EAAE,IAAI;IACxB,KAAK,EAAE,IAAI;IACX,oBAAoB,EAAE,IAAI;IAC1B,kBAAkB,EAAE,IAAI;IACxB,aAAa,EAAE,IAAI;IACnB,mBAAmB,EAAE,IAAI,EAAE,sDAAsD;IACjF,qBAAqB,EAAE,IAAI;IAC3B,aAAa,EAAE,IAAI,EAAE,4BAA4B;IACjD,qBAAqB,EAAE,IAAI;IAC3B,gBAAgB,EAAE,IAAI;IAEtB,+DAA+D;IAC/D,kBAAkB,EAAE,IAAI,EAAE,0DAA0D;IACpF,iBAAiB,EAAE,IAAI,EAAE,gGAAgG;IACzH,2BAA2B,EAAE,IAAI;IAEjC,0FAA0F;IAC1F,aAAa,EAAE;QACb,KAAK,EACH,8KAA8K;KACjL;IACD,0BAA0B,EAAE;QAC1B,KAAK,EACH,iIAAiI;KACpI;IACD,YAAY,EAAE;QACZ,KAAK,EACH,2IAA2I;KAC9I;IACD,eAAe,EAAE;QACf,KAAK,EACH,mIAAmI;KACtI;IACD,aAAa,EAAE;QACb,KAAK,EACH,+HAA+H;KAClI;IACD,aAAa,EAAE;QACb,KAAK,EACH,8IAA8I;KACjJ;CACF,CAAC;AAsDF,MAAM,CAAC,MAAM,WAAW,GAA6B;IACnD,iBAAiB;IACjB,aAAa;IACb,cAAc;CACf,CAAC;AASF,MAAM,CAAC,MAAM,iBAAiB,GAA+C;IAC3E,8EAA8E;IAC9E,kEAAkE;IAClE,mBAAmB,EAAE,EAAE,SAAS,EAAE,iBAAiB,EAAE,EAAE,wDAAwD;IAC/G,mBAAmB,EAAE,EAAE,SAAS,EAAE,iBAAiB,EAAE,EAAE,iFAAiF;IACxI,YAAY,EAAE,EAAE,SAAS,EAAE,iBAAiB,EAAE,EAAE,yHAAyH;IAEzK,iFAAiF;IACjF,eAAe,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,cAAc;IAC7D,aAAa,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,qBAAqB;IAClE,gBAAgB,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,yBAAyB;IACzE,eAAe,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,kIAAkI;IACjL,YAAY,EAAE,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,kEAAkE;IAE9G,qDAAqD;IACrD,gBAAgB,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,EAAE,4EAA4E;IAE7H,+EAA+E;IAC/E,iFAAiF;IACjF,aAAa,EAAE,KAAK,EAAE,sCAAsC;IAC5D,UAAU,EAAE,KAAK,EAAE,iDAAiD;IACpE,uBAAuB,EAAE,KAAK,EAAE,8FAA8F;IAC9H,kBAAkB,EAAE,KAAK,EAAE,8BAA8B;IACzD,aAAa,EAAE,KAAK,EAAE,+EAA+E;IACrG,eAAe,EAAE,KAAK,EAAE,sEAAsE;IAC9F,mBAAmB,EAAE,KAAK,EAAE,8EAA8E;IAC1G,eAAe,EAAE,KAAK,EAAE,6CAA6C;IACrE,sBAAsB,EAAE,KAAK,EAAE,yEAAyE;IACxG,mBAAmB,EAAE,KAAK,EAAE,8DAA8D;IAC1F,wBAAwB,EAAE,KAAK,EAAE,oCAAoC;IACrE,kBAAkB,EAAE,KAAK,EAAE,uFAAuF;IAClH,0BAA0B,EAAE,KAAK,EAAE,kEAAkE;IAErG,qFAAqF;IACrF,UAAU,EAAE,KAAK,EAAE,sCAAsC;IACzD,YAAY,EAAE,KAAK,EAAE,qDAAqD;IAC1E,eAAe,EAAE,KAAK,EAAE,uDAAuD;IAC/E,qBAAqB,EAAE,KAAK,EAAE,wEAAwE;IACtG,iBAAiB,EAAE,KAAK,EAAE,oDAAoD;IAC9E,aAAa,EAAE,KAAK,EAAE,oDAAoD;IAC1E,iBAAiB,EAAE,KAAK,EAAE,8BAA8B;IACxD,iBAAiB,EAAE,KAAK,EAAE,6BAA6B;IACvD,YAAY,EAAE,KAAK,EAAE,sCAAsC;IAC3D,gBAAgB,EAAE,KAAK,EAAE,oCAAoC;IAC7D,cAAc,EAAE,KAAK,EAAE,uBAAuB;IAC9C,eAAe,EAAE,KAAK,EAAE,qEAAqE;IAE7F,4FAA4F;IAC5F,kBAAkB,EAAE,KAAK,EAAE,4DAA4D;IACvF,iBAAiB,EAAE,KAAK,EAAE,0BAA0B;IACpD,UAAU,EAAE,KAAK,EAAE,yBAAyB;IAC5C,eAAe,EAAE,KAAK,EAAE,qBAAqB;IAC7C,mBAAmB,EAAE,KAAK,EAAE,8BAA8B;IAC1D,mBAAmB,EAAE,KAAK,EAAE,8BAA8B;IAC1D,4BAA4B,EAAE,KAAK,EAAE,2EAA2E;IAChH,kBAAkB,EAAE,KAAK,EAAE,oCAAoC;IAC/D,KAAK,EAAE,KAAK,EAAE,oCAAoC;IAClD,oBAAoB,EAAE,KAAK,EAAE,oCAAoC;IACjE,kBAAkB,EAAE,KAAK,EAAE,2CAA2C;IACtE,aAAa,EAAE,KAAK,EAAE,2CAA2C;IACjE,mBAAmB,EAAE,KAAK,EAAE,yCAAyC;IACrE,qBAAqB,EAAE,KAAK,EAAE,sCAAsC;IACpE,aAAa,EAAE,KAAK,EAAE,kCAAkC;IACxD,qBAAqB,EAAE,KAAK,EAAE,8CAA8C;IAC5E,kBAAkB,EAAE,KAAK,EAAE,gCAAgC;IAC3D,iBAAiB,EAAE,KAAK,EAAE,uCAAuC;IACjE,2BAA2B,EAAE,KAAK,EAAE,mCAAmC;CACxE,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-07-03T22:
|
|
5
|
-
"instarVersion": "1.3.
|
|
4
|
+
"generatedAt": "2026-07-03T22:57:15.282Z",
|
|
5
|
+
"instarVersion": "1.3.739",
|
|
6
6
|
"entryCount": 202,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -226,3 +226,137 @@ export const LLM_UNTRUSTED_INPUT: Readonly<Record<string, UntrustedInputFlag>> =
|
|
|
226
226
|
'attribution-manifest alias only (a legacy prompt-pattern matcher); the live matcher calls with attribution PromptGate, classified true there',
|
|
227
227
|
},
|
|
228
228
|
};
|
|
229
|
+
|
|
230
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
231
|
+
// Evidence-Bar Extension — Judge Prompts (defect class 3 / claim-vs-evidence —
|
|
232
|
+
// docs/specs/evidence-bar-judge-extension.md §2 "the judge-nature classification")
|
|
233
|
+
//
|
|
234
|
+
// The `judgesClaims` axis of the program's shared per-callsite metadata record
|
|
235
|
+
// (class-closure-gate.md §"Program-shared machinery"; co-located in THIS file
|
|
236
|
+
// with the bench-coverage record it extends, exactly like the sibling
|
|
237
|
+
// `untrustedInput` axis). The consolidated axis ratchet derives the required
|
|
238
|
+
// bench axes from all these per-callsite axes together.
|
|
239
|
+
//
|
|
240
|
+
// WHY (earned): on 2026-07-02 the INSTAR-Bench v2 defect-class review found the
|
|
241
|
+
// completion judge (and four other model routes on the same case) credited a
|
|
242
|
+
// BARE assertion ("tests pass," no output shown) as satisfied evidence — the
|
|
243
|
+
// judge prompt never defined what counts as evidence, so models defaulted to
|
|
244
|
+
// crediting the claim. The agent-facing "Bug-Fix Evidence Bar" holds the
|
|
245
|
+
// CLAIMANT ("verify before you claim"); the prompts that JUDGE such claims were
|
|
246
|
+
// never given the same rule. This classification is the structural record of
|
|
247
|
+
// WHICH callsites judge a claim, so the (deferred) bench-axis ratchet can
|
|
248
|
+
// require both a bare-claim (false-accept) AND a real-evidence (false-reject)
|
|
249
|
+
// case for each — the known over-correction hazard (the first fix rejected REAL
|
|
250
|
+
// evidence on 6 routes) makes the false-reject direction mandatory.
|
|
251
|
+
//
|
|
252
|
+
// INCLUSION CRITERIA (spec §2): the callsite's task is to CREDIT or REFUSE an
|
|
253
|
+
// agent/session claim of completion, progress, or health — verdict gates,
|
|
254
|
+
// completion evaluators, stuck/stall/health classifiers, and scored evaluators
|
|
255
|
+
// that award credit for claimed work. EXCLUSION: pure summarizers and
|
|
256
|
+
// extractors are out unless they score or credit a claim.
|
|
257
|
+
//
|
|
258
|
+
// THE FIELD IS REQUIRED AND EXPLICIT FOR EVERY COMPONENT_CATEGORY KEY — there is
|
|
259
|
+
// NO DEFAULT (same polarity rule as `untrustedInput`). A `{ claimKind }` entry
|
|
260
|
+
// is a judge (true) and declares WHICH kind of claim it judges (the axis cases +
|
|
261
|
+
// accepted evidence classes are authored per kind). Plain `false` is a callsite
|
|
262
|
+
// that does not judge a completion/progress/health claim. A judge-SHAPED
|
|
263
|
+
// callsite that argues it does NOT judge claims is written as
|
|
264
|
+
// `{ false: '<argued reason>' }` and pinned shrink-only in
|
|
265
|
+
// tests/unit/judges-claims-classification-ratchet.test.ts — a silent omission
|
|
266
|
+
// is red CI, so the flag can never default toward the un-benched state.
|
|
267
|
+
//
|
|
268
|
+
// DETERMINISTIC ARM (spec "Honest reach"): the real-check `verification_command`
|
|
269
|
+
// verifier is NAMED in the spec seed but is NOT an LLM callsite (it runs the
|
|
270
|
+
// actual command on a met-verdict) — it has no COMPONENT_CATEGORY key and is
|
|
271
|
+
// therefore out of this LLM classification by construction. A prompt bar governs
|
|
272
|
+
// what is SHOWN; verification of what is TRUE belongs to that deterministic arm.
|
|
273
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
274
|
+
|
|
275
|
+
/** The kind of claim a judge callsite credits/refuses (spec §2). Different kinds
|
|
276
|
+
* take different evidence, so each `judgesClaims: true` entry declares its kind:
|
|
277
|
+
* - completionClaim: proof of asserted work ("the task is done / tests pass");
|
|
278
|
+
* - healthClaim: behavioral-signal sufficiency (stall/stuck/health classifiers);
|
|
279
|
+
* - scoredCredit: rubric evaluators that award credit for claimed work. */
|
|
280
|
+
export type ClaimKind = 'completionClaim' | 'healthClaim' | 'scoredCredit';
|
|
281
|
+
|
|
282
|
+
export const CLAIM_KINDS: ReadonlyArray<ClaimKind> = [
|
|
283
|
+
'completionClaim',
|
|
284
|
+
'healthClaim',
|
|
285
|
+
'scoredCredit',
|
|
286
|
+
];
|
|
287
|
+
|
|
288
|
+
/** The `judgesClaims` classification of one LLM callsite:
|
|
289
|
+
* - `{ claimKind }` → judges a claim (true), of the declared kind;
|
|
290
|
+
* - `false` → does not judge a completion/progress/health claim;
|
|
291
|
+
* - `{ false: '<reason>' }` → a judge-SHAPED callsite argued OUT of scope
|
|
292
|
+
* (pinned shrink-only + reviewed, like the untrustedInput argued-false set). */
|
|
293
|
+
export type JudgesClaimsFlag = { claimKind: ClaimKind } | false | { false: string };
|
|
294
|
+
|
|
295
|
+
export const LLM_JUDGES_CLAIMS: Readonly<Record<string, JudgesClaimsFlag>> = {
|
|
296
|
+
// ── Judges (judgesClaims: true) ────────────────────────────────────────────
|
|
297
|
+
// Completion evaluators — credit/refuse a claim of asserted work.
|
|
298
|
+
CompletionEvaluator: { claimKind: 'completionClaim' }, // THE motivating callsite: credited a bare "tests pass"
|
|
299
|
+
UnjustifiedStopGate: { claimKind: 'completionClaim' }, // judges whether a stop is justified = is the claimed work actually done/blocked
|
|
300
|
+
JobReflector: { claimKind: 'completionClaim' }, // reflects on job success/completion (spec §2 pending-wave judge; classified now so the asymmetry can't silently reopen)
|
|
301
|
+
|
|
302
|
+
// Stall/stuck/health classifiers — credit/refuse a health claim about a session.
|
|
303
|
+
SessionWatchdog: { claimKind: 'healthClaim' }, // stuck-judge
|
|
304
|
+
PresenceProxy: { claimKind: 'healthClaim' }, // tier-3 stall judge
|
|
305
|
+
StallTriageNurse: { claimKind: 'healthClaim' }, // stall-triage diagnosis
|
|
306
|
+
TelegramAdapter: { claimKind: 'healthClaim' }, // confirmStallAlert — judges whether a session is genuinely stalled before alerting (inclusion criteria: stall/health classifier)
|
|
307
|
+
SlackAdapter: { claimKind: 'healthClaim' }, // confirmStallAlert (byte-identical prompt to Telegram's; parity)
|
|
308
|
+
|
|
309
|
+
// Scored evaluators — award credit for claimed work.
|
|
310
|
+
'mentor-stage-b': { claimKind: 'scoredCredit' }, // classifies mentor signals over mentee output (spec §2 pending-wave judge)
|
|
311
|
+
|
|
312
|
+
// ── Not a completion/progress/health judge (judgesClaims: false) ────────────
|
|
313
|
+
// Sentinels that classify/summarize but do not credit a completion/health claim.
|
|
314
|
+
InputDetector: false, // legacy prompt-pattern matcher alias
|
|
315
|
+
InputGuard: false, // input-coherence, not a completion/health claim
|
|
316
|
+
SessionActivitySentinel: false, // activity DIGEST — summarizes activity, does not credit a claim (exclusion: pure summarizer)
|
|
317
|
+
CommitmentSentinel: false, // detects commitments in text
|
|
318
|
+
PromiseBeacon: false, // no live LLM prompt (hooks unwired at construction — see its bench exemption)
|
|
319
|
+
MessageSentinel: false, // classifies message intent (pause/emergency), not a completion claim
|
|
320
|
+
TopicIntentArcCheck: false, // arc-check classification of a topic's intent, not a completion/health claim
|
|
321
|
+
InputClassifier: false, // input auto-approve vs relay classification
|
|
322
|
+
SessionSummarySentinel: false, // summarizes tmux output → task/phase/files (exclusion: pure summarizer)
|
|
323
|
+
ProjectDriftChecker: false, // is-work-on-project coherence, not a completion/health claim
|
|
324
|
+
TemporalCoherenceChecker: false, // temporal-coherence classification
|
|
325
|
+
ResumeQueueDrainer: false, // inspects session state before a resume — judges STATE validity, not an agent's claim
|
|
326
|
+
InteractivePoolCanaryJudge: false, // judges a FIXED known-answer canary constant, not an agent claim
|
|
327
|
+
|
|
328
|
+
// Gates that classify an action/message but do not credit a completion/health claim.
|
|
329
|
+
PromptGate: false, // detects prompt-injection in content
|
|
330
|
+
AutoApprover: false, // mechanical key injection, no LLM prompt of its own
|
|
331
|
+
IntegrationGate: false, // delegates to JobReflector.reflect(), no own callsite
|
|
332
|
+
ExternalOperationGate: false, // classifies operation mutability/reversibility, not a completion claim
|
|
333
|
+
WarrantsReplyGate: false, // "should I reply?" — not a completion/health claim
|
|
334
|
+
CoherenceGate: false, // no own callsite — flows through CoherenceReviewer
|
|
335
|
+
MessagingToneGate: false, // reviews outbound tone/leaks
|
|
336
|
+
CoherenceReviewer: false, // reviews outbound coherence
|
|
337
|
+
LLMSanitizer: false, // sanitizes untrusted inbound content
|
|
338
|
+
OverrideDetector: false, // detects override intent in a turn
|
|
339
|
+
TaskClassifier: false, // classifies task type
|
|
340
|
+
ResumeValidator: false, // matches a resume UUID against a topic — a state match, not a claim
|
|
341
|
+
|
|
342
|
+
// Reflectors/jobs that extract/summarize/route but do not credit a completion/health claim.
|
|
343
|
+
crossModelReviewer: false, // reviews a SPEC document, not a session's completion claim
|
|
344
|
+
SelfKnowledgeTree: false, // extracts self-knowledge
|
|
345
|
+
TreeTriage: false, // triages tree fragments
|
|
346
|
+
TopicSummarizer: false, // summarizes a topic
|
|
347
|
+
ContextualEvaluator: false, // evaluates context relevance
|
|
348
|
+
RelationshipManager: false, // extracts relationship facts
|
|
349
|
+
StandardsConformanceReviewer: false, // reviews artifact-vs-standard conformance, not a session completion claim
|
|
350
|
+
DiscoveryEvaluator: false, // evaluates serendipity discoveries
|
|
351
|
+
Usher: false, // routes a turn to candidate topics
|
|
352
|
+
TopicIntentExtractor: false, // extracts topic intent from a turn
|
|
353
|
+
PreCompactionFlush: false, // extracts durable facts before compaction
|
|
354
|
+
TreeSynthesis: false, // synthesizes knowledge fragments → answer
|
|
355
|
+
LLMConflictResolver: false, // resolves divergent multi-machine state
|
|
356
|
+
openConversationBrief: false, // generates an A2A conversation brief
|
|
357
|
+
'a2a-checkin': false, // summarizes A2A check-in threads
|
|
358
|
+
'correction-learning': false, // distills recurring corrections → preference
|
|
359
|
+
PipeSessionSpawner: false, // spawns from task descriptions
|
|
360
|
+
CartographerSweep: false, // authors doc-tree summaries over code
|
|
361
|
+
StandardsCoverageEnrichment: false, // enriches standards-coverage rows
|
|
362
|
+
};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# Upgrade Guide — vNEXT
|
|
2
|
+
|
|
3
|
+
<!-- assembled-by: assemble-next-md -->
|
|
4
|
+
<!-- bump: minor -->
|
|
5
|
+
|
|
6
|
+
## What Changed
|
|
7
|
+
|
|
8
|
+
The mechanical arm of the **Evidence-Bar Extension to Judge Prompts** standard
|
|
9
|
+
(`docs/specs/evidence-bar-judge-extension.md`, defect class 3 / `claim-vs-evidence` closure),
|
|
10
|
+
shipped as a self-contained DARK increment — no runtime wiring, no operator-gated registry
|
|
11
|
+
amendment, no config key, no behavioral prompt change. It is the structural answer to the
|
|
12
|
+
2026-07-02 INSTAR-Bench v2 finding that the completion judge (and four other model routes on
|
|
13
|
+
the same case) credited a BARE assertion ("tests pass," no output shown) as satisfied evidence:
|
|
14
|
+
the agent-facing **Bug-Fix Evidence Bar** holds the CLAIMANT ("verify before you claim"), but
|
|
15
|
+
the prompts that JUDGE such claims were never given the same rule — an asymmetry where we hold
|
|
16
|
+
the claimer to a bar the judge does not know exists.
|
|
17
|
+
|
|
18
|
+
This increment ships **the judge-nature classification + its CI ratchet only**:
|
|
19
|
+
|
|
20
|
+
- The **`judgesClaims` classification** (`LLM_JUDGES_CLAIMS` in `src/data/llmBenchCoverage.ts`):
|
|
21
|
+
the `judgesClaims` axis of the program's ONE shared per-callsite metadata record (sibling of
|
|
22
|
+
the authority-clause `untrustedInput` axis), required-explicit for every LLM component (no
|
|
23
|
+
default — a silent omission is red CI). A judge is a `{ claimKind }` entry declaring WHICH
|
|
24
|
+
kind of claim it credits/refuses — `completionClaim` (proof of asserted work), `healthClaim`
|
|
25
|
+
(stall/stuck/health signal sufficiency), or `scoredCredit` (rubric evaluators) — because those
|
|
26
|
+
are NOT one evidence problem and their axis cases are authored per kind. Nine callsites classify
|
|
27
|
+
as judges (the five measured completion/health judges — CompletionEvaluator, UnjustifiedStopGate,
|
|
28
|
+
SessionWatchdog, PresenceProxy, StallTriageNurse — plus the two stall-confirm adapters and the
|
|
29
|
+
two spec-named pending-wave judges JobReflector + mentor-stage-b); the deterministic real-check
|
|
30
|
+
`verification_command` verifier is seed-named but is NOT an LLM callsite (it runs the actual
|
|
31
|
+
command), out of the classification by construction.
|
|
32
|
+
- A **classification ratchet** (`tests/unit/judges-claims-classification-ratchet.test.ts`):
|
|
33
|
+
required-explicit + no-dangling + valid-claimKind-per-judge + the spec-named JUDGE SEED pinned
|
|
34
|
+
as a judge (the callsites measured crediting a bare claim can't silently slip out of scope) +
|
|
35
|
+
the argued-false set pinned shrink-only with a real-reason floor. Same pinned-baseline family as
|
|
36
|
+
`llm-bench-coverage-ratchet` and `untrusted-input-classification-ratchet`.
|
|
37
|
+
|
|
38
|
+
Deliberately OUT of scope (not orphan deferrals — see `upgrades/side-effects/`):
|
|
39
|
+
- The **registry amendment** (spec §1) — operator-gated; ships ONLY with Justin's explicit
|
|
40
|
+
sign-off. It is DRAFTED in the spec; this run does not edit the standards registry.
|
|
41
|
+
- The **bench-axis pair ratchet** (spec §3: a bare-claim false-accept case + a real-evidence
|
|
42
|
+
false-reject case per judge) — blocked on the SAME program-wide "batteries readable by CI"
|
|
43
|
+
decision the sibling authority-clause axis ratchet deferred on (`research/` is absent from
|
|
44
|
+
canonical main), owned by `class-closure-gate.md` §"Program-shared machinery" and binding all
|
|
45
|
+
three axis specs.
|
|
46
|
+
- The **`evidenceBar()` prompt clause** (spec §4) — a sibling of the authority clause in
|
|
47
|
+
`src/core/promptClauses.ts`. That shared library is introduced by the still-open sibling PR
|
|
48
|
+
(authority-clause); adding the clause here would collide on the same new file. The clause is
|
|
49
|
+
only consumed via the A/B-gated per-component migrations (spec §4/rollout §2), which are
|
|
50
|
+
behavioral and deferred regardless. It lands as a follow-up once the shared library is on main.
|
|
51
|
+
|
|
52
|
+
## What to Tell Your User
|
|
53
|
+
|
|
54
|
+
Nothing changes for you right now — this ships **dark**, and it is maintainer-only machinery (a
|
|
55
|
+
no-op on your install unless you develop instar itself). It records WHICH of my AI checks exist to
|
|
56
|
+
JUDGE a completion/health claim, so the same "a claim is not evidence — show the output" bar that
|
|
57
|
+
already binds what I claim can be extended to the checks that judge those claims. The judge-prompt
|
|
58
|
+
wording and the registry text itself change later, and only after an operator sign-off.
|
|
59
|
+
|
|
60
|
+
## Summary of New Capabilities
|
|
61
|
+
|
|
62
|
+
None active for end users in this increment — everything ships dark and additive. (For instar
|
|
63
|
+
maintainers: a required per-callsite `judgesClaims` classification with a shrink-only ratchet and
|
|
64
|
+
a pinned JUDGE-SEED floor, so a claim-judging callsite can never silently ship un-benched against
|
|
65
|
+
the bare-claim / real-evidence axis pair.)
|
|
66
|
+
|
|
67
|
+
## Evidence
|
|
68
|
+
|
|
69
|
+
- `npx vitest run tests/unit/judges-claims-classification-ratchet.test.ts tests/unit/llm-bench-coverage-ratchet.test.ts`
|
|
70
|
+
→ **2 files, 13 tests, 0 failures** (required-explicit + no-dangling + valid-claimKind + the
|
|
71
|
+
JUDGE-SEED floor + shrink-only argued-false + the grounding canary; the pre-existing coverage
|
|
72
|
+
ratchet still green against the additive record). During authoring the ratchet caught three
|
|
73
|
+
unclassified callsites (TopicIntentArcCheck, InputClassifier, SessionSummarySentinel) and
|
|
74
|
+
failed the build until they were classified — the required-explicit guard doing its job.
|
|
75
|
+
- `npx tsc --noEmit` → exit 0. `npm run lint` → exit 0.
|
|
76
|
+
- Dark-by-construction: `LLM_JUDGES_CLAIMS` is build-time metadata read only by the new pinned
|
|
77
|
+
ratchet; it changes NO prompt text and wires NO runtime gate.
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# Side-Effects Review — Evidence-Bar Extension (defect class 3, dark increment)
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `evidence-bar-judge-extension`
|
|
4
|
+
**Date:** `2026-07-03`
|
|
5
|
+
**Author:** `echo`
|
|
6
|
+
**Tier:** `1 (dark increment — additive classification + one pinned ratchet; no runtime wiring, no operator-gated text)`
|
|
7
|
+
|
|
8
|
+
## Summary of the change
|
|
9
|
+
|
|
10
|
+
The mechanical arm of the "Evidence-Bar Extension to Judge Prompts" standard (defect class 3 / `claim-vs-evidence` closure; `docs/specs/evidence-bar-judge-extension.md`), shipped as a self-contained DARK increment. It ships:
|
|
11
|
+
|
|
12
|
+
1. **`src/data/llmBenchCoverage.ts`** (additive) — `LLM_JUDGES_CLAIMS`, the `judgesClaims` axis of the program's ONE shared per-callsite metadata record (sibling of the authority-clause `untrustedInput` axis, co-located in the same file). `ClaimKind` = `completionClaim | healthClaim | scoredCredit`; `JudgesClaimsFlag` = `{ claimKind }` (a judge) | `false` (not a judge) | `{ false: reason }` (a judge-shaped callsite argued out of scope). Required-explicit for every `COMPONENT_CATEGORY` key.
|
|
13
|
+
2. **`tests/unit/judges-claims-classification-ratchet.test.ts`** — required-explicit + no-dangling + valid-claimKind-per-judge + the spec-named JUDGE-SEED pinned as a judge + argued-false-shrink-only + real-reason floor. Same pinned-baseline family as the sibling ratchets.
|
|
14
|
+
3. **`docs/specs/evidence-bar-judge-extension.md`** + **`.eli16.md`** — the converged spec + plain-English overview (carried in with the closure so the standard's authority is on canonical main).
|
|
15
|
+
|
|
16
|
+
## What is DELIBERATELY out of scope (not orphan deferrals)
|
|
17
|
+
|
|
18
|
+
- **The registry amendment (spec §1).** Operator-gated — ships ONLY with Justin's explicit sign-off (spec front-matter `operator-gate`; the amendment is DRAFTED in spec §1). This run does not write `docs/STANDARDS-REGISTRY.md`.
|
|
19
|
+
- **The bench-axis pair ratchet (spec §3).** A bare-claim (false-accept) case + a real-evidence (false-reject) case per `judgesClaims` judge. Blocked on the program-wide "batteries readable by CI" decision — `research/` is absent from canonical main, so main's CI cannot read axis fields (`class-closure-gate.md` §"Program-shared machinery" #2, binding all three sibling axis specs). The sibling authority-clause axis ratchet deferred on the identical blocker; the keystone (#1347) staged its axis-requirements ratchet into its OWN dark increment for the same reason. Landing it under one standard's name would be doing shared cross-program infra as a side effect.
|
|
20
|
+
- **The `evidenceBar()` prompt clause (spec §4).** A sibling of the authority clause, defined by the spec to live in `src/core/promptClauses.ts`. That shared library is introduced by the still-OPEN sibling PR (authority-clause, #1351); creating it here would hard-collide on the same new file and break auto-merge for whichever PR merges second. The clause is consumed only by the A/B-gated per-component migrations (spec §4 / rollout §2), which are behavioral and deferred regardless — so shipping the clause function now would add a caller-less string with a merge hazard and no live value. It lands as a follow-up once the shared library is on main.
|
|
21
|
+
- **Per-component A/B clause migrations (rollout §2).** Behavioral prompt-text changes, each gated by the A/B protocol; the completion judge goes LAST (three wordings have already failed honestly — the named legitimate terminal state is "incumbent stands + routing mitigation + tracked gap"). A downstream task, not this dark increment.
|
|
22
|
+
|
|
23
|
+
## Decision-point inventory
|
|
24
|
+
|
|
25
|
+
- `LLM_JUDGES_CLAIMS` (`src/data/llmBenchCoverage.ts`) — **add** — build-time metadata only. Read by the new pinned ratchet; no runtime consumer. A SIGNAL producer (which callsites judge a completion/health claim), never a runtime authority.
|
|
26
|
+
- `tests/unit/judges-claims-classification-ratchet.test.ts` — **add** — a CI-only pinned baseline (same family as `llm-bench-coverage-ratchet`). It gates the build (adding an unclassified LLM callsite / mis-shaping a judge entry is red CI), never a runtime path.
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## 1. Over-block
|
|
31
|
+
|
|
32
|
+
**What legitimate inputs does this change reject that it shouldn't?**
|
|
33
|
+
|
|
34
|
+
None at runtime — nothing is wired to a runtime allow/deny path. The only new red-CI surfaces are build-time and intentional: (a) adding a new LLM component to `COMPONENT_CATEGORY` without an explicit `judgesClaims` classification, (b) shaping a judge entry with an invalid `claimKind`, (c) marking a spec-named JUDGE-SEED callsite as `false`, or (d) drifting the argued-false pinned baseline. Each is a self-describing failure with fix-instructions in the assertion message — the "visible, reviewed act" the standard exists to force, not an over-block of legitimate work.
|
|
35
|
+
|
|
36
|
+
## 2. Under-block
|
|
37
|
+
|
|
38
|
+
**What failure modes does this still miss?**
|
|
39
|
+
|
|
40
|
+
- The classification records WHICH callsites judge claims; it does NOT yet enforce that each judge carries the bare-claim + real-evidence axis PAIR (that is the deferred bench-axis ratchet, spec §3). So a classified judge can still ship with no false-accept/false-reject coverage until that ratchet lands. This is the known staged-rollout gap, tracked in the spec rollout, not a silent miss.
|
|
41
|
+
- Classification accuracy is author-judged (like the sibling `untrustedInput` axis). A judge mislabeled `false` beyond the spec-named seed would pass — the JUDGE-SEED floor + the argued-false shrink-only pin catch the measured judges and every judge-shaped exemption; the seeding PR IS that review.
|
|
42
|
+
- A prompt bar cannot AUTHENTICATE material — a fabricated transcript showing a fake PASS passes any in-prompt bar (spec "Honest reach"). The authoritative verification arm is the deterministic real-check `verification_command`, which is out of this classification by construction and named in the spec as the true-verification owner.
|
|
43
|
+
|
|
44
|
+
## 3. Level-of-abstraction fit
|
|
45
|
+
|
|
46
|
+
**Is this at the right layer?**
|
|
47
|
+
|
|
48
|
+
Yes. The classification extends the ONE shared per-callsite metadata record the program mandates (`src/data/llmBenchCoverage.ts`), not a new parallel registry — exactly where the sibling `untrustedInput` axis lives; the ratchet is in the established pinned-baseline vitest family. No higher gate already owns "which callsites judge a completion claim"; no lower primitive is duplicated.
|
|
49
|
+
|
|
50
|
+
## 4. Signal vs authority compliance
|
|
51
|
+
|
|
52
|
+
`LLM_JUDGES_CLAIMS` and the ratchet are SIGNALS (which callsites judge a claim; is each judge classified and axis-eligible) — they inform the build and the reviewer, never grant or deny a runtime action. No model-produced field is wired to satisfy any check. The (future, deferred) prompt clause, once consumed, would make a model REPORT that a bare claim is unverified rather than CREDIT it — it never becomes the authority; the deterministic real-check verifier owns what is TRUE.
|
|
53
|
+
|
|
54
|
+
## 5. Interactions with existing systems
|
|
55
|
+
|
|
56
|
+
- **`llm-bench-coverage-ratchet`** — unaffected: `LLM_JUDGES_CLAIMS` is a NEW additive export; the existing `LLM_BENCH_COVERAGE` union and its ratchet are untouched (both green post-change under a bounded run).
|
|
57
|
+
- **`untrusted-input-classification-ratchet` / `promptClauses.ts`** (sibling authority-clause, PR #1351, still open) — this increment adds a PARALLEL axis export in the same file and a parallel ratchet; it does not import or depend on the authority-clause work. Both append to `LLM_BENCH_COVERAGE`'s file region, so a textual rebase against #1351 may be needed depending on merge order — an additive, mechanical resolve, no semantic conflict.
|
|
58
|
+
- **`class-closure-lint.mjs`** — already names `src/data/llmBenchCoverage.ts` as an agent-authored artifact; touching it routes the PR to operator review (the lint is report-only today). The defect class `claim-vs-evidence` already carries `closureStandard: evidence-bar-judge-extension` on master — untouched here.
|
|
59
|
+
|
|
60
|
+
## 6. Failure modes / rollback
|
|
61
|
+
|
|
62
|
+
Pure additive TypeScript + one test + spec docs. Rollback = revert the commit; nothing persists state, nothing runs at runtime, no migration, no config key (spec §"Decision points touched": none at runtime; no agent-side config key exists or is wanted — repo posture only, so no Migration Parity work). tsc clean, `npm run lint` exit 0, both ratchets green under bounded single-file runs (machine under intermittent external CPU pressure; verification used bounded vitest per the CI-as-gate pattern, full suite gated by CI).
|
|
63
|
+
|
|
64
|
+
## 7. Second-pass reviewer
|
|
65
|
+
|
|
66
|
+
Self-review (bounded machine-pressure build). The change is additive, dark, and test-pinned; no runtime surface. Key self-checks: (a) confirmed the JUDGE-SEED is exactly the spec §2 named LLM judges (real-check verifier correctly excluded as the deterministic arm); (b) the two stall-confirm adapters (Telegram/Slack) classify true by the spec's stated inclusion criteria (stall/health classifiers), the safe over-inclusive direction; (c) the required-explicit ratchet actually fired during authoring (caught 3 unclassified callsites), proving it is not a no-op; (d) argued-false baseline is empty and the type still supports it for future judge-shaped exemptions.
|