@event4u/agent-config 9.1.0 → 9.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +38 -234
- package/README.md +1 -1
- package/dist/agent-src/commands/fix/route.md +49 -0
- package/dist/agent-src/commands/fix.md +4 -2
- package/dist/agent-src/commands/implement-ticket.md +2 -0
- package/dist/agent-src/commands/review/changes.md +22 -0
- package/dist/agent-src/commands/tdd/green.md +32 -0
- package/dist/agent-src/commands/tdd/red.md +33 -0
- package/dist/agent-src/commands/tdd/refactor.md +31 -0
- package/dist/agent-src/commands/tdd.md +50 -0
- package/dist/agent-src/contexts/communication/rules-auto/guidelines-mechanics.md +1 -0
- package/dist/agent-src/contexts/execution/subagent-modes-detail.md +41 -6
- package/dist/agent-src/contexts/execution/subagent-spawn-contract.md +81 -2
- package/dist/agent-src/contexts/execution/verify-budget.md +15 -0
- package/dist/agent-src/rules/architecture.md +7 -0
- package/dist/agent-src/rules/cross-source-consistency.md +92 -0
- package/dist/agent-src/rules/delegation-policy.md +1 -1
- package/dist/agent-src/skills/estimate-ticket/SKILL.md +2 -0
- package/dist/agent-src/skills/fe-design/SKILL.md +1 -0
- package/dist/agent-src/skills/feature-planning/SKILL.md +1 -1
- package/dist/agent-src/skills/judge-code-quality/SKILL.md +1 -0
- package/dist/agent-src/skills/persona-improvement/SKILL.md +81 -0
- package/dist/agent-src/skills/po-discovery/SKILL.md +2 -0
- package/dist/agent-src/skills/react-shadcn-ui/SKILL.md +10 -0
- package/dist/agent-src/skills/refine-ticket/SKILL.md +7 -0
- package/dist/agent-src/skills/roadmap-writing/SKILL.md +14 -0
- package/dist/agent-src/skills/subagent-orchestration/SKILL.md +38 -28
- package/dist/agent-src/skills/subagent-orchestration/evals/evals.json +18 -0
- package/dist/agent-src/skills/subagent-orchestration/prompts/README.md +3 -2
- package/dist/agent-src/skills/subagent-orchestration/prompts/adversarial-verification-council.md +79 -0
- package/dist/agent-src/skills/subagent-orchestration/schemas/adversarial-findings.json +56 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +205 -29
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +5 -5
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +19 -11
- package/dist/discovery/trust-report.md +4 -4
- package/dist/discovery/workspaces.json +16 -8
- package/dist/install/install.mjs +9 -0
- package/dist/mcp/registry-manifest.json +3 -3
- package/dist/router.json +1 -1
- package/dist/server/schemas/settings.js +5 -0
- package/dist/server/schemas/settings.js.map +1 -1
- package/docs/CLAIMS.md +10 -3
- package/docs/architecture.md +4 -4
- package/docs/archive/CHANGELOG-pre-9.2.0.md +276 -0
- package/docs/benchmark.md +24 -0
- package/docs/command-flows.md +9 -3
- package/docs/contracts/command-clusters.md +2 -1
- package/docs/decisions/ADR-122-adversarial-verification-council.md +136 -0
- package/docs/decisions/INDEX.md +1 -0
- package/docs/design/adversarial-council-eval.md +106 -0
- package/docs/development.md +10 -1
- package/docs/featured-skills.md +2 -2
- package/docs/getting-started-by-role.md +1 -1
- package/docs/getting-started.md +1 -1
- package/docs/governance-advantage.md +1 -1
- package/docs/guidelines/agent-infra/cross-source-consistency-mechanics.md +109 -0
- package/docs/guidelines/component-oriented-and-oop-development.md +181 -0
- package/docs/proof.md +7 -7
- package/package.json +3 -2
- package/src/config/agent-settings.template.yml +42 -0
- package/src/scripts/_lib/adversarial_council_gate.test.ts +112 -0
- package/src/scripts/_lib/adversarial_council_gate.ts +98 -0
- package/src/scripts/_lib/adversarial_reconcile.test.ts +197 -0
- package/src/scripts/_lib/adversarial_reconcile.ts +187 -0
- package/src/scripts/_lib/verify_budget.ts +4 -1
- package/src/scripts/ai_council/prompts.ts +7 -0
- package/src/scripts/hook_manifest.yaml +17 -6
- package/src/scripts/install-hooks.sh +32 -17
- package/src/scripts/pr_url_reminder_hook.ts +177 -0
- package/src/scripts/schemas/agent-settings.schema.json +11 -0
- package/src/server/schemas/settings.ts +9 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adversarial council prove-or-drop gate — coverage (Phase 4).
|
|
3
|
+
*
|
|
4
|
+
* Locks the pre-registered dual-threshold semantics: BOTH relative (>=+25%) and
|
|
5
|
+
* absolute (>=+8pp) residual-recall lift, AND panel FP not worse than baseline
|
|
6
|
+
* within noise. Guards the base-rate-compression case the council flagged
|
|
7
|
+
* (high single-judge baseline → relative easy but absolute must still clear).
|
|
8
|
+
*/
|
|
9
|
+
import { describe, expect, it } from 'vitest';
|
|
10
|
+
|
|
11
|
+
import {
|
|
12
|
+
ABSOLUTE_LIFT_THRESHOLD_PP,
|
|
13
|
+
RELATIVE_LIFT_THRESHOLD,
|
|
14
|
+
evaluateCouncilBench,
|
|
15
|
+
} from './adversarial_council_gate.js';
|
|
16
|
+
|
|
17
|
+
const CLEAN_FP = { single_judge_fp_rate: 0.05, panel_fp_rate: 0.05, fp_noise_margin: 0.03 };
|
|
18
|
+
|
|
19
|
+
describe('evaluateCouncilBench', () => {
|
|
20
|
+
it('exposes the locked thresholds', () => {
|
|
21
|
+
expect(RELATIVE_LIFT_THRESHOLD).toBe(0.25);
|
|
22
|
+
expect(ABSOLUTE_LIFT_THRESHOLD_PP).toBe(0.08);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('backs the claim when both recall thresholds and FP pass', () => {
|
|
26
|
+
const v = evaluateCouncilBench({
|
|
27
|
+
single_judge_residual_recall: 0.5,
|
|
28
|
+
panel_residual_recall: 0.65, // +30% relative, +15pp absolute
|
|
29
|
+
...CLEAN_FP,
|
|
30
|
+
});
|
|
31
|
+
expect(v.relative_pass).toBe(true);
|
|
32
|
+
expect(v.absolute_pass).toBe(true);
|
|
33
|
+
expect(v.fp_pass).toBe(true);
|
|
34
|
+
expect(v.verdict).toBe('backed');
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it('honest-null when relative passes but absolute fails (base-rate compression)', () => {
|
|
38
|
+
// baseline 0.90; +0.05 → relative 5.5% (fails) — the high-baseline trap.
|
|
39
|
+
const v = evaluateCouncilBench({
|
|
40
|
+
single_judge_residual_recall: 0.9,
|
|
41
|
+
panel_residual_recall: 0.95,
|
|
42
|
+
...CLEAN_FP,
|
|
43
|
+
});
|
|
44
|
+
expect(v.verdict).toBe('honest-null');
|
|
45
|
+
expect(v.reasons.join(' ')).toMatch(/relative lift/);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('honest-null when absolute passes but relative fails', () => {
|
|
49
|
+
// baseline 0.6; +0.09 absolute (passes) but +15% relative (fails at 25%).
|
|
50
|
+
const v = evaluateCouncilBench({
|
|
51
|
+
single_judge_residual_recall: 0.6,
|
|
52
|
+
panel_residual_recall: 0.69,
|
|
53
|
+
...CLEAN_FP,
|
|
54
|
+
});
|
|
55
|
+
expect(v.absolute_pass).toBe(true);
|
|
56
|
+
expect(v.relative_pass).toBe(false);
|
|
57
|
+
expect(v.verdict).toBe('honest-null');
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it('honest-null when recall passes but FP regresses beyond noise', () => {
|
|
61
|
+
const v = evaluateCouncilBench({
|
|
62
|
+
single_judge_residual_recall: 0.5,
|
|
63
|
+
panel_residual_recall: 0.7,
|
|
64
|
+
single_judge_fp_rate: 0.05,
|
|
65
|
+
panel_fp_rate: 0.12, // +7pp > 3pp noise
|
|
66
|
+
fp_noise_margin: 0.03,
|
|
67
|
+
});
|
|
68
|
+
expect(v.relative_pass).toBe(true);
|
|
69
|
+
expect(v.absolute_pass).toBe(true);
|
|
70
|
+
expect(v.fp_pass).toBe(false);
|
|
71
|
+
expect(v.verdict).toBe('honest-null');
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('allows a within-noise FP increase', () => {
|
|
75
|
+
const v = evaluateCouncilBench({
|
|
76
|
+
single_judge_residual_recall: 0.5,
|
|
77
|
+
panel_residual_recall: 0.7,
|
|
78
|
+
single_judge_fp_rate: 0.05,
|
|
79
|
+
panel_fp_rate: 0.075, // +2.5pp <= 3pp noise
|
|
80
|
+
fp_noise_margin: 0.03,
|
|
81
|
+
});
|
|
82
|
+
expect(v.fp_pass).toBe(true);
|
|
83
|
+
expect(v.verdict).toBe('backed');
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it('handles a zero baseline (infinite relative lift, absolute still gates)', () => {
|
|
87
|
+
const backed = evaluateCouncilBench({
|
|
88
|
+
single_judge_residual_recall: 0,
|
|
89
|
+
panel_residual_recall: 0.2,
|
|
90
|
+
...CLEAN_FP,
|
|
91
|
+
});
|
|
92
|
+
expect(backed.relative_lift).toBe(Infinity);
|
|
93
|
+
expect(backed.verdict).toBe('backed'); // +20pp absolute clears
|
|
94
|
+
|
|
95
|
+
const nullAbs = evaluateCouncilBench({
|
|
96
|
+
single_judge_residual_recall: 0,
|
|
97
|
+
panel_residual_recall: 0.05, // +5pp < 8pp
|
|
98
|
+
...CLEAN_FP,
|
|
99
|
+
});
|
|
100
|
+
expect(nullAbs.verdict).toBe('honest-null');
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
it('rejects out-of-range inputs', () => {
|
|
104
|
+
expect(() =>
|
|
105
|
+
evaluateCouncilBench({
|
|
106
|
+
single_judge_residual_recall: 1.5,
|
|
107
|
+
panel_residual_recall: 0.5,
|
|
108
|
+
...CLEAN_FP,
|
|
109
|
+
}),
|
|
110
|
+
).toThrow(/\[0,1\]/);
|
|
111
|
+
});
|
|
112
|
+
});
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adversarial verification council — prove-or-drop gate (Phase 4).
|
|
3
|
+
*
|
|
4
|
+
* Pure, deterministic. Encodes the pre-registered `adversarial-council-finding-
|
|
5
|
+
* coverage` claim (docs/CLAIMS.md, ADR-122) as a machine-checkable verdict on
|
|
6
|
+
* the two-stage residual-detection benchmark: does a cross-vendor skeptic panel
|
|
7
|
+
* out-find a single strong judge on the RESIDUAL defect pool (defects that
|
|
8
|
+
* survived stage 1), at a false-positive rate no worse than baseline on the
|
|
9
|
+
* controversial-but-correct control?
|
|
10
|
+
*
|
|
11
|
+
* Dual threshold, both required, to guard against base-rate compression when the
|
|
12
|
+
* single-judge residual recall is already high (the council's Round-1 critique):
|
|
13
|
+
* (a) relative residual-recall lift >= +25%
|
|
14
|
+
* (b) absolute residual-recall lift >= +8 percentage points
|
|
15
|
+
* AND panel FP not worse than single-judge FP within the noise margin.
|
|
16
|
+
*
|
|
17
|
+
* Thresholds are LOCKED at pre-registration (roadmap Phase 0). This module is the
|
|
18
|
+
* "countable logic in TS with tests" the anti-lesson mandates — never LLM-computed.
|
|
19
|
+
*
|
|
20
|
+
* Design: docs/design/adversarial-council-eval.md
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** Relative residual-recall lift threshold (fraction). */
|
|
24
|
+
export const RELATIVE_LIFT_THRESHOLD = 0.25;
|
|
25
|
+
/** Absolute residual-recall lift threshold (percentage points, as a fraction). */
|
|
26
|
+
export const ABSOLUTE_LIFT_THRESHOLD_PP = 0.08;
|
|
27
|
+
|
|
28
|
+
export interface CouncilBenchInputs {
|
|
29
|
+
/** Single-judge recall on the judge-passed residual pool (0..1). */
|
|
30
|
+
single_judge_residual_recall: number;
|
|
31
|
+
/** Panel recall on the same judge-passed residual pool (0..1). */
|
|
32
|
+
panel_residual_recall: number;
|
|
33
|
+
/** Single-judge false-positive rate on the controversial-clean control (0..1). */
|
|
34
|
+
single_judge_fp_rate: number;
|
|
35
|
+
/** Panel false-positive rate on the same control (0..1). */
|
|
36
|
+
panel_fp_rate: number;
|
|
37
|
+
/** Noise margin within which a higher panel FP still counts as "not worse" (0..1). */
|
|
38
|
+
fp_noise_margin: number;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export type CouncilVerdict = 'backed' | 'honest-null';
|
|
42
|
+
|
|
43
|
+
export interface CouncilBenchVerdict {
|
|
44
|
+
relative_lift: number;
|
|
45
|
+
absolute_lift_pp: number;
|
|
46
|
+
relative_pass: boolean;
|
|
47
|
+
absolute_pass: boolean;
|
|
48
|
+
fp_pass: boolean;
|
|
49
|
+
verdict: CouncilVerdict;
|
|
50
|
+
reasons: string[];
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function assertUnit(name: string, v: number): void {
|
|
54
|
+
if (typeof v !== 'number' || Number.isNaN(v) || v < 0 || v > 1) {
|
|
55
|
+
throw new Error(`${name} must be a number in [0,1], got ${JSON.stringify(v)}`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Evaluate the pre-registered dual-threshold gate. Returns `backed` only when
|
|
61
|
+
* BOTH recall thresholds AND the FP constraint pass; otherwise `honest-null`
|
|
62
|
+
* (which keeps the surface default-off permanently, like recursive-verification).
|
|
63
|
+
*/
|
|
64
|
+
export function evaluateCouncilBench(inputs: CouncilBenchInputs): CouncilBenchVerdict {
|
|
65
|
+
assertUnit('single_judge_residual_recall', inputs.single_judge_residual_recall);
|
|
66
|
+
assertUnit('panel_residual_recall', inputs.panel_residual_recall);
|
|
67
|
+
assertUnit('single_judge_fp_rate', inputs.single_judge_fp_rate);
|
|
68
|
+
assertUnit('panel_fp_rate', inputs.panel_fp_rate);
|
|
69
|
+
assertUnit('fp_noise_margin', inputs.fp_noise_margin);
|
|
70
|
+
|
|
71
|
+
const single = inputs.single_judge_residual_recall;
|
|
72
|
+
const panel = inputs.panel_residual_recall;
|
|
73
|
+
const absolute_lift_pp = panel - single;
|
|
74
|
+
// A zero baseline makes the ratio undefined — any positive panel recall is an
|
|
75
|
+
// infinite relative lift; no lift is zero. This never masks the absolute gate.
|
|
76
|
+
const relative_lift = single === 0 ? (panel > 0 ? Infinity : 0) : absolute_lift_pp / single;
|
|
77
|
+
|
|
78
|
+
const relative_pass = relative_lift >= RELATIVE_LIFT_THRESHOLD;
|
|
79
|
+
const absolute_pass = absolute_lift_pp >= ABSOLUTE_LIFT_THRESHOLD_PP;
|
|
80
|
+
const fp_pass = inputs.panel_fp_rate <= inputs.single_judge_fp_rate + inputs.fp_noise_margin;
|
|
81
|
+
|
|
82
|
+
const reasons: string[] = [];
|
|
83
|
+
if (!relative_pass) {
|
|
84
|
+
reasons.push(`relative lift ${(relative_lift * 100).toFixed(1)}% < ${RELATIVE_LIFT_THRESHOLD * 100}%`);
|
|
85
|
+
}
|
|
86
|
+
if (!absolute_pass) {
|
|
87
|
+
reasons.push(`absolute lift ${(absolute_lift_pp * 100).toFixed(1)}pp < ${ABSOLUTE_LIFT_THRESHOLD_PP * 100}pp`);
|
|
88
|
+
}
|
|
89
|
+
if (!fp_pass) {
|
|
90
|
+
reasons.push(
|
|
91
|
+
`panel FP ${(inputs.panel_fp_rate * 100).toFixed(1)}% > baseline ${(inputs.single_judge_fp_rate * 100).toFixed(1)}% + noise ${(inputs.fp_noise_margin * 100).toFixed(1)}%`,
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const verdict: CouncilVerdict = relative_pass && absolute_pass && fp_pass ? 'backed' : 'honest-null';
|
|
96
|
+
if (verdict === 'backed') reasons.push('all thresholds met');
|
|
97
|
+
return { relative_lift, absolute_lift_pp, relative_pass, absolute_pass, fp_pass, verdict, reasons };
|
|
98
|
+
}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adversarial reconciliation core — deterministic coverage (Phase 1).
|
|
3
|
+
*
|
|
4
|
+
* Covers: dedup across skeptics, severity aggregation (strict-er wins),
|
|
5
|
+
* quorum-confidence math, false-positive suppression (demote, never drop),
|
|
6
|
+
* empty panel, single-finding, unanimous vs split, malformed-input rejection,
|
|
7
|
+
* and hand-rolled validation of every output against
|
|
8
|
+
* adversarial-findings.json (no jsonschema runtime dep — mirrors
|
|
9
|
+
* test_subagent_status_schema.py's approach).
|
|
10
|
+
*/
|
|
11
|
+
import { readFileSync } from 'node:fs';
|
|
12
|
+
import { fileURLToPath } from 'node:url';
|
|
13
|
+
import path from 'node:path';
|
|
14
|
+
|
|
15
|
+
import { describe, expect, it } from 'vitest';
|
|
16
|
+
|
|
17
|
+
import {
|
|
18
|
+
type AdversarialFindings,
|
|
19
|
+
type RawFinding,
|
|
20
|
+
type SkepticReturn,
|
|
21
|
+
assertValidRawFinding,
|
|
22
|
+
findingKey,
|
|
23
|
+
isSuppressedFalsePositive,
|
|
24
|
+
reconcileFindings,
|
|
25
|
+
severityQuorum,
|
|
26
|
+
} from './adversarial_reconcile.js';
|
|
27
|
+
|
|
28
|
+
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
|
29
|
+
const SCHEMA_PATH = path.resolve(
|
|
30
|
+
HERE,
|
|
31
|
+
'../../skills/subagent-orchestration/schemas/adversarial-findings.json',
|
|
32
|
+
);
|
|
33
|
+
|
|
34
|
+
const SEVERITIES = new Set(['critical', 'high', 'medium', 'low']);
|
|
35
|
+
const CONFIDENCES = new Set(['high', 'medium', 'low']);
|
|
36
|
+
|
|
37
|
+
/** Minimal hand-rolled validator matching the schema's required keys + enums. */
|
|
38
|
+
function validateEnvelope(env: AdversarialFindings): void {
|
|
39
|
+
expect(env).toHaveProperty('panel');
|
|
40
|
+
expect(env.panel.models.length).toBeGreaterThanOrEqual(1);
|
|
41
|
+
expect(env.panel.skeptic_count).toBe(env.panel.models.length);
|
|
42
|
+
for (const bucket of [env.findings, env.false_positives_suppressed]) {
|
|
43
|
+
for (const f of bucket) {
|
|
44
|
+
for (const val of [f.id, f.severity, f.category, f.location, f.description, f.confidence]) {
|
|
45
|
+
expect(typeof val).toBe('string');
|
|
46
|
+
expect(val.length).toBeGreaterThan(0);
|
|
47
|
+
}
|
|
48
|
+
expect(SEVERITIES.has(f.severity)).toBe(true);
|
|
49
|
+
expect(CONFIDENCES.has(f.confidence)).toBe(true);
|
|
50
|
+
expect(Array.isArray(f.raised_by)).toBe(true);
|
|
51
|
+
expect(f.raised_by.length).toBeGreaterThanOrEqual(1);
|
|
52
|
+
expect(Array.isArray(f.refuted_by)).toBe(true);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function raw(over: Partial<RawFinding> = {}): RawFinding {
|
|
58
|
+
return {
|
|
59
|
+
severity: 'high',
|
|
60
|
+
category: 'correctness',
|
|
61
|
+
location: 'src/a.ts:42',
|
|
62
|
+
description: 'off-by-one in loop bound',
|
|
63
|
+
...over,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
describe('adversarial-findings.json schema', () => {
|
|
68
|
+
it('parses and declares the finding contract', () => {
|
|
69
|
+
const schema = JSON.parse(readFileSync(SCHEMA_PATH, 'utf8'));
|
|
70
|
+
expect(schema.required).toEqual(['panel', 'findings', 'false_positives_suppressed']);
|
|
71
|
+
expect(schema.additionalProperties).toBe(false);
|
|
72
|
+
expect(schema.definitions.finding.required).toContain('raised_by');
|
|
73
|
+
expect(schema.definitions.finding.required).toContain('confidence');
|
|
74
|
+
expect(schema.definitions.finding.properties.severity.enum).toEqual([
|
|
75
|
+
'critical',
|
|
76
|
+
'high',
|
|
77
|
+
'medium',
|
|
78
|
+
'low',
|
|
79
|
+
]);
|
|
80
|
+
});
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
describe('findingKey', () => {
|
|
84
|
+
it('is location + normalized category', () => {
|
|
85
|
+
expect(findingKey({ location: 'src/a.ts:42 ', category: 'Security' })).toBe(
|
|
86
|
+
'src/a.ts:42::security',
|
|
87
|
+
);
|
|
88
|
+
});
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
describe('assertValidRawFinding', () => {
|
|
92
|
+
it('rejects a bad severity', () => {
|
|
93
|
+
expect(() => assertValidRawFinding(raw({ severity: 'blocker' as never }))).toThrow(/severity/);
|
|
94
|
+
});
|
|
95
|
+
it('rejects an empty location', () => {
|
|
96
|
+
expect(() => assertValidRawFinding(raw({ location: ' ' }))).toThrow(/location/);
|
|
97
|
+
});
|
|
98
|
+
it('accepts a well-formed finding', () => {
|
|
99
|
+
expect(() => assertValidRawFinding(raw())).not.toThrow();
|
|
100
|
+
});
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
describe('severityQuorum', () => {
|
|
104
|
+
it('high when corroborated by a panel quorum', () => {
|
|
105
|
+
expect(severityQuorum(2, 0, 2)).toBe('high');
|
|
106
|
+
expect(severityQuorum(2, 0, 3)).toBe('high');
|
|
107
|
+
});
|
|
108
|
+
it('medium when corroborated below quorum', () => {
|
|
109
|
+
expect(severityQuorum(2, 0, 5)).toBe('medium');
|
|
110
|
+
});
|
|
111
|
+
it('medium for a lone raiser on a solo panel', () => {
|
|
112
|
+
expect(severityQuorum(1, 0, 1)).toBe('medium');
|
|
113
|
+
});
|
|
114
|
+
it('low for a single raiser on a multi-skeptic panel', () => {
|
|
115
|
+
expect(severityQuorum(1, 0, 3)).toBe('low');
|
|
116
|
+
});
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
describe('isSuppressedFalsePositive', () => {
|
|
120
|
+
it('suppresses a lone finding refuted by a strict majority of the rest', () => {
|
|
121
|
+
expect(isSuppressedFalsePositive(1, 1, 2)).toBe(true); // rest=1, refuted=1 > 0.5
|
|
122
|
+
expect(isSuppressedFalsePositive(1, 2, 3)).toBe(true); // rest=2, refuted=2 > 1
|
|
123
|
+
});
|
|
124
|
+
it('does not suppress when refutation is not a strict majority', () => {
|
|
125
|
+
expect(isSuppressedFalsePositive(1, 1, 3)).toBe(false); // rest=2, refuted=1 == 1, not > 1
|
|
126
|
+
});
|
|
127
|
+
it('never suppresses a corroborated finding', () => {
|
|
128
|
+
expect(isSuppressedFalsePositive(2, 5, 7)).toBe(false);
|
|
129
|
+
});
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
describe('reconcileFindings', () => {
|
|
133
|
+
it('handles an empty panel', () => {
|
|
134
|
+
const env = reconcileFindings([]);
|
|
135
|
+
expect(env.panel).toEqual({ models: [], skeptic_count: 0 });
|
|
136
|
+
expect(env.findings).toEqual([]);
|
|
137
|
+
expect(env.false_positives_suppressed).toEqual([]);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
it('dedups the same defect across skeptics and records both as raisers', () => {
|
|
141
|
+
const env = reconcileFindings([
|
|
142
|
+
{ model: 'anthropic:x', findings: [raw()] },
|
|
143
|
+
{ model: 'openai:y', findings: [raw({ description: 'the loop reads one past the end of the array' })] },
|
|
144
|
+
]);
|
|
145
|
+
expect(env.findings).toHaveLength(1);
|
|
146
|
+
expect(env.findings[0]!.raised_by.sort()).toEqual(['anthropic:x', 'openai:y']);
|
|
147
|
+
expect(env.findings[0]!.confidence).toBe('high'); // 2/2 = quorum
|
|
148
|
+
// Longest description wins.
|
|
149
|
+
expect(env.findings[0]!.description).toMatch(/one past the end/);
|
|
150
|
+
validateEnvelope(env);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
it('aggregates severity strict-er-wins', () => {
|
|
154
|
+
const env = reconcileFindings([
|
|
155
|
+
{ model: 'a', findings: [raw({ severity: 'medium' })] },
|
|
156
|
+
{ model: 'b', findings: [raw({ severity: 'critical' })] },
|
|
157
|
+
]);
|
|
158
|
+
expect(env.findings[0]!.severity).toBe('critical');
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
it('takes the union of distinct findings and orders by severity then key', () => {
|
|
162
|
+
const env = reconcileFindings([
|
|
163
|
+
{ model: 'a', findings: [raw({ location: 'src/a.ts:1', severity: 'low', category: 'quality' })] },
|
|
164
|
+
{ model: 'b', findings: [raw({ location: 'src/b.ts:9', severity: 'critical', category: 'security' })] },
|
|
165
|
+
]);
|
|
166
|
+
expect(env.findings.map((f) => f.severity)).toEqual(['critical', 'low']);
|
|
167
|
+
expect(env.findings.map((f) => f.id)).toEqual(['avc-001', 'avc-002']);
|
|
168
|
+
validateEnvelope(env);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
it('demotes a lone finding refuted by a majority into false_positives_suppressed', () => {
|
|
172
|
+
const loneKey = findingKey({ location: 'src/c.ts:5', category: 'security' });
|
|
173
|
+
const env = reconcileFindings([
|
|
174
|
+
{ model: 'a', findings: [raw({ location: 'src/c.ts:5', category: 'security' })] },
|
|
175
|
+
{ model: 'b', findings: [], refutes: [loneKey] },
|
|
176
|
+
]);
|
|
177
|
+
expect(env.findings).toHaveLength(0);
|
|
178
|
+
expect(env.false_positives_suppressed).toHaveLength(1);
|
|
179
|
+
expect(env.false_positives_suppressed[0]!.refuted_by).toEqual(['b']);
|
|
180
|
+
validateEnvelope(env);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
it('is deterministic across input order', () => {
|
|
184
|
+
const a: SkepticReturn = { model: 'a', findings: [raw({ location: 'src/z.ts:1', severity: 'high' })] };
|
|
185
|
+
const b: SkepticReturn = { model: 'b', findings: [raw({ location: 'src/a.ts:1', severity: 'high', category: 'security' })] };
|
|
186
|
+
const one = reconcileFindings([a, b]);
|
|
187
|
+
const two = reconcileFindings([b, a]);
|
|
188
|
+
expect(one.findings.map((f) => f.location)).toEqual(two.findings.map((f) => f.location));
|
|
189
|
+
expect(one.findings.map((f) => f.id)).toEqual(two.findings.map((f) => f.id));
|
|
190
|
+
});
|
|
191
|
+
|
|
192
|
+
it('rejects malformed findings from a skeptic', () => {
|
|
193
|
+
expect(() =>
|
|
194
|
+
reconcileFindings([{ model: 'a', findings: [raw({ severity: 'nope' as never })] }]),
|
|
195
|
+
).toThrow(/severity/);
|
|
196
|
+
});
|
|
197
|
+
});
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Adversarial verification council — reconciliation core (Phase 1).
|
|
3
|
+
*
|
|
4
|
+
* Pure, no-I/O, DETERMINISTIC (no Date/random — ids come from sorted key order).
|
|
5
|
+
* Turns per-skeptic finding lists into one `AdversarialFindings` envelope with
|
|
6
|
+
* per-finding provenance (which models raised it) and a cross-model confidence
|
|
7
|
+
* signal. This is the countable heart the ADR mandates lives in TS with tests,
|
|
8
|
+
* never as LLM-computed prose.
|
|
9
|
+
*
|
|
10
|
+
* Contract: it VERIFIES (ranks + annotates findings). It NEVER gates the change
|
|
11
|
+
* — the panel is advisory only (Hard Floor). "Suppression" of a false positive
|
|
12
|
+
* demotes a finding into `false_positives_suppressed`; it is never silently
|
|
13
|
+
* dropped.
|
|
14
|
+
*
|
|
15
|
+
* Schema: src/skills/subagent-orchestration/schemas/adversarial-findings.json
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
export type Severity = 'critical' | 'high' | 'medium' | 'low';
|
|
19
|
+
export type Confidence = 'high' | 'medium' | 'low';
|
|
20
|
+
|
|
21
|
+
/** Worst → best; index is the rank used for severity aggregation + sorting. */
|
|
22
|
+
const SEVERITY_ORDER: readonly Severity[] = ['critical', 'high', 'medium', 'low'];
|
|
23
|
+
|
|
24
|
+
/** A single defect as one skeptic reports it. */
|
|
25
|
+
export interface RawFinding {
|
|
26
|
+
severity: Severity;
|
|
27
|
+
category: string;
|
|
28
|
+
location: string;
|
|
29
|
+
description: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** One skeptic's return: what it raised, plus keys it examined and judged clean. */
|
|
33
|
+
export interface SkepticReturn {
|
|
34
|
+
model: string;
|
|
35
|
+
findings: RawFinding[];
|
|
36
|
+
/** `findingKey` values this skeptic actively refuted (examined → not a defect). */
|
|
37
|
+
refutes?: string[];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface Finding {
|
|
41
|
+
id: string;
|
|
42
|
+
severity: Severity;
|
|
43
|
+
category: string;
|
|
44
|
+
location: string;
|
|
45
|
+
description: string;
|
|
46
|
+
raised_by: string[];
|
|
47
|
+
refuted_by: string[];
|
|
48
|
+
confidence: Confidence;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface AdversarialFindings {
|
|
52
|
+
panel: { models: string[]; skeptic_count: number };
|
|
53
|
+
findings: Finding[];
|
|
54
|
+
false_positives_suppressed: Finding[];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Dedup key — same location + same category is the same defect. */
|
|
58
|
+
export function findingKey(f: Pick<RawFinding, 'location' | 'category'>): string {
|
|
59
|
+
return `${f.location.trim()}::${f.category.trim().toLowerCase()}`;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Throw on a malformed raw finding — the reconciler refuses garbage in. */
|
|
63
|
+
export function assertValidRawFinding(f: RawFinding): void {
|
|
64
|
+
if (!SEVERITY_ORDER.includes(f.severity)) {
|
|
65
|
+
throw new Error(`invalid severity: ${JSON.stringify(f.severity)}`);
|
|
66
|
+
}
|
|
67
|
+
for (const [k, v] of Object.entries({ category: f.category, location: f.location, description: f.description })) {
|
|
68
|
+
if (typeof v !== 'string' || v.trim().length === 0) {
|
|
69
|
+
throw new Error(`finding.${k} must be a non-empty string`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Most-severe (lowest index) of two severities. */
|
|
75
|
+
function moreSevere(a: Severity, b: Severity): Severity {
|
|
76
|
+
return SEVERITY_ORDER.indexOf(a) <= SEVERITY_ORDER.indexOf(b) ? a : b;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Confidence from corroboration:
|
|
81
|
+
* - `high` — raised by >=2 skeptics AND by at least half the panel (quorum).
|
|
82
|
+
* - `medium` — raised by >=2 (below quorum), OR the lone raiser on a solo panel.
|
|
83
|
+
* - `low` — a single raiser on a multi-skeptic panel (uncorroborated doubt).
|
|
84
|
+
*/
|
|
85
|
+
export function severityQuorum(raisedCount: number, refutedCount: number, panelSize: number): Confidence {
|
|
86
|
+
if (raisedCount >= 2 && raisedCount * 2 >= panelSize) return 'high';
|
|
87
|
+
if (raisedCount >= 2) return 'medium';
|
|
88
|
+
if (panelSize === 1 && refutedCount === 0) return 'medium';
|
|
89
|
+
return 'low';
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* A finding is a suppressible false positive when a single skeptic raised it and
|
|
94
|
+
* a strict majority of the OTHER skeptics actively refuted it. Demote, never drop.
|
|
95
|
+
*/
|
|
96
|
+
export function isSuppressedFalsePositive(raisedCount: number, refutedCount: number, panelSize: number): boolean {
|
|
97
|
+
if (raisedCount !== 1) return false;
|
|
98
|
+
const others = panelSize - 1;
|
|
99
|
+
return others > 0 && refutedCount * 2 > others;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
interface Aggregate {
|
|
103
|
+
key: string;
|
|
104
|
+
severity: Severity;
|
|
105
|
+
category: string;
|
|
106
|
+
location: string;
|
|
107
|
+
description: string;
|
|
108
|
+
raised_by: string[];
|
|
109
|
+
refuted_by: string[];
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Reconcile per-skeptic returns into one advisory findings envelope.
|
|
114
|
+
* Deterministic: findings are id'd and ordered by (severity rank, key).
|
|
115
|
+
*/
|
|
116
|
+
export function reconcileFindings(returns: SkepticReturn[]): AdversarialFindings {
|
|
117
|
+
const models = returns.map((r) => r.model);
|
|
118
|
+
const skepticCount = returns.length;
|
|
119
|
+
const byKey = new Map<string, Aggregate>();
|
|
120
|
+
|
|
121
|
+
for (const ret of returns) {
|
|
122
|
+
for (const raw of ret.findings) {
|
|
123
|
+
assertValidRawFinding(raw);
|
|
124
|
+
const key = findingKey(raw);
|
|
125
|
+
const existing = byKey.get(key);
|
|
126
|
+
if (existing) {
|
|
127
|
+
existing.severity = moreSevere(existing.severity, raw.severity);
|
|
128
|
+
if (!existing.raised_by.includes(ret.model)) existing.raised_by.push(ret.model);
|
|
129
|
+
// Keep the longest description — usually the most informative.
|
|
130
|
+
if (raw.description.trim().length > existing.description.length) {
|
|
131
|
+
existing.description = raw.description.trim();
|
|
132
|
+
}
|
|
133
|
+
} else {
|
|
134
|
+
byKey.set(key, {
|
|
135
|
+
key,
|
|
136
|
+
severity: raw.severity,
|
|
137
|
+
category: raw.category.trim(),
|
|
138
|
+
location: raw.location.trim(),
|
|
139
|
+
description: raw.description.trim(),
|
|
140
|
+
raised_by: [ret.model],
|
|
141
|
+
refuted_by: [],
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Second pass: attach refutations only to findings that were actually raised.
|
|
148
|
+
for (const ret of returns) {
|
|
149
|
+
for (const refutedKey of ret.refutes ?? []) {
|
|
150
|
+
const agg = byKey.get(refutedKey);
|
|
151
|
+
if (agg && !agg.raised_by.includes(ret.model) && !agg.refuted_by.includes(ret.model)) {
|
|
152
|
+
agg.refuted_by.push(ret.model);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const ordered = [...byKey.values()].sort((a, b) => {
|
|
158
|
+
const bySeverity = SEVERITY_ORDER.indexOf(a.severity) - SEVERITY_ORDER.indexOf(b.severity);
|
|
159
|
+
return bySeverity !== 0 ? bySeverity : a.key.localeCompare(b.key);
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
const findings: Finding[] = [];
|
|
163
|
+
const suppressed: Finding[] = [];
|
|
164
|
+
ordered.forEach((agg, i) => {
|
|
165
|
+
const finding: Finding = {
|
|
166
|
+
id: `avc-${String(i + 1).padStart(3, '0')}`,
|
|
167
|
+
severity: agg.severity,
|
|
168
|
+
category: agg.category,
|
|
169
|
+
location: agg.location,
|
|
170
|
+
description: agg.description,
|
|
171
|
+
raised_by: agg.raised_by,
|
|
172
|
+
refuted_by: agg.refuted_by,
|
|
173
|
+
confidence: severityQuorum(agg.raised_by.length, agg.refuted_by.length, skepticCount),
|
|
174
|
+
};
|
|
175
|
+
if (isSuppressedFalsePositive(agg.raised_by.length, agg.refuted_by.length, skepticCount)) {
|
|
176
|
+
suppressed.push(finding);
|
|
177
|
+
} else {
|
|
178
|
+
findings.push(finding);
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
return {
|
|
183
|
+
panel: { models, skeptic_count: skepticCount },
|
|
184
|
+
findings,
|
|
185
|
+
false_positives_suppressed: suppressed,
|
|
186
|
+
};
|
|
187
|
+
}
|
|
@@ -13,7 +13,10 @@
|
|
|
13
13
|
* silent pass.
|
|
14
14
|
*/
|
|
15
15
|
|
|
16
|
-
|
|
16
|
+
// 'council' is the opt-in adversarial-verification-council escalation (Mode 9,
|
|
17
|
+
// ADR-122): default-off, advisory, recorded but never returned by
|
|
18
|
+
// selectVerifyMode — the orchestrator chooses it for high-risk changes.
|
|
19
|
+
export type VerifyMode = 'deterministic' | 'judge' | 'none' | 'council';
|
|
17
20
|
|
|
18
21
|
/** Max changed-line count (inclusive) that still counts as trivial. */
|
|
19
22
|
export const TRIVIAL_CHANGE_FLOOR = 20;
|
|
@@ -88,6 +88,13 @@ correctness lens AND a shipping-risk lens:
|
|
|
88
88
|
Is the blast radius bigger than the title implies?
|
|
89
89
|
5. Reviewer fatigue — is anything in the diff that a tired reviewer
|
|
90
90
|
would rubber-stamp but should not?
|
|
91
|
+
6. Reuse & object-oriented shape — does the diff reinvent a component
|
|
92
|
+
or abstraction the codebase likely already provides (should compose
|
|
93
|
+
/ reuse instead), and where it adds structure, is the OO shape
|
|
94
|
+
(encapsulation, composition over inheritance) justified by
|
|
95
|
+
complexity it removes rather than speculative abstraction? Judge in
|
|
96
|
+
the codebase's own paradigm — do not push classes onto functional /
|
|
97
|
+
pipeline code; smallest-change still wins over OO advocacy.
|
|
91
98
|
End with: APPROVE / REQUEST_CHANGES / REJECT, one sentence why, and
|
|
92
99
|
the single highest-leverage change the PR author should make before
|
|
93
100
|
merge.`;
|