@event4u/agent-config 8.8.0 → 8.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/CHANGELOG.md +139 -263
- package/README.md +5 -5
- package/dist/agent-src/commands/agent-handoff.md +39 -2
- package/dist/agent-src/commands/cost/report.md +13 -3
- package/dist/agent-src/commands/research/deep.md +8 -0
- package/dist/agent-src/commands/research/report.md +7 -0
- package/dist/agent-src/commands/roadmap/create.md +12 -0
- package/dist/agent-src/commands/roadmap/process-full.md +31 -5
- package/dist/agent-src/commands/tests/create.md +28 -6
- package/dist/agent-src/commands/work.md +33 -0
- package/dist/agent-src/commands/worktree/cleanup.md +78 -0
- package/dist/agent-src/commands/worktree/create.md +62 -0
- package/dist/agent-src/commands/worktree/status.md +63 -0
- package/dist/agent-src/commands/worktree/verify.md +70 -0
- package/dist/agent-src/commands/worktree.md +85 -0
- package/dist/agent-src/contexts/communication/rules-auto/think-before-action-mechanics.md +30 -1
- package/dist/agent-src/contexts/execution/orchestration-telemetry.md +60 -31
- package/dist/agent-src/contexts/execution/roadmap-process-loop.md +37 -0
- package/dist/agent-src/contexts/execution/subagent-topologies.md +34 -0
- package/dist/agent-src/contexts/execution/verification-mechanics.md +9 -9
- package/dist/agent-src/rules/agent-authority.md +1 -1
- package/dist/agent-src/rules/communication-through-line.md +1 -1
- package/dist/agent-src/rules/delegation-policy.md +5 -0
- package/dist/agent-src/rules/design-fidelity.md +40 -0
- package/dist/agent-src/rules/direct-answers.md +1 -1
- package/dist/agent-src/rules/downstream-changes.md +1 -0
- package/dist/agent-src/rules/external-code-graph-interop.md +65 -0
- package/dist/agent-src/rules/git-history-discipline.md +18 -0
- package/dist/agent-src/rules/lethal-trifecta-guard.md +6 -1
- package/dist/agent-src/rules/minimal-safe-diff.md +58 -10
- package/dist/agent-src/rules/non-destructive-by-default.md +3 -0
- package/dist/agent-src/rules/question-not-instruction.md +76 -0
- package/dist/agent-src/rules/senior-engineering-discipline.md +1 -1
- package/dist/agent-src/rules/spreadsheet-source-quality.md +61 -0
- package/dist/agent-src/rules/think-before-action.md +1 -0
- package/dist/agent-src/rules/tool-safety.md +8 -0
- package/dist/agent-src/rules/untrusted-input-defense.md +87 -22
- package/dist/agent-src/scripts/update_roadmap_progress.ts +2 -1
- package/dist/agent-src/skills/accessibility-auditor/SKILL.md +19 -0
- package/dist/agent-src/skills/accessibility-auditor/evals/evals.json +15 -0
- package/dist/agent-src/skills/ai-code-blindspots/SKILL.md +10 -3
- package/dist/agent-src/skills/analysis-skill-router/evals/evals.json +13 -0
- package/dist/agent-src/skills/api-testing/SKILL.md +15 -1
- package/dist/agent-src/skills/character-consistency/evals/evals.json +16 -0
- package/dist/agent-src/skills/code-review/SKILL.md +2 -0
- package/dist/agent-src/skills/command-routing/evals/evals.json +14 -0
- package/dist/agent-src/skills/command-writing/SKILL.md +53 -40
- package/dist/agent-src/skills/dashboard-design/evals/evals.json +16 -0
- package/dist/agent-src/skills/dcf-modeling/evals/domain-truth.json +29 -0
- package/dist/agent-src/skills/dcf-modeling/evals/evals.json +20 -0
- package/dist/agent-src/skills/decision-record/evals/evals.json +20 -0
- package/dist/agent-src/skills/deep-reading-analyst/SKILL.md +1 -0
- package/dist/agent-src/skills/design-intelligence/SKILL.md +49 -12
- package/dist/agent-src/skills/design-intelligence/evals/evals.json +14 -0
- package/dist/agent-src/skills/design-review/SKILL.md +35 -15
- package/dist/agent-src/skills/design-system-capture/SKILL.md +84 -24
- package/dist/agent-src/skills/design-system-capture/evals/evals.json +15 -0
- package/dist/agent-src/skills/developer-like-execution/evals/evals.json +16 -0
- package/dist/agent-src/skills/doc-coauthoring/SKILL.md +27 -0
- package/dist/agent-src/skills/doc-coauthoring/evals/evals.json +16 -0
- package/dist/agent-src/skills/editorial-calendar/evals/domain-truth.json +16 -0
- package/dist/agent-src/skills/editorial-calendar/evals/evals.json +16 -0
- package/dist/agent-src/skills/estimate-ticket/evals/evals.json +16 -0
- package/dist/agent-src/skills/existing-ui-audit/SKILL.md +30 -0
- package/dist/agent-src/skills/existing-ui-audit/evals/triggers.json +47 -0
- package/dist/agent-src/skills/fe-design/SKILL.md +36 -5
- package/dist/agent-src/skills/fe-design/evals/triggers.json +47 -0
- package/dist/agent-src/skills/forecasting/evals/domain-truth.json +29 -0
- package/dist/agent-src/skills/forecasting/evals/evals.json +20 -0
- package/dist/agent-src/skills/frontend-render-security/SKILL.md +2 -2
- package/dist/agent-src/skills/fundraising-narrative/evals/domain-truth.json +16 -0
- package/dist/agent-src/skills/fundraising-narrative/evals/evals.json +16 -0
- package/dist/agent-src/skills/html-deck/SKILL.md +27 -0
- package/dist/agent-src/skills/iconography/SKILL.md +24 -0
- package/dist/agent-src/skills/incident-commander/evals/domain-truth.json +16 -0
- package/dist/agent-src/skills/incident-commander/evals/evals.json +20 -0
- package/dist/agent-src/skills/launch-readiness/evals/evals.json +21 -0
- package/dist/agent-src/skills/logging-monitoring/evals/evals.json +16 -0
- package/dist/agent-src/skills/markitdown/SKILL.md +1 -0
- package/dist/agent-src/skills/mcp/SKILL.md +24 -0
- package/dist/agent-src/skills/mcp-builder/SKILL.md +9 -1
- package/dist/agent-src/skills/memory-consolidation/SKILL.md +55 -4
- package/dist/agent-src/skills/messaging-architecture/evals/domain-truth.json +16 -0
- package/dist/agent-src/skills/messaging-architecture/evals/evals.json +16 -0
- package/dist/agent-src/skills/perf-feedback-craft/evals/evals.json +16 -0
- package/dist/agent-src/skills/pest-testing/SKILL.md +1 -1
- package/dist/agent-src/skills/playwright-testing/SKILL.md +16 -0
- package/dist/agent-src/skills/prompt-engineering-patterns/evals/evals.json +16 -0
- package/dist/agent-src/skills/refine-prompt/evals/evals.json +16 -0
- package/dist/agent-src/skills/refine-ticket/evals/evals.json +16 -0
- package/dist/agent-src/skills/release-comms/evals/evals.json +16 -0
- package/dist/agent-src/skills/rice-prioritization/evals/evals.json +20 -0
- package/dist/agent-src/skills/roadmap-writing/SKILL.md +39 -5
- package/dist/agent-src/skills/runway-cognition/evals/domain-truth.json +29 -0
- package/dist/agent-src/skills/runway-cognition/evals/evals.json +20 -0
- package/dist/agent-src/skills/scenario-modeling/evals/domain-truth.json +29 -0
- package/dist/agent-src/skills/scenario-modeling/evals/evals.json +20 -0
- package/dist/agent-src/skills/secrets-management/SKILL.md +7 -0
- package/dist/agent-src/skills/skill-writing/SKILL.md +12 -3
- package/dist/agent-src/skills/spreadsheet-authoring/SKILL.md +98 -0
- package/dist/agent-src/skills/spreadsheet-authoring/evals/triggers.json +47 -0
- package/dist/agent-src/skills/subagent-orchestration/SKILL.md +27 -33
- package/dist/agent-src/skills/supply-chain-intake/SKILL.md +12 -0
- package/dist/agent-src/skills/systematic-debugging/evals/evals.json +20 -0
- package/dist/agent-src/skills/test-case-discovery/SKILL.md +118 -0
- package/dist/agent-src/skills/test-case-discovery/evals/triggers.json +52 -0
- package/dist/agent-src/skills/test-driven-development/SKILL.md +32 -6
- package/dist/agent-src/skills/test-driven-development/evals/evals.json +16 -0
- package/dist/agent-src/skills/testing-anti-patterns/SKILL.md +34 -4
- package/dist/agent-src/skills/threat-modeling/evals/evals.json +21 -0
- package/dist/agent-src/skills/typography-system/evals/evals.json +14 -0
- package/dist/agent-src/skills/ui-component-architect/SKILL.md +9 -0
- package/dist/agent-src/skills/unit-economics-modeling/evals/domain-truth.json +40 -0
- package/dist/agent-src/skills/unit-economics-modeling/evals/evals.json +24 -0
- package/dist/agent-src/skills/verify-completion-evidence/evals/evals.json +16 -0
- package/dist/agent-src/skills/vision-articulation/evals/evals.json +16 -0
- package/dist/agent-src/skills/voice-and-tone-design/evals/evals.json +16 -0
- package/dist/agent-src/skills/worktree-lifecycle/SKILL.md +180 -0
- package/dist/agent-src/skills/worktree-lifecycle/evals/triggers.json +52 -0
- package/dist/agent-src/templates/agents/agent-project-settings.example.yml +1 -1
- package/dist/agent-src/templates/roadmaps.md +36 -0
- package/dist/cli/registry.js +3 -0
- package/dist/cli/registry.js.map +1 -1
- package/dist/discovery/deprecation-report.md +1 -1
- package/dist/discovery/discovery-manifest.json +326 -67
- package/dist/discovery/discovery-manifest.json.sha256 +1 -1
- package/dist/discovery/discovery-manifest.summary.md +6 -6
- package/dist/discovery/orphan-report.md +1 -1
- package/dist/discovery/packs.json +25 -14
- package/dist/discovery/trust-report.md +4 -4
- package/dist/discovery/workspaces.json +25 -12
- package/dist/install/install.mjs +136 -1
- package/dist/mcp/registry-manifest.json +3 -3
- package/dist/router.json +1 -1
- package/dist/scripts/_lib/claude_builtin_names.js +145 -0
- package/dist/scripts/_lib/claude_builtin_names.js.map +1 -0
- package/dist/scripts/condense.js +14 -0
- package/dist/scripts/condense.js.map +1 -1
- package/docs/CLAIMS.md +36 -8
- package/docs/architecture.md +4 -4
- package/docs/archive/CHANGELOG-pre-8.9.0.md +306 -0
- package/docs/benchmark.md +113 -0
- package/docs/catalog.md +32 -8
- package/docs/command-flows.md +6 -1
- package/docs/contracts/CHANGELOG-conventions.md +6 -0
- package/docs/contracts/command-clusters.md +18 -0
- package/docs/contracts/design-artifact-lifecycle.md +93 -0
- package/docs/contracts/design-artifact-verification.md +153 -0
- package/docs/contracts/injected-block-authenticity-model.md +66 -0
- package/docs/contracts/mcp-tool-inventory.md +28 -28
- package/docs/contracts/release-pr-gating.md +26 -0
- package/docs/contracts/release-sizing.md +85 -0
- package/docs/contracts/surface-agent-contracts.md +153 -0
- package/docs/decisions/ADR-061-corpus-grounding-layer.md +13 -0
- package/docs/decisions/ADR-115-command-cluster-consolidation-phase-4.md +1 -1
- package/docs/decisions/ADR-116-memory-tripwire-activation-path.md +9 -0
- package/docs/distribution/consumer-matrix.md +96 -0
- package/docs/featured-skills.md +2 -2
- package/docs/getting-started-by-role.md +1 -1
- package/docs/getting-started.md +2 -2
- package/docs/governance-advantage.md +1 -1
- package/docs/guidelines/agent-infra/asking-and-brevity-examples.md +27 -0
- package/docs/guidelines/agent-infra/emphasis-budget.md +45 -0
- package/docs/guidelines/agent-infra/simplicity-and-goal-demos.md +151 -0
- package/docs/guidelines/agent-infra/tool-description-as-policy.md +35 -0
- package/docs/guidelines/design-canon.md +1 -0
- package/docs/proof.md +38 -23
- package/docs/release-runbook.md +3 -0
- package/docs/second-brain-scope.md +69 -26
- package/docs/self-review-gate.md +51 -0
- package/package.json +1 -1
- package/src/agent-src/scripts/update_roadmap_progress.ts +2 -1
- package/src/scripts/_dispatch.bash +27 -0
- package/src/scripts/_lib/claude_builtin_names.ts +148 -0
- package/src/scripts/_lib/file_slicer.ts +117 -0
- package/src/scripts/_lib/lexical_index.ts +155 -0
- package/src/scripts/_lib/orchestration_record.ts +183 -0
- package/src/scripts/_lib/orchestration_savings.ts +114 -9
- package/src/scripts/_lib/retrieval_sanitize.ts +71 -0
- package/src/scripts/_lib/security_lint.ts +1 -0
- package/src/scripts/_lib/stat_index.ts +98 -0
- package/src/scripts/benchmark.ts +122 -0
- package/src/scripts/build_proof.ts +30 -17
- package/src/scripts/check_memory.ts +2 -0
- package/src/scripts/check_quality_regression.ts +51 -0
- package/src/scripts/check_release_adjacent_health.ts +121 -0
- package/src/scripts/check_roadmap_trackable.ts +3 -2
- package/src/scripts/check_rule_invariants.ts +275 -0
- package/src/scripts/condense.ts +14 -0
- package/src/scripts/consumer_matrix.ts +353 -0
- package/src/scripts/discovery_graph.ts +0 -0
- package/src/scripts/external_sources_denylist.json +4 -1
- package/src/scripts/generate_index.ts +6 -2
- package/src/scripts/install-hooks.sh +25 -0
- package/src/scripts/install.ts +35 -2
- package/src/scripts/learning_sidecar.ts +0 -0
- package/src/scripts/lint_agent_skill_names.ts +29 -0
- package/src/scripts/lint_changelog_rollback.ts +293 -0
- package/src/scripts/lint_knowledge_scale.ts +4 -3
- package/src/scripts/lint_roadmap_complexity.ts +124 -0
- package/src/scripts/lint_versioned_cache.ts +195 -0
- package/src/scripts/mcp_server/consumer_tool_catalog.json +2 -1
- package/src/scripts/mcp_server/tools.ts +24 -1
- package/src/scripts/measure_lexical_ranking.ts +173 -0
- package/src/scripts/memory_lookup.ts +0 -0
- package/src/scripts/orchestration_record.ts +156 -0
- package/src/scripts/orchestration_savings_report.ts +65 -8
- package/src/scripts/schemas/skill.schema.json +4 -0
- package/src/scripts/score_domain_truth.ts +192 -0
- package/src/scripts/second_brain_retrieval.ts +320 -0
- package/src/scripts/second_brain_run.ts +341 -0
- package/src/scripts/self_review_gate.test.ts +65 -0
- package/src/scripts/self_review_gate.ts +272 -0
- package/src/scripts/trigger_eval_grandfather.json +223 -224
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
#!/usr/bin/env tsx
|
|
2
|
+
/**
|
|
3
|
+
* Second-brain paired measurement — substrate on vs off vs placebo
|
|
4
|
+
* (road-to-second-brain-delta-proof Phase 2).
|
|
5
|
+
*
|
|
6
|
+
* Runs the deterministic recall corpus (Phase 1) through a FIXED host under
|
|
7
|
+
* three arms and scores each transcript with `second_brain_score` (no
|
|
8
|
+
* model-in-the-loop grading):
|
|
9
|
+
*
|
|
10
|
+
* - memory-off — the k+1 question alone (no prior-session context): the
|
|
11
|
+
* no-memory baseline; the agent must re-derive from scratch.
|
|
12
|
+
* - memory-on — the question + the prior-session fact the substrate would
|
|
13
|
+
* surface (perfect retrieval): the substrate's ideal output.
|
|
14
|
+
* - placebo — the question + an EQUAL-BYTE inert block (no relevant
|
|
15
|
+
* content): isolates "relevant retrieved content" from "mere extra
|
|
16
|
+
* context", exactly as the discipline benchmark isolates content from
|
|
17
|
+
* length.
|
|
18
|
+
*
|
|
19
|
+
* A PASS requires memory-on to beat BOTH memory-off AND placebo (a sign test
|
|
20
|
+
* over the paired per-task scores). Otherwise an honest NULL is recorded.
|
|
21
|
+
*
|
|
22
|
+
* HONEST SCOPING (stated, not buried): the Phase-1 corpus is one-fact-per-task
|
|
23
|
+
* by construction, so memory-on injects the exact fact — this measures the
|
|
24
|
+
* CONTEXT-VALUE UPPER BOUND (does the right prior fact, perfectly retrieved,
|
|
25
|
+
* let the model answer) NOT the substrate's retrieval PRECISION under a large
|
|
26
|
+
* store. A large-store retrieval-precision corpus is the follow-up; this run
|
|
27
|
+
* bounds the ceiling and isolates it from mere-extra-context via the placebo.
|
|
28
|
+
*
|
|
29
|
+
* Modes:
|
|
30
|
+
* --dry-run build every prompt + score a STUBBED "ideal" transcript per
|
|
31
|
+
* arm (memory-on stub contains the fact, off/placebo do not) — no
|
|
32
|
+
* API calls, no spend; proves the harness + arms wire correctly.
|
|
33
|
+
* --run live calls to the fixed host (SPEND). Writes the pinned report.
|
|
34
|
+
* --host <id> fixed host model id (default claude-haiku-4-5-20251001).
|
|
35
|
+
* --seeds <n> repetitions per arm×task (default 3).
|
|
36
|
+
*
|
|
37
|
+
* Exit codes: 0 ok / 1 run error / 2 usage.
|
|
38
|
+
*/
|
|
39
|
+
import * as fs from 'node:fs';
|
|
40
|
+
import * as path from 'node:path';
|
|
41
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
42
|
+
|
|
43
|
+
import { load_anthropic_key } from './ai_council/clients.js';
|
|
44
|
+
import { loadCorpus, scoreTask, type RecallTask } from './second_brain_score.js';
|
|
45
|
+
|
|
46
|
+
const _HERE = fileURLToPath(import.meta.url);
|
|
47
|
+
export const REPO_ROOT = path.resolve(path.dirname(_HERE), '..', '..');
|
|
48
|
+
export const REPORT_DIR = path.join(REPO_ROOT, 'internal', 'bench', 'reports');
|
|
49
|
+
|
|
50
|
+
const DEFAULT_HOST = 'claude-haiku-4-5-20251001';
|
|
51
|
+
const DEFAULT_SEEDS = 3;
|
|
52
|
+
|
|
53
|
+
export type Arm = 'memory-off' | 'memory-on' | 'placebo';
|
|
54
|
+
export const ARMS: readonly Arm[] = ['memory-off', 'memory-on', 'placebo'];
|
|
55
|
+
|
|
56
|
+
/** The prior-session memory block the substrate would surface for a task. */
|
|
57
|
+
function memoryBlock(task: RecallTask): string {
|
|
58
|
+
return `Prior-session memory (recalled by the second-brain substrate):\n- ${task.session_k}`;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Equal-byte inert filler, sized to the memory block, with no relevant content. */
|
|
62
|
+
function placeboBlock(task: RecallTask): string {
|
|
63
|
+
const target = memoryBlock(task).length;
|
|
64
|
+
const filler =
|
|
65
|
+
'Prior-session memory (recalled by the second-brain substrate):\n- ' +
|
|
66
|
+
'Note: routine housekeeping; no decisions or constraints were recorded in this window. ';
|
|
67
|
+
let out = filler;
|
|
68
|
+
while (out.length < target) out += 'Nothing further of record. ';
|
|
69
|
+
return out.slice(0, target);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Build the arm-specific prompt for a task. */
|
|
73
|
+
export function buildPrompt(task: RecallTask, arm: Arm): string {
|
|
74
|
+
const q = task.session_k1_prompt;
|
|
75
|
+
const preamble =
|
|
76
|
+
'You are continuing a multi-session engineering project. Answer the ' +
|
|
77
|
+
'question using what the project has decided so far. Be concise and concrete.';
|
|
78
|
+
if (arm === 'memory-off') {
|
|
79
|
+
return `${preamble}\n\n${q}`;
|
|
80
|
+
}
|
|
81
|
+
const block = arm === 'memory-on' ? memoryBlock(task) : placeboBlock(task);
|
|
82
|
+
return `${preamble}\n\n${block}\n\n${q}`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
// --- minimal Anthropic Messages call (reuses the repo key resolution) -------
|
|
86
|
+
|
|
87
|
+
interface CallResult {
|
|
88
|
+
text: string;
|
|
89
|
+
tokens_in: number;
|
|
90
|
+
tokens_out: number;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
async function callAnthropic(
|
|
94
|
+
model: string,
|
|
95
|
+
prompt: string,
|
|
96
|
+
apiKey: string,
|
|
97
|
+
): Promise<CallResult> {
|
|
98
|
+
const res = await fetch('https://api.anthropic.com/v1/messages', {
|
|
99
|
+
method: 'POST',
|
|
100
|
+
headers: {
|
|
101
|
+
'content-type': 'application/json',
|
|
102
|
+
'x-api-key': apiKey,
|
|
103
|
+
'anthropic-version': '2023-06-01',
|
|
104
|
+
},
|
|
105
|
+
body: JSON.stringify({
|
|
106
|
+
model,
|
|
107
|
+
max_tokens: 256,
|
|
108
|
+
temperature: 0.7,
|
|
109
|
+
messages: [{ role: 'user', content: prompt }],
|
|
110
|
+
}),
|
|
111
|
+
});
|
|
112
|
+
if (!res.ok) {
|
|
113
|
+
throw new Error(`anthropic ${res.status}: ${(await res.text()).slice(0, 300)}`);
|
|
114
|
+
}
|
|
115
|
+
const data = (await res.json()) as {
|
|
116
|
+
content?: Array<{ type: string; text?: string }>;
|
|
117
|
+
usage?: { input_tokens?: number; output_tokens?: number };
|
|
118
|
+
};
|
|
119
|
+
const text = (data.content ?? [])
|
|
120
|
+
.filter((b) => b.type === 'text')
|
|
121
|
+
.map((b) => b.text ?? '')
|
|
122
|
+
.join('');
|
|
123
|
+
return {
|
|
124
|
+
text,
|
|
125
|
+
tokens_in: data.usage?.input_tokens ?? 0,
|
|
126
|
+
tokens_out: data.usage?.output_tokens ?? 0,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Stub transcript for --dry-run: memory-on "recalls" the fact, others don't. */
|
|
131
|
+
function stubTranscript(task: RecallTask, arm: Arm): string {
|
|
132
|
+
if (arm === 'memory-on') {
|
|
133
|
+
return `${task.session_k} — carrying that forward: ${task.answer_key.must_contain.join(' ')}.`;
|
|
134
|
+
}
|
|
135
|
+
return "I don't have that from a prior session; I'd need it re-stated.";
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
interface ArmTaskResult {
|
|
139
|
+
task: string;
|
|
140
|
+
metric: string;
|
|
141
|
+
arm: Arm;
|
|
142
|
+
passes: number; // over seeds
|
|
143
|
+
seeds: number;
|
|
144
|
+
tokens_in: number;
|
|
145
|
+
tokens_out: number;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export interface RunReport {
|
|
149
|
+
schema_version: number;
|
|
150
|
+
kind: string;
|
|
151
|
+
roadmap: string;
|
|
152
|
+
host: string;
|
|
153
|
+
seeds: number;
|
|
154
|
+
mode: 'dry-run' | 'live';
|
|
155
|
+
scoping: string;
|
|
156
|
+
per_arm_task: ArmTaskResult[];
|
|
157
|
+
aggregate: Record<Arm, { pass: number; total: number; pass_rate: number }>;
|
|
158
|
+
paired: {
|
|
159
|
+
'on-vs-off': { on_wins: number; off_wins: number; ties: number; sign_p: number };
|
|
160
|
+
'on-vs-placebo': { on_wins: number; placebo_wins: number; ties: number; sign_p: number };
|
|
161
|
+
};
|
|
162
|
+
verdict: 'PASS' | 'NULL';
|
|
163
|
+
cost: { calls: number; tokens_in: number; tokens_out: number };
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** Two-sided sign-test p-value via exact binomial (ties excluded). */
|
|
167
|
+
export function signTestP(wins: number, losses: number): number {
|
|
168
|
+
const n = wins + losses;
|
|
169
|
+
if (n === 0) return 1;
|
|
170
|
+
const k = Math.min(wins, losses);
|
|
171
|
+
// sum of binomial tail P(X<=k) under p=0.5, times 2, capped at 1
|
|
172
|
+
let cum = 0;
|
|
173
|
+
for (let i = 0; i <= k; i++) cum += _choose(n, i);
|
|
174
|
+
const p = (cum / 2 ** n) * 2;
|
|
175
|
+
return Math.min(1, p);
|
|
176
|
+
}
|
|
177
|
+
function _choose(n: number, k: number): number {
|
|
178
|
+
let r = 1;
|
|
179
|
+
for (let i = 0; i < k; i++) r = (r * (n - i)) / (i + 1);
|
|
180
|
+
return r;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
export async function run(opts: {
|
|
184
|
+
mode: 'dry-run' | 'live';
|
|
185
|
+
host: string;
|
|
186
|
+
seeds: number;
|
|
187
|
+
}): Promise<RunReport> {
|
|
188
|
+
const tasks = loadCorpus();
|
|
189
|
+
const apiKey = opts.mode === 'live' ? load_anthropic_key() : '';
|
|
190
|
+
const rows: ArmTaskResult[] = [];
|
|
191
|
+
let calls = 0;
|
|
192
|
+
let tIn = 0;
|
|
193
|
+
let tOut = 0;
|
|
194
|
+
|
|
195
|
+
for (const task of tasks) {
|
|
196
|
+
for (const arm of ARMS) {
|
|
197
|
+
let passes = 0;
|
|
198
|
+
let ai = 0;
|
|
199
|
+
let ao = 0;
|
|
200
|
+
for (let s = 0; s < opts.seeds; s++) {
|
|
201
|
+
let transcript: string;
|
|
202
|
+
if (opts.mode === 'dry-run') {
|
|
203
|
+
transcript = stubTranscript(task, arm);
|
|
204
|
+
} else {
|
|
205
|
+
const r = await callAnthropic(opts.host, buildPrompt(task, arm), apiKey);
|
|
206
|
+
transcript = r.text;
|
|
207
|
+
ai += r.tokens_in;
|
|
208
|
+
ao += r.tokens_out;
|
|
209
|
+
calls += 1;
|
|
210
|
+
tIn += r.tokens_in;
|
|
211
|
+
tOut += r.tokens_out;
|
|
212
|
+
}
|
|
213
|
+
if (scoreTask(transcript, task).pass) passes += 1;
|
|
214
|
+
}
|
|
215
|
+
rows.push({
|
|
216
|
+
task: task.id,
|
|
217
|
+
metric: task.metric,
|
|
218
|
+
arm,
|
|
219
|
+
passes,
|
|
220
|
+
seeds: opts.seeds,
|
|
221
|
+
tokens_in: ai,
|
|
222
|
+
tokens_out: ao,
|
|
223
|
+
});
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
const agg = {} as RunReport['aggregate'];
|
|
228
|
+
for (const arm of ARMS) {
|
|
229
|
+
const armRows = rows.filter((r) => r.arm === arm);
|
|
230
|
+
const pass = armRows.reduce((s, r) => s + r.passes, 0);
|
|
231
|
+
const total = armRows.reduce((s, r) => s + r.seeds, 0);
|
|
232
|
+
agg[arm] = { pass, total, pass_rate: total ? pass / total : 0 };
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
const pairCompare = (a: Arm, b: Arm) => {
|
|
236
|
+
let aw = 0;
|
|
237
|
+
let bw = 0;
|
|
238
|
+
let ties = 0;
|
|
239
|
+
for (const task of tasks) {
|
|
240
|
+
const ra = rows.find((r) => r.task === task.id && r.arm === a)!;
|
|
241
|
+
const rb = rows.find((r) => r.task === task.id && r.arm === b)!;
|
|
242
|
+
if (ra.passes > rb.passes) aw += 1;
|
|
243
|
+
else if (rb.passes > ra.passes) bw += 1;
|
|
244
|
+
else ties += 1;
|
|
245
|
+
}
|
|
246
|
+
return { aw, bw, ties, p: signTestP(aw, bw) };
|
|
247
|
+
};
|
|
248
|
+
const onOff = pairCompare('memory-on', 'memory-off');
|
|
249
|
+
const onPlacebo = pairCompare('memory-on', 'placebo');
|
|
250
|
+
|
|
251
|
+
// PASS: memory-on beats BOTH off and placebo, each significant at p<0.05.
|
|
252
|
+
const verdict: 'PASS' | 'NULL' =
|
|
253
|
+
onOff.aw > onOff.bw &&
|
|
254
|
+
onOff.p < 0.05 &&
|
|
255
|
+
onPlacebo.aw > onPlacebo.bw &&
|
|
256
|
+
onPlacebo.p < 0.05
|
|
257
|
+
? 'PASS'
|
|
258
|
+
: 'NULL';
|
|
259
|
+
|
|
260
|
+
return {
|
|
261
|
+
schema_version: 1,
|
|
262
|
+
kind: `second-brain-delta-${opts.mode}`,
|
|
263
|
+
roadmap: 'road-to-second-brain-delta-proof',
|
|
264
|
+
host: opts.host,
|
|
265
|
+
seeds: opts.seeds,
|
|
266
|
+
mode: opts.mode,
|
|
267
|
+
scoping:
|
|
268
|
+
'CONTEXT-VALUE UPPER BOUND on a one-fact-per-task corpus: memory-on injects the ' +
|
|
269
|
+
'exact prior fact (perfect retrieval). Measures whether the right prior fact + the ' +
|
|
270
|
+
'placebo-isolated retrieval mechanism let the model answer — NOT retrieval precision ' +
|
|
271
|
+
'under a large store (that is the follow-up corpus).',
|
|
272
|
+
per_arm_task: rows,
|
|
273
|
+
aggregate: agg,
|
|
274
|
+
paired: {
|
|
275
|
+
'on-vs-off': { on_wins: onOff.aw, off_wins: onOff.bw, ties: onOff.ties, sign_p: onOff.p },
|
|
276
|
+
'on-vs-placebo': {
|
|
277
|
+
on_wins: onPlacebo.aw,
|
|
278
|
+
placebo_wins: onPlacebo.bw,
|
|
279
|
+
ties: onPlacebo.ties,
|
|
280
|
+
sign_p: onPlacebo.p,
|
|
281
|
+
},
|
|
282
|
+
},
|
|
283
|
+
verdict,
|
|
284
|
+
cost: { calls, tokens_in: tIn, tokens_out: tOut },
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
export async function main(argv: readonly string[] = process.argv.slice(2)): Promise<number> {
|
|
289
|
+
const mode = argv.includes('--run') ? 'live' : argv.includes('--dry-run') ? 'dry-run' : null;
|
|
290
|
+
if (mode === null) {
|
|
291
|
+
process.stderr.write('usage: second_brain_run --dry-run | --run [--host <id>] [--seeds <n>]\n');
|
|
292
|
+
return 2;
|
|
293
|
+
}
|
|
294
|
+
const hostIdx = argv.indexOf('--host');
|
|
295
|
+
const host = hostIdx >= 0 ? String(argv[hostIdx + 1]) : DEFAULT_HOST;
|
|
296
|
+
const seedIdx = argv.indexOf('--seeds');
|
|
297
|
+
const seeds = seedIdx >= 0 ? Number(argv[seedIdx + 1]) : DEFAULT_SEEDS;
|
|
298
|
+
|
|
299
|
+
let report: RunReport;
|
|
300
|
+
try {
|
|
301
|
+
report = await run({ mode, host, seeds });
|
|
302
|
+
} catch (e) {
|
|
303
|
+
process.stderr.write(`second_brain_run: ${String(e)}\n`);
|
|
304
|
+
return 1;
|
|
305
|
+
}
|
|
306
|
+
|
|
307
|
+
if (mode === 'live') {
|
|
308
|
+
fs.mkdirSync(REPORT_DIR, { recursive: true });
|
|
309
|
+
const out = path.join(REPORT_DIR, 'second-brain-delta.json');
|
|
310
|
+
fs.writeFileSync(out, JSON.stringify(report, null, 2) + '\n');
|
|
311
|
+
process.stdout.write(`wrote ${path.relative(REPO_ROOT, out)}\n`);
|
|
312
|
+
}
|
|
313
|
+
const a = report.aggregate;
|
|
314
|
+
process.stdout.write(
|
|
315
|
+
`${mode}: on=${a['memory-on'].pass}/${a['memory-on'].total} ` +
|
|
316
|
+
`off=${a['memory-off'].pass}/${a['memory-off'].total} ` +
|
|
317
|
+
`placebo=${a.placebo.pass}/${a.placebo.total} · ` +
|
|
318
|
+
`on-vs-off p=${report.paired['on-vs-off'].sign_p.toFixed(4)} ` +
|
|
319
|
+
`on-vs-placebo p=${report.paired['on-vs-placebo'].sign_p.toFixed(4)} · ` +
|
|
320
|
+
`${report.verdict} · calls=${report.cost.calls}\n`,
|
|
321
|
+
);
|
|
322
|
+
return 0;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function _isCliEntry(): boolean {
|
|
326
|
+
if (process.argv[1] === undefined) return false;
|
|
327
|
+
const argvUrl = pathToFileURL(path.resolve(process.argv[1])).href;
|
|
328
|
+
if (import.meta.url === argvUrl) return true;
|
|
329
|
+
try {
|
|
330
|
+
return (
|
|
331
|
+
fs.realpathSync(fileURLToPath(import.meta.url)) ===
|
|
332
|
+
fs.realpathSync(path.resolve(process.argv[1]))
|
|
333
|
+
);
|
|
334
|
+
} catch {
|
|
335
|
+
return false;
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
if (_isCliEntry()) {
|
|
340
|
+
main().then((code) => process.exit(code));
|
|
341
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
|
|
3
|
+
import {
|
|
4
|
+
classifyBlocking,
|
|
5
|
+
gateVerdict,
|
|
6
|
+
isReviewablePath,
|
|
7
|
+
renderReview,
|
|
8
|
+
type Finding,
|
|
9
|
+
} from './self_review_gate.js';
|
|
10
|
+
|
|
11
|
+
const f = (severity: Finding['severity'], kind: Finding['kind']): Finding => ({
|
|
12
|
+
severity,
|
|
13
|
+
kind,
|
|
14
|
+
title: `${severity} ${kind}`,
|
|
15
|
+
detail: 'x',
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
describe('classifyBlocking', () => {
|
|
19
|
+
it('blocks security + high and claim + critical', () => {
|
|
20
|
+
expect(classifyBlocking(f('high', 'security'))).toBe(true);
|
|
21
|
+
expect(classifyBlocking(f('critical', 'claim'))).toBe(true);
|
|
22
|
+
});
|
|
23
|
+
it('does not block low/medium security, or any style/correctness', () => {
|
|
24
|
+
expect(classifyBlocking(f('medium', 'security'))).toBe(false);
|
|
25
|
+
expect(classifyBlocking(f('low', 'claim'))).toBe(false);
|
|
26
|
+
expect(classifyBlocking(f('critical', 'style'))).toBe(false);
|
|
27
|
+
expect(classifyBlocking(f('high', 'correctness'))).toBe(false);
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
describe('gateVerdict', () => {
|
|
32
|
+
const blocking = [f('high', 'security')];
|
|
33
|
+
it('advisory (default shipped mode) never blocks, even with a blocking finding', () => {
|
|
34
|
+
expect(gateVerdict(blocking, { enforce: false })).toBe(0);
|
|
35
|
+
});
|
|
36
|
+
it('enforce blocks only on a merge-blocking finding', () => {
|
|
37
|
+
expect(gateVerdict(blocking, { enforce: true })).toBe(2);
|
|
38
|
+
expect(gateVerdict([f('critical', 'style')], { enforce: true })).toBe(0);
|
|
39
|
+
expect(gateVerdict([], { enforce: true })).toBe(0);
|
|
40
|
+
});
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
describe('isReviewablePath', () => {
|
|
44
|
+
it('skips generated projections + lockfiles, keeps source', () => {
|
|
45
|
+
expect(isReviewablePath('dist/agent-src/skills/x/SKILL.md')).toBe(false);
|
|
46
|
+
expect(isReviewablePath('.augment/rules/x.md')).toBe(false);
|
|
47
|
+
expect(isReviewablePath('.windsurfrules')).toBe(false);
|
|
48
|
+
expect(isReviewablePath('package-lock.json')).toBe(false);
|
|
49
|
+
expect(isReviewablePath('src/rules/x.md')).toBe(true);
|
|
50
|
+
expect(isReviewablePath('src/scripts/y.ts')).toBe(true);
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
describe('renderReview', () => {
|
|
55
|
+
it('carries the HUMAN REVIEW REQUIRED banner + the floor-not-human-review caveat', () => {
|
|
56
|
+
const out = renderReview([], false);
|
|
57
|
+
expect(out).toContain('HUMAN REVIEW REQUIRED');
|
|
58
|
+
expect(out).toContain('not** independent human review');
|
|
59
|
+
});
|
|
60
|
+
it('advisory phrasing says WOULD block, enforce phrasing says blocking', () => {
|
|
61
|
+
const blocking = [f('high', 'security')];
|
|
62
|
+
expect(renderReview(blocking, false)).toContain('WOULD block');
|
|
63
|
+
expect(renderReview(blocking, true)).toContain('merge-blocking');
|
|
64
|
+
});
|
|
65
|
+
});
|
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Self-review gate — the package reviewing its own PRs with the exact machinery
|
|
3
|
+
* it sells (`adversarial-review` + `agent-security-review`).
|
|
4
|
+
*
|
|
5
|
+
* road-to-maintainer-bus-factor Phase 1. Ships ADVISORY + INERT-WITHOUT-SECRET:
|
|
6
|
+
* - `--dry-run` (the PR gate job): no API, no spend, no secret. Assembles and
|
|
7
|
+
* prints the review plan (which skills, which files, budget estimate) and
|
|
8
|
+
* validates the harness. Always exit 0.
|
|
9
|
+
* - live `--advisory` (the secret-gated job): loads the two review skill bodies
|
|
10
|
+
* as the system prompt, sends the diff to the model, collects structured
|
|
11
|
+
* findings, classifies them, records a verdict, and posts a PR review. In
|
|
12
|
+
* advisory mode it NEVER blocks the merge (exit 0) — it records what WOULD
|
|
13
|
+
* block. If no key is available it is a logged no-op (exit 0), never a failure.
|
|
14
|
+
*
|
|
15
|
+
* Turning the gate REQUIRED (block-on-verdict) and wiring the API secret + the
|
|
16
|
+
* per-PR budget are the maintainer's acts (blocker `self-review-gate-cost`); this
|
|
17
|
+
* script exposes the `--enforce` path so that flip is a one-flag change, but the
|
|
18
|
+
* shipped workflow runs advisory only.
|
|
19
|
+
*
|
|
20
|
+
* `classifyBlocking` + `gateVerdict` are pure and unit-tested (no API) — mirrors
|
|
21
|
+
* the `check_quality_regression.gateVerdict` pattern.
|
|
22
|
+
*/
|
|
23
|
+
import { spawnSync } from 'node:child_process';
|
|
24
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
25
|
+
import path from 'node:path';
|
|
26
|
+
import { fileURLToPath } from 'node:url';
|
|
27
|
+
|
|
28
|
+
import { AnthropicClient, load_anthropic_key } from './ai_council/clients.js';
|
|
29
|
+
|
|
30
|
+
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
31
|
+
const REPO_ROOT = path.resolve(__dirname, '..', '..');
|
|
32
|
+
|
|
33
|
+
// ── Finding model ─────────────────────────────────────────────────────
|
|
34
|
+
export type Severity = 'critical' | 'high' | 'medium' | 'low';
|
|
35
|
+
export type FindingKind = 'security' | 'claim' | 'correctness' | 'style';
|
|
36
|
+
|
|
37
|
+
export interface Finding {
|
|
38
|
+
severity: Severity;
|
|
39
|
+
kind: FindingKind;
|
|
40
|
+
title: string;
|
|
41
|
+
detail: string;
|
|
42
|
+
file?: string;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* A finding is merge-blocking iff it is security- or claim-affecting AND at
|
|
47
|
+
* least `high` severity. Style and correctness findings advise; low/medium
|
|
48
|
+
* security findings advise. Council 2026-07-08 (claude-sonnet-4-5 + gpt-4o):
|
|
49
|
+
* a 100 %-blocking gate at solo-maintainer token cost gets ignored or gamed —
|
|
50
|
+
* block ONLY on the narrow security/claim × {critical,high} intersection.
|
|
51
|
+
*/
|
|
52
|
+
export function classifyBlocking(f: Finding): boolean {
|
|
53
|
+
return (f.kind === 'security' || f.kind === 'claim') && (f.severity === 'critical' || f.severity === 'high');
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Deterministic gate verdict. Mirrors `check_quality_regression.gateVerdict`:
|
|
58
|
+
* exit 2 blocks the merge, exit 0 passes.
|
|
59
|
+
* - advisory (default shipped mode): always 0; the caller reports the
|
|
60
|
+
* would-block set without failing the check.
|
|
61
|
+
* - enforce (maintainer flip): 2 iff any finding is merge-blocking.
|
|
62
|
+
*/
|
|
63
|
+
export function gateVerdict(findings: readonly Finding[], opts: { enforce: boolean }): 0 | 2 {
|
|
64
|
+
if (!opts.enforce) return 0;
|
|
65
|
+
return findings.some(classifyBlocking) ? 2 : 0;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// ── Diff collection ───────────────────────────────────────────────────
|
|
69
|
+
/** Generated projections + lockfiles a self-review should not re-read. */
|
|
70
|
+
const SKIP_PREFIXES = ['dist/agent-src/', '.augment/', '.claude/', '.cursor/', '.clinerules/'];
|
|
71
|
+
const SKIP_EXACT = new Set(['.windsurfrules', 'GEMINI.md', 'package-lock.json']);
|
|
72
|
+
|
|
73
|
+
export function isReviewablePath(p: string): boolean {
|
|
74
|
+
if (SKIP_EXACT.has(p)) return false;
|
|
75
|
+
return !SKIP_PREFIXES.some((pre) => p.startsWith(pre));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function changedFiles(baseRef: string): string[] {
|
|
79
|
+
const r = spawnSync('git', ['diff', '--name-only', `${baseRef}...HEAD`], {
|
|
80
|
+
cwd: REPO_ROOT,
|
|
81
|
+
encoding: 'utf8',
|
|
82
|
+
});
|
|
83
|
+
if (r.status !== 0) {
|
|
84
|
+
throw new Error(`git diff failed: ${(r.stderr ?? '').toString().slice(0, 300)}`);
|
|
85
|
+
}
|
|
86
|
+
return (r.stdout ?? '')
|
|
87
|
+
.split('\n')
|
|
88
|
+
.map((s) => s.trim())
|
|
89
|
+
.filter(Boolean)
|
|
90
|
+
.filter(isReviewablePath);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function diffText(baseRef: string, files: string[]): string {
|
|
94
|
+
if (files.length === 0) return '';
|
|
95
|
+
const r = spawnSync('git', ['diff', `${baseRef}...HEAD`, '--', ...files], {
|
|
96
|
+
cwd: REPO_ROOT,
|
|
97
|
+
encoding: 'utf8',
|
|
98
|
+
maxBuffer: 32 * 1024 * 1024,
|
|
99
|
+
});
|
|
100
|
+
return (r.stdout ?? '').toString();
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// ── Review skill bodies (the system prompt) ───────────────────────────
|
|
104
|
+
const REVIEW_SKILLS = ['adversarial-review', 'agent-security-review'] as const;
|
|
105
|
+
|
|
106
|
+
function loadSkillBody(name: string): string {
|
|
107
|
+
const p = path.join(REPO_ROOT, 'src', 'skills', name, 'SKILL.md');
|
|
108
|
+
return readFileSync(p, 'utf8');
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function buildSystemPrompt(): string {
|
|
112
|
+
const bodies = REVIEW_SKILLS.map((s) => `# Skill: ${s}\n\n${loadSkillBody(s)}`).join('\n\n---\n\n');
|
|
113
|
+
return [
|
|
114
|
+
'You are the self-review gate for an AI-agent governance package. Apply the',
|
|
115
|
+
'two skills below to the supplied PR diff. Return ONLY a JSON object of the',
|
|
116
|
+
'shape {"findings":[{"severity":"critical|high|medium|low",',
|
|
117
|
+
'"kind":"security|claim|correctness|style","title":"...","detail":"...",',
|
|
118
|
+
'"file":"optional/path"}]}. Use kind="claim" for an unbacked headline number',
|
|
119
|
+
'or a proof/CLAIMS-affecting statement, kind="security" for an auth/tenant/',
|
|
120
|
+
'secret/egress/injection gap. No prose outside the JSON.',
|
|
121
|
+
'',
|
|
122
|
+
bodies,
|
|
123
|
+
].join('\n');
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// ── Plan (dry-run) ────────────────────────────────────────────────────
|
|
127
|
+
export interface ReviewPlan {
|
|
128
|
+
skills: string[];
|
|
129
|
+
files: string[];
|
|
130
|
+
promptChars: number;
|
|
131
|
+
note: string;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export function buildPlan(baseRef: string): ReviewPlan {
|
|
135
|
+
const files = changedFiles(baseRef);
|
|
136
|
+
const diff = diffText(baseRef, files);
|
|
137
|
+
return {
|
|
138
|
+
skills: [...REVIEW_SKILLS],
|
|
139
|
+
files,
|
|
140
|
+
promptChars: buildSystemPrompt().length + diff.length,
|
|
141
|
+
note:
|
|
142
|
+
files.length === 0
|
|
143
|
+
? 'No reviewable (non-generated) files changed — the live review would no-op.'
|
|
144
|
+
: `${files.length} reviewable file(s); live review would send ~${Math.ceil(
|
|
145
|
+
(buildSystemPrompt().length + diff.length) / 4,
|
|
146
|
+
)} input tokens.`,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// ── Live review ───────────────────────────────────────────────────────
|
|
151
|
+
function parseFindings(text: string): Finding[] {
|
|
152
|
+
const start = text.indexOf('{');
|
|
153
|
+
const end = text.lastIndexOf('}');
|
|
154
|
+
if (start < 0 || end <= start) return [];
|
|
155
|
+
let parsed: unknown;
|
|
156
|
+
try {
|
|
157
|
+
parsed = JSON.parse(text.slice(start, end + 1));
|
|
158
|
+
} catch {
|
|
159
|
+
return [];
|
|
160
|
+
}
|
|
161
|
+
const arr = (parsed as { findings?: unknown })?.findings;
|
|
162
|
+
if (!Array.isArray(arr)) return [];
|
|
163
|
+
const sev = new Set<Severity>(['critical', 'high', 'medium', 'low']);
|
|
164
|
+
const kind = new Set<FindingKind>(['security', 'claim', 'correctness', 'style']);
|
|
165
|
+
return arr
|
|
166
|
+
.filter((f): f is Record<string, unknown> => typeof f === 'object' && f !== null)
|
|
167
|
+
.filter((f) => sev.has(f.severity as Severity) && kind.has(f.kind as FindingKind))
|
|
168
|
+
.map((f) => ({
|
|
169
|
+
severity: f.severity as Severity,
|
|
170
|
+
kind: f.kind as FindingKind,
|
|
171
|
+
title: String(f.title ?? '').slice(0, 300),
|
|
172
|
+
detail: String(f.detail ?? '').slice(0, 2000),
|
|
173
|
+
...(typeof f.file === 'string' ? { file: f.file } : {}),
|
|
174
|
+
}));
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export function renderReview(findings: Finding[], enforce: boolean): string {
|
|
178
|
+
const banner =
|
|
179
|
+
'> HUMAN REVIEW REQUIRED — dogfooded AI adversarial-review + security gate. ' +
|
|
180
|
+
'Findings are decision support, not a guarantee; detection is probabilistic. ' +
|
|
181
|
+
'This is a floor, **not** independent human review.';
|
|
182
|
+
if (findings.length === 0) {
|
|
183
|
+
return `${banner}\n\n✅ Self-review gate: no findings.`;
|
|
184
|
+
}
|
|
185
|
+
const blocking = findings.filter(classifyBlocking);
|
|
186
|
+
const rows = findings
|
|
187
|
+
.map((f) => `| ${f.severity} | ${f.kind} | ${f.file ?? '—'} | ${f.title} |`)
|
|
188
|
+
.join('\n');
|
|
189
|
+
const verdictLine = blocking.length
|
|
190
|
+
? enforce
|
|
191
|
+
? `❌ ${blocking.length} merge-blocking finding(s) (security/claim × high+).`
|
|
192
|
+
: `⚠️ ${blocking.length} finding(s) WOULD block merge under an enforced gate (advisory now).`
|
|
193
|
+
: '✅ No merge-blocking findings (style/correctness advise only).';
|
|
194
|
+
return `${banner}\n\n${verdictLine}\n\n| severity | kind | file | finding |\n|---|---|---|---|\n${rows}`;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
function postReview(body: string): void {
|
|
198
|
+
const pr = process.env.PR_NUMBER;
|
|
199
|
+
if (!pr) {
|
|
200
|
+
process.stdout.write('::notice::self-review-gate — no PR_NUMBER; printing review instead of posting.\n');
|
|
201
|
+
process.stdout.write(body + '\n');
|
|
202
|
+
return;
|
|
203
|
+
}
|
|
204
|
+
const r = spawnSync('gh', ['pr', 'comment', pr, '--body', body], { cwd: REPO_ROOT, encoding: 'utf8' });
|
|
205
|
+
if (r.status !== 0) {
|
|
206
|
+
process.stdout.write(`::warning::self-review-gate — gh pr comment failed: ${(r.stderr ?? '').toString().slice(0, 300)}\n`);
|
|
207
|
+
process.stdout.write(body + '\n');
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function resolveKey(): string | null {
|
|
212
|
+
const env = process.env.ANTHROPIC_API_KEY;
|
|
213
|
+
if (env && env.trim()) return env.trim();
|
|
214
|
+
try {
|
|
215
|
+
return load_anthropic_key();
|
|
216
|
+
} catch {
|
|
217
|
+
return null;
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// ── CLI ───────────────────────────────────────────────────────────────
|
|
222
|
+
export function main(argv: string[]): 0 | 2 {
|
|
223
|
+
const dryRun = argv.includes('--dry-run');
|
|
224
|
+
const enforce = argv.includes('--enforce');
|
|
225
|
+
const baseIdx = argv.indexOf('--base');
|
|
226
|
+
const baseRef = (baseIdx >= 0 ? argv[baseIdx + 1] : undefined) ?? 'origin/main';
|
|
227
|
+
|
|
228
|
+
const plan = buildPlan(baseRef);
|
|
229
|
+
|
|
230
|
+
if (dryRun) {
|
|
231
|
+
process.stdout.write(
|
|
232
|
+
`self-review-gate (dry-run — no spend, advisory):\n` +
|
|
233
|
+
` skills: ${plan.skills.join(', ')}\n` +
|
|
234
|
+
` files: ${plan.files.length}\n` +
|
|
235
|
+
` ${plan.note}\n`,
|
|
236
|
+
);
|
|
237
|
+
return 0;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
if (plan.files.length === 0) {
|
|
241
|
+
process.stdout.write('::notice::self-review-gate — no reviewable files changed; skipping.\n');
|
|
242
|
+
return 0;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
const key = resolveKey();
|
|
246
|
+
if (!key) {
|
|
247
|
+
process.stdout.write(
|
|
248
|
+
'::notice::self-review-gate skipped — set the ANTHROPIC_API_KEY repo secret to enable the live dogfooded review.\n',
|
|
249
|
+
);
|
|
250
|
+
return 0;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const client = new AnthropicClient({ api_key: key });
|
|
254
|
+
const resp = client.ask(buildSystemPrompt(), diffText(baseRef, plan.files), 4096);
|
|
255
|
+
if (resp.error) {
|
|
256
|
+
process.stdout.write(`::warning::self-review-gate — model call failed: ${resp.error}\n`);
|
|
257
|
+
return 0; // advisory: a transport failure never blocks the merge
|
|
258
|
+
}
|
|
259
|
+
const findings = parseFindings(resp.text);
|
|
260
|
+
const body = renderReview(findings, enforce);
|
|
261
|
+
postReview(body);
|
|
262
|
+
|
|
263
|
+
const code = gateVerdict(findings, { enforce });
|
|
264
|
+
if (code === 2) {
|
|
265
|
+
process.stdout.write('::error::self-review-gate — merge-blocking findings under enforce mode.\n');
|
|
266
|
+
}
|
|
267
|
+
return code;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
if (existsSync(process.argv[1] ?? '') && import.meta.url === `file://${process.argv[1]}`) {
|
|
271
|
+
process.exit(main(process.argv.slice(2)));
|
|
272
|
+
}
|