kodelyth-ecc 2.5.4 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +115 -0
- package/VERSION +1 -1
- package/bin/kodelyth-ecc.js +137 -0
- package/commands/arena.md +101 -0
- package/commands/evil-mode.md +106 -0
- package/commands/god-mode.md +85 -0
- package/package.json +1 -1
- package/scripts/arena/arena.js +280 -0
- package/scripts/arena/contract.js +169 -0
- package/scripts/arena/evil.js +175 -0
- package/scripts/arena/god.js +208 -0
- package/scripts/arena/state.js +195 -0
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
// scripts/arena/arena.js
|
|
2
|
+
// The Arena — GOD builds, EVIL attacks, repeat until the attacker gives up.
|
|
3
|
+
//
|
|
4
|
+
// This is a STATE MACHINE, not an agent dispatcher. It decides what should
|
|
5
|
+
// happen next and grades what comes back; the actual agent invocations are made
|
|
6
|
+
// by the AI driving `/arena`. Keeping dispatch out of here is what makes the
|
|
7
|
+
// loop testable without spending a single token.
|
|
8
|
+
//
|
|
9
|
+
// round N: god_build -> evil_hunt -> evil_verify -> round_close
|
|
10
|
+
// |
|
|
11
|
+
// converged / out of budget? --------+
|
|
12
|
+
// | no -> round N+1
|
|
13
|
+
// yes
|
|
14
|
+
// v
|
|
15
|
+
// report
|
|
16
|
+
|
|
17
|
+
'use strict';
|
|
18
|
+
|
|
19
|
+
const god = require('./god');
|
|
20
|
+
const evil = require('./evil');
|
|
21
|
+
const state = require('./state');
|
|
22
|
+
const { dedupe, effectiveRisk, VERDICT } = require('./contract');
|
|
23
|
+
|
|
24
|
+
// ── Phases within a round ────────────────────────────────────────────────────
|
|
25
|
+
|
|
26
|
+
const PHASE = {
|
|
27
|
+
GOD_BUILD: 'god_build',
|
|
28
|
+
EVIL_HUNT: 'evil_hunt',
|
|
29
|
+
EVIL_VERIFY: 'evil_verify',
|
|
30
|
+
ROUND_CLOSE: 'round_close',
|
|
31
|
+
REPORT: 'report',
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
// ── Starting a run ───────────────────────────────────────────────────────────
|
|
35
|
+
|
|
36
|
+
function startArena({ task, scope = '.', flags = [], limits = {} } = {}) {
|
|
37
|
+
const run = state.createRun({ task, limits });
|
|
38
|
+
run.scope = scope;
|
|
39
|
+
run.flags = flags;
|
|
40
|
+
run.phase = PHASE.GOD_BUILD;
|
|
41
|
+
run.pending = { artifacts: [], findings: [], addressedIds: [] };
|
|
42
|
+
state.save(run);
|
|
43
|
+
return run;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// ── The state machine ────────────────────────────────────────────────────────
|
|
47
|
+
// Given a run, what happens next? Returns an actionable step with the brief the
|
|
48
|
+
// AI should execute, plus a cost estimate the budget can veto.
|
|
49
|
+
|
|
50
|
+
function nextAction(run) {
|
|
51
|
+
if (!run) throw new Error('arena: no run');
|
|
52
|
+
|
|
53
|
+
// A finished run has nothing left but its report.
|
|
54
|
+
if (run.status !== 'running') {
|
|
55
|
+
return { action: PHASE.REPORT, reason: run.stopReason, run: state.summarize(run) };
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const round = run.rounds.length + 1;
|
|
59
|
+
const carried = carriedFindings(run);
|
|
60
|
+
|
|
61
|
+
switch (run.phase) {
|
|
62
|
+
case PHASE.GOD_BUILD: {
|
|
63
|
+
const plan = god.planBuild({ task: run.task, findings: carried, round });
|
|
64
|
+
return {
|
|
65
|
+
action: PHASE.GOD_BUILD,
|
|
66
|
+
round,
|
|
67
|
+
stages: plan.stages,
|
|
68
|
+
// Round 1 builds; later rounds are fixing what EVIL proved.
|
|
69
|
+
intent: round === 1 ? 'build' : 'fix',
|
|
70
|
+
carriedFindings: carried.length,
|
|
71
|
+
estimatedTokens: plan.estimatedTokens,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
case PHASE.EVIL_HUNT: {
|
|
76
|
+
const plan = evil.planSweep({
|
|
77
|
+
scope: run.scope,
|
|
78
|
+
flags: run.flags,
|
|
79
|
+
round,
|
|
80
|
+
knownFindingIds: run.seenFindingIds,
|
|
81
|
+
});
|
|
82
|
+
return {
|
|
83
|
+
action: PHASE.EVIL_HUNT,
|
|
84
|
+
round,
|
|
85
|
+
crew: plan.crew,
|
|
86
|
+
briefs: plan.briefs,
|
|
87
|
+
estimatedTokens: plan.estimatedTokens,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
case PHASE.EVIL_VERIFY: {
|
|
92
|
+
const targets = evil.selectForVerification(run.pending.findings);
|
|
93
|
+
return {
|
|
94
|
+
action: PHASE.EVIL_VERIFY,
|
|
95
|
+
round,
|
|
96
|
+
targets: targets.map(f => ({ id: f.id, title: f.title, brief: evil.verifyBrief(f) })),
|
|
97
|
+
// Verification is cheap per finding but must stay bounded.
|
|
98
|
+
estimatedTokens: targets.length * 3000,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
case PHASE.ROUND_CLOSE:
|
|
103
|
+
default:
|
|
104
|
+
return { action: PHASE.ROUND_CLOSE, round };
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// Findings GOD still has to answer for: open, real, high-signal, worst first.
|
|
109
|
+
function carriedFindings(run) {
|
|
110
|
+
const last = run.rounds[run.rounds.length - 1];
|
|
111
|
+
if (!last) return [];
|
|
112
|
+
return evil.actionable(last.findings).slice(0, 10);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// ── Recording each phase ─────────────────────────────────────────────────────
|
|
116
|
+
|
|
117
|
+
function submitGodWork(run, { artifacts = [], addressedIds = [], tokensSpent = 0, elapsedMs = 0 } = {}) {
|
|
118
|
+
requirePhase(run, PHASE.GOD_BUILD);
|
|
119
|
+
run.pending.artifacts = artifacts;
|
|
120
|
+
run.pending.addressedIds = addressedIds;
|
|
121
|
+
run.spent.tokens += Math.max(0, Number(tokensSpent) || 0);
|
|
122
|
+
run.spent.ms += Math.max(0, Number(elapsedMs) || 0);
|
|
123
|
+
run.phase = PHASE.EVIL_HUNT;
|
|
124
|
+
state.save(run);
|
|
125
|
+
return run;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function submitEvilHunt(run, { findings = [], tokensSpent = 0, elapsedMs = 0 } = {}) {
|
|
129
|
+
requirePhase(run, PHASE.EVIL_HUNT);
|
|
130
|
+
run.pending.findings = dedupe(findings);
|
|
131
|
+
run.spent.tokens += Math.max(0, Number(tokensSpent) || 0);
|
|
132
|
+
run.spent.ms += Math.max(0, Number(elapsedMs) || 0);
|
|
133
|
+
run.phase = PHASE.EVIL_VERIFY;
|
|
134
|
+
state.save(run);
|
|
135
|
+
return run;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function submitVerdicts(run, { verdicts = {}, tokensSpent = 0, elapsedMs = 0 } = {}) {
|
|
139
|
+
requirePhase(run, PHASE.EVIL_VERIFY);
|
|
140
|
+
run.pending.findings = evil.applyVerdicts(run.pending.findings, verdicts);
|
|
141
|
+
run.spent.tokens += Math.max(0, Number(tokensSpent) || 0);
|
|
142
|
+
run.spent.ms += Math.max(0, Number(elapsedMs) || 0);
|
|
143
|
+
run.phase = PHASE.ROUND_CLOSE;
|
|
144
|
+
state.save(run);
|
|
145
|
+
return run;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// Close the round: grade GOD's work, record the verdict, decide whether to
|
|
149
|
+
// run another round. state.recordRound() owns convergence + hard stops.
|
|
150
|
+
function closeRound(run) {
|
|
151
|
+
requirePhase(run, PHASE.ROUND_CLOSE);
|
|
152
|
+
|
|
153
|
+
const completion = god.roundComplete({
|
|
154
|
+
artifacts: run.pending.artifacts,
|
|
155
|
+
findings: run.pending.findings,
|
|
156
|
+
addressedIds: run.pending.addressedIds,
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
const verdict = state.recordRound(run, {
|
|
160
|
+
findings: run.pending.findings,
|
|
161
|
+
artifacts: run.pending.artifacts,
|
|
162
|
+
tokensSpent: 0, // already accrued per-phase; do not double count
|
|
163
|
+
elapsedMs: 0,
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
// Attach whether GOD actually finished its side of the round.
|
|
167
|
+
verdict.godComplete = completion.complete;
|
|
168
|
+
verdict.unverifiedArtifacts = completion.unverifiedArtifacts;
|
|
169
|
+
verdict.outstandingFindings = completion.outstandingFindings;
|
|
170
|
+
run.rounds[run.rounds.length - 1] = verdict;
|
|
171
|
+
|
|
172
|
+
run.pending = { artifacts: [], findings: [], addressedIds: [] };
|
|
173
|
+
run.phase = run.status === 'running' ? PHASE.GOD_BUILD : PHASE.REPORT;
|
|
174
|
+
state.save(run);
|
|
175
|
+
return verdict;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
function requirePhase(run, expected) {
|
|
179
|
+
if (run.phase !== expected) {
|
|
180
|
+
throw new Error(`arena: expected phase "${expected}" but run is in "${run.phase}"`);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// ── Budget guard ─────────────────────────────────────────────────────────────
|
|
185
|
+
// Called before dispatching an action. The arena aborts cleanly rather than
|
|
186
|
+
// overspending — a partial result you can read beats a surprise bill.
|
|
187
|
+
|
|
188
|
+
function affordOrAbort(run, action) {
|
|
189
|
+
const est = action.estimatedTokens || 0;
|
|
190
|
+
const check = state.canAffordRound(run, est);
|
|
191
|
+
if (check.ok) return { ok: true, ...check };
|
|
192
|
+
run.status = 'aborted';
|
|
193
|
+
run.stopReason = `budget guard: next step needs ~${est.toLocaleString()} tokens, only ${check.remaining.toLocaleString()} left`;
|
|
194
|
+
run.phase = PHASE.REPORT;
|
|
195
|
+
state.save(run);
|
|
196
|
+
return { ok: false, ...check };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// ── Report ───────────────────────────────────────────────────────────────────
|
|
200
|
+
|
|
201
|
+
function buildReport(run) {
|
|
202
|
+
const s = state.summarize(run);
|
|
203
|
+
const lines = [];
|
|
204
|
+
|
|
205
|
+
lines.push(`# Arena report — ${s.task}`, '');
|
|
206
|
+
lines.push(`- **Run:** \`${s.runId}\``);
|
|
207
|
+
lines.push(`- **Status:** ${s.status}${s.stopReason ? ` — ${s.stopReason}` : ''}`);
|
|
208
|
+
lines.push(`- **Rounds:** ${s.rounds}`);
|
|
209
|
+
lines.push(`- **Tokens:** ${s.tokensSpent.toLocaleString()}`);
|
|
210
|
+
lines.push('');
|
|
211
|
+
|
|
212
|
+
// The curve that matters: new findings per round should fall to zero.
|
|
213
|
+
lines.push('## Did the attacker give up?', '');
|
|
214
|
+
if (s.trend.length) {
|
|
215
|
+
const max = Math.max(...s.trend, 1);
|
|
216
|
+
for (let i = 0; i < s.trend.length; i++) {
|
|
217
|
+
const n = s.trend[i];
|
|
218
|
+
const bar = '█'.repeat(Math.round((n / max) * 24)) || '·';
|
|
219
|
+
lines.push(`round ${i + 1} ${String(n).padStart(3)} new ${bar}`);
|
|
220
|
+
}
|
|
221
|
+
lines.push('');
|
|
222
|
+
lines.push(s.status === 'converged'
|
|
223
|
+
? '**Converged** — two consecutive rounds surfaced nothing new.'
|
|
224
|
+
: `**Not converged** (${s.stopReason || 'still running'}). Remaining risk is unproven, not absent.`);
|
|
225
|
+
} else {
|
|
226
|
+
lines.push('_No rounds completed._');
|
|
227
|
+
}
|
|
228
|
+
lines.push('');
|
|
229
|
+
|
|
230
|
+
// Per-round detail
|
|
231
|
+
lines.push('## Rounds', '');
|
|
232
|
+
for (const r of run.rounds) {
|
|
233
|
+
const verified = (r.artifacts || []).filter(a => a.verified).length;
|
|
234
|
+
lines.push(`### Round ${r.round}`);
|
|
235
|
+
lines.push(`- GOD: ${(r.artifacts || []).length} artifact(s), ${verified} verified` +
|
|
236
|
+
(r.godComplete === false ? ' — **round incomplete**' : ''));
|
|
237
|
+
if (r.unverifiedArtifacts?.length) {
|
|
238
|
+
lines.push(` - unverified: ${r.unverifiedArtifacts.join(', ')}`);
|
|
239
|
+
}
|
|
240
|
+
if (r.outstandingFindings?.length) {
|
|
241
|
+
lines.push(` - not addressed: ${r.outstandingFindings.join(', ')}`);
|
|
242
|
+
}
|
|
243
|
+
lines.push(`- EVIL: ${r.counts.total} finding(s) — ${r.counts.new} new, ${r.counts.confirmed} confirmed, ${r.counts.refuted} refuted`);
|
|
244
|
+
lines.push(`- Open risk after round: ${r.openRisk}`);
|
|
245
|
+
lines.push('');
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// What is still standing
|
|
249
|
+
const last = run.rounds[run.rounds.length - 1];
|
|
250
|
+
const open = last ? evil.actionable(last.findings, { minRisk: 0 }) : [];
|
|
251
|
+
lines.push('## Still open', '');
|
|
252
|
+
if (!open.length) {
|
|
253
|
+
lines.push('_Nothing open. Every finding was either fixed or refuted._');
|
|
254
|
+
} else {
|
|
255
|
+
for (const f of open) {
|
|
256
|
+
lines.push(`- **[${effectiveRisk(f)}] ${f.title}** — ${f.severity}/${f.confidence}` +
|
|
257
|
+
(f.file ? ` · \`${f.file}:${f.line ?? '?'}\`` : ''));
|
|
258
|
+
if (f.repro) lines.push(` - repro: ${String(f.repro).split('\n')[0].slice(0, 160)}`);
|
|
259
|
+
if (f.fix) lines.push(` - fix: ${String(f.fix).split('\n')[0].slice(0, 160)}`);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
lines.push('');
|
|
263
|
+
|
|
264
|
+
// Refuted — kept visible so the same false positive isn't re-litigated.
|
|
265
|
+
const refuted = last ? last.findings.filter(f => f.verdict === VERDICT.REFUTED) : [];
|
|
266
|
+
if (refuted.length) {
|
|
267
|
+
lines.push('## Refuted (checked, not real)', '');
|
|
268
|
+
for (const f of refuted) lines.push(`- ~~${f.title}~~`);
|
|
269
|
+
lines.push('');
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
return lines.join('\n');
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
module.exports = {
|
|
276
|
+
PHASE,
|
|
277
|
+
startArena, nextAction, carriedFindings,
|
|
278
|
+
submitGodWork, submitEvilHunt, submitVerdicts, closeRound,
|
|
279
|
+
affordOrAbort, buildReport,
|
|
280
|
+
};
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
// scripts/arena/contract.js
|
|
2
|
+
// The shared data contract between GOD mode (builds) and EVIL mode (attacks).
|
|
3
|
+
//
|
|
4
|
+
// Without one schema both crews agree on, they cannot exchange results:
|
|
5
|
+
// EVIL produces Findings, GOD consumes them and produces Artifacts, and the
|
|
6
|
+
// Arena scores both to decide whether to run another round.
|
|
7
|
+
//
|
|
8
|
+
// Pure data + validation. Zero dependencies, no I/O.
|
|
9
|
+
|
|
10
|
+
'use strict';
|
|
11
|
+
|
|
12
|
+
// ── Severity / confidence scales ─────────────────────────────────────────────
|
|
13
|
+
// Severity is what it costs if real. Confidence is how sure we are it IS real.
|
|
14
|
+
// Exploitability is how hard it is to actually trigger. Risk multiplies all three
|
|
15
|
+
// so a "critical but almost certainly a false positive" ranks below a
|
|
16
|
+
// "high, confirmed, trivially exploitable".
|
|
17
|
+
|
|
18
|
+
const SEVERITY = { critical: 10, high: 7, medium: 4, low: 2, info: 1 };
|
|
19
|
+
const CONFIDENCE = { confirmed: 1.0, likely: 0.7, suspected: 0.4, speculative: 0.15 };
|
|
20
|
+
const EXPLOITABILITY = { trivial: 1.0, moderate: 0.7, hard: 0.4, theoretical: 0.2 };
|
|
21
|
+
|
|
22
|
+
const VERDICT = {
|
|
23
|
+
CONFIRMED: 'confirmed', // verification reproduced it
|
|
24
|
+
REFUTED: 'refuted', // verification proved it is a false positive
|
|
25
|
+
UNVERIFIED: 'unverified', // not yet challenged
|
|
26
|
+
NEEDS_CONTEXT: 'needs_context', // can't tell without runtime/secrets we don't have
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
// ── Finding ──────────────────────────────────────────────────────────────────
|
|
30
|
+
// One thing EVIL mode believes is wrong. `repro` is what makes a finding
|
|
31
|
+
// actionable rather than an opinion — GOD mode cannot fix what it cannot reproduce.
|
|
32
|
+
|
|
33
|
+
function makeFinding(input = {}) {
|
|
34
|
+
const f = {
|
|
35
|
+
id: input.id || null, // stable hash, assigned by dedupe
|
|
36
|
+
agent: input.agent || 'unknown',
|
|
37
|
+
title: String(input.title || '').slice(0, 200),
|
|
38
|
+
severity: SEVERITY[input.severity] ? input.severity : 'medium',
|
|
39
|
+
confidence: CONFIDENCE[input.confidence] ? input.confidence : 'suspected',
|
|
40
|
+
exploitability: EXPLOITABILITY[input.exploitability] ? input.exploitability : 'moderate',
|
|
41
|
+
file: input.file || null,
|
|
42
|
+
line: Number.isInteger(input.line) ? input.line : null,
|
|
43
|
+
evidence: String(input.evidence || '').slice(0, 2000),
|
|
44
|
+
repro: input.repro ? String(input.repro).slice(0, 2000) : null,
|
|
45
|
+
fix: input.fix ? String(input.fix).slice(0, 2000) : null,
|
|
46
|
+
verdict: VERDICT[String(input.verdict || '').toUpperCase()] || input.verdict || VERDICT.UNVERIFIED,
|
|
47
|
+
round: Number.isInteger(input.round) ? input.round : 0,
|
|
48
|
+
};
|
|
49
|
+
f.id = f.id || fingerprint(f);
|
|
50
|
+
f.risk = riskScore(f);
|
|
51
|
+
return f;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Stable identity: same issue in the same place is the same finding across
|
|
55
|
+
// rounds, even if the wording of the title drifts between agents.
|
|
56
|
+
function fingerprint(f) {
|
|
57
|
+
const basis = [
|
|
58
|
+
(f.file || 'nofile').toLowerCase(),
|
|
59
|
+
f.line == null ? 'noline' : String(f.line),
|
|
60
|
+
normalizeTitle(f.title),
|
|
61
|
+
].join('::');
|
|
62
|
+
// FNV-1a — deterministic, dependency-free, good enough for dedupe keys.
|
|
63
|
+
let h = 0x811c9dc5;
|
|
64
|
+
for (let i = 0; i < basis.length; i++) {
|
|
65
|
+
h ^= basis.charCodeAt(i);
|
|
66
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
67
|
+
}
|
|
68
|
+
return h.toString(16).padStart(8, '0');
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function normalizeTitle(title) {
|
|
72
|
+
return String(title || '')
|
|
73
|
+
.toLowerCase()
|
|
74
|
+
.replace(/[^a-z0-9 ]+/g, ' ')
|
|
75
|
+
.replace(/\b(a|an|the|is|are|in|on|of|to|for|with|at|by)\b/g, ' ')
|
|
76
|
+
.replace(/\s+/g, ' ')
|
|
77
|
+
.trim()
|
|
78
|
+
.slice(0, 80);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// risk = severity × confidence × exploitability → 0..10
|
|
82
|
+
function riskScore(f) {
|
|
83
|
+
const s = SEVERITY[f.severity] || SEVERITY.medium;
|
|
84
|
+
const c = CONFIDENCE[f.confidence] || CONFIDENCE.suspected;
|
|
85
|
+
const e = EXPLOITABILITY[f.exploitability] || EXPLOITABILITY.moderate;
|
|
86
|
+
return Math.round(s * c * e * 100) / 100;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Refuted findings carry no risk — that is the whole point of verification.
|
|
90
|
+
function effectiveRisk(f) {
|
|
91
|
+
return f.verdict === VERDICT.REFUTED ? 0 : riskScore(f);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ── Dedupe / diff across rounds ──────────────────────────────────────────────
|
|
95
|
+
|
|
96
|
+
function dedupe(findings = []) {
|
|
97
|
+
const byId = new Map();
|
|
98
|
+
for (const raw of findings) {
|
|
99
|
+
const f = raw && raw.id && raw.risk != null ? raw : makeFinding(raw || {});
|
|
100
|
+
const prev = byId.get(f.id);
|
|
101
|
+
// Keep the strongest claim when two agents report the same thing.
|
|
102
|
+
if (!prev || effectiveRisk(f) > effectiveRisk(prev)) byId.set(f.id, f);
|
|
103
|
+
}
|
|
104
|
+
return [...byId.values()].sort((a, b) => effectiveRisk(b) - effectiveRisk(a));
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// What EVIL found this round that it had never found before.
|
|
108
|
+
function newFindings(current = [], seenIds = []) {
|
|
109
|
+
const seen = new Set(seenIds);
|
|
110
|
+
return dedupe(current).filter(f => !seen.has(f.id));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// ── Artifact (what GOD produces) ─────────────────────────────────────────────
|
|
114
|
+
// GOD must emit something checkable. `verified` is only true when a command
|
|
115
|
+
// actually ran and passed — claims alone never count.
|
|
116
|
+
|
|
117
|
+
function makeArtifact(input = {}) {
|
|
118
|
+
return {
|
|
119
|
+
kind: ['code', 'test', 'doc', 'config', 'benchmark'].includes(input.kind) ? input.kind : 'code',
|
|
120
|
+
files: Array.isArray(input.files) ? input.files.slice(0, 50) : [],
|
|
121
|
+
summary: String(input.summary || '').slice(0, 1000),
|
|
122
|
+
verifyCommand: input.verifyCommand ? String(input.verifyCommand).slice(0, 300) : null,
|
|
123
|
+
verified: input.verified === true,
|
|
124
|
+
round: Number.isInteger(input.round) ? input.round : 0,
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// ── Round verdict ────────────────────────────────────────────────────────────
|
|
129
|
+
|
|
130
|
+
function makeRoundVerdict(input = {}) {
|
|
131
|
+
const findings = dedupe(input.findings || []);
|
|
132
|
+
const fresh = Array.isArray(input.newFindingIds) ? input.newFindingIds : [];
|
|
133
|
+
const open = findings.filter(f => f.verdict !== VERDICT.REFUTED);
|
|
134
|
+
const confirmed = findings.filter(f => f.verdict === VERDICT.CONFIRMED);
|
|
135
|
+
return {
|
|
136
|
+
round: Number.isInteger(input.round) ? input.round : 0,
|
|
137
|
+
findings,
|
|
138
|
+
newFindingIds: fresh,
|
|
139
|
+
counts: {
|
|
140
|
+
total: findings.length,
|
|
141
|
+
open: open.length,
|
|
142
|
+
confirmed: confirmed.length,
|
|
143
|
+
refuted: findings.length - open.length,
|
|
144
|
+
new: fresh.length,
|
|
145
|
+
},
|
|
146
|
+
// Total unmitigated risk still standing after verification.
|
|
147
|
+
openRisk: Math.round(open.reduce((sum, f) => sum + effectiveRisk(f), 0) * 100) / 100,
|
|
148
|
+
artifacts: (input.artifacts || []).map(makeArtifact),
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// ── Convergence ──────────────────────────────────────────────────────────────
|
|
153
|
+
// The arena stops when the attacker gives up: N consecutive rounds where EVIL
|
|
154
|
+
// found nothing genuinely new. Not "zero findings" — findings may remain open
|
|
155
|
+
// and accepted; what matters is that attacking harder stops yielding anything.
|
|
156
|
+
|
|
157
|
+
function hasConverged(roundVerdicts = [], quietRoundsRequired = 2) {
|
|
158
|
+
if (roundVerdicts.length < quietRoundsRequired) return false;
|
|
159
|
+
return roundVerdicts
|
|
160
|
+
.slice(-quietRoundsRequired)
|
|
161
|
+
.every(r => (r.counts?.new || 0) === 0);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
module.exports = {
|
|
165
|
+
SEVERITY, CONFIDENCE, EXPLOITABILITY, VERDICT,
|
|
166
|
+
makeFinding, fingerprint, riskScore, effectiveRisk,
|
|
167
|
+
dedupe, newFindings,
|
|
168
|
+
makeArtifact, makeRoundVerdict, hasConverged,
|
|
169
|
+
};
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
// scripts/arena/evil.js
|
|
2
|
+
// EVIL mode — the adversarial half of the arena.
|
|
3
|
+
//
|
|
4
|
+
// The 8 devil-mode agents already hunt well (real ripgrep/jq detection commands).
|
|
5
|
+
// What they lacked was judgment: everything came back as an undifferentiated pile
|
|
6
|
+
// with no way to tell a confirmed exploit from a hunch.
|
|
7
|
+
//
|
|
8
|
+
// This module adds the three missing pieces:
|
|
9
|
+
// 1. SCORING — severity x confidence x exploitability, so noise sinks
|
|
10
|
+
// 2. VERIFICATION — each finding gets challenged: prove it, or it is refuted
|
|
11
|
+
// 3. LOOP-UNTIL-DRY — keep sweeping until two rounds turn up nothing new
|
|
12
|
+
//
|
|
13
|
+
// It does not call an LLM itself. It builds the agent briefs the orchestrator
|
|
14
|
+
// dispatches, and it grades what comes back. That keeps this file testable.
|
|
15
|
+
|
|
16
|
+
'use strict';
|
|
17
|
+
|
|
18
|
+
const { makeFinding, dedupe, VERDICT, effectiveRisk } = require('./contract');
|
|
19
|
+
|
|
20
|
+
// ── The crew ─────────────────────────────────────────────────────────────────
|
|
21
|
+
// `core` fires on every sweep. The rest are opt-in via flags, because each one
|
|
22
|
+
// costs a full agent invocation and not every task needs a license audit.
|
|
23
|
+
|
|
24
|
+
const CREW = {
|
|
25
|
+
'prompt-injection-hunter': { core: true, hunts: 'jailbreaks, indirect injection, system-prompt leaks, tool-call hijacking' },
|
|
26
|
+
'supply-chain-auditor': { core: true, hunts: 'typosquats, malicious install scripts, lockfile drift, unsigned packages' },
|
|
27
|
+
'secret-hunter': { core: true, hunts: 'live credentials, git-history leaks, encoded keys, client-bundled env vars' },
|
|
28
|
+
'backdoor-hunter': { core: true, hunts: 'obfuscated payloads, network beacons, time bombs, eval/exec abuse' },
|
|
29
|
+
'license-violation-finder':{ core: false, flag: '--license', hunts: 'GPL contamination, missing attribution, copyleft risk' },
|
|
30
|
+
'code-stealer-detector': { core: false, flag: '--theft', hunts: 'copy-paste provenance, leaked private code, AI-gen origin' },
|
|
31
|
+
'jailbreak-tester': { core: false, flag: '--jailbreak', hunts: 'live AI-feature red-team, refusal bypass, overrefusal' },
|
|
32
|
+
'chaos-engineer': { core: false, flag: '--chaos', hunts: 'fault injection, resource exhaustion, hidden assumptions' },
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
const PRESETS = {
|
|
36
|
+
default: ['--core'],
|
|
37
|
+
'--all': Object.keys(CREW),
|
|
38
|
+
'--pre-public':['--core', '--license', '--theft'],
|
|
39
|
+
'--pre-launch':['--core', '--jailbreak', '--chaos'],
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
function selectCrew(flags = []) {
|
|
43
|
+
const set = new Set(Object.entries(CREW).filter(([, m]) => m.core).map(([n]) => n));
|
|
44
|
+
if (flags.includes('--all')) return Object.keys(CREW);
|
|
45
|
+
for (const [name, meta] of Object.entries(CREW)) {
|
|
46
|
+
if (meta.flag && flags.includes(meta.flag)) set.add(name);
|
|
47
|
+
}
|
|
48
|
+
for (const preset of ['--pre-public', '--pre-launch']) {
|
|
49
|
+
if (flags.includes(preset)) {
|
|
50
|
+
for (const f of PRESETS[preset]) {
|
|
51
|
+
if (f === '--core') continue;
|
|
52
|
+
const hit = Object.entries(CREW).find(([, m]) => m.flag === f);
|
|
53
|
+
if (hit) set.add(hit[0]);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
return [...set];
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// ── Stage 1: hunt briefs ─────────────────────────────────────────────────────
|
|
61
|
+
// Each agent is told to report in the Finding contract, and — critically — that
|
|
62
|
+
// a finding without a reproduction is worth less than no finding at all.
|
|
63
|
+
|
|
64
|
+
function huntBrief({ agent, scope, round, knownFindingIds = [] }) {
|
|
65
|
+
const meta = CREW[agent] || { hunts: 'issues' };
|
|
66
|
+
return [
|
|
67
|
+
`You are ${agent}. Hunt: ${meta.hunts}.`,
|
|
68
|
+
`Scope: ${scope || 'the whole repository'}.`,
|
|
69
|
+
`Round ${round}.`,
|
|
70
|
+
knownFindingIds.length
|
|
71
|
+
? `Already-reported findings (${knownFindingIds.length}) are known — do NOT re-report them. Hunt for what a previous sweep MISSED. Go deeper: different files, different attack classes, indirect paths.`
|
|
72
|
+
: `First sweep — cover your full detection surface.`,
|
|
73
|
+
'',
|
|
74
|
+
'Report every finding as JSON matching this shape:',
|
|
75
|
+
'{ "title", "severity": critical|high|medium|low|info, "confidence": confirmed|likely|suspected|speculative,',
|
|
76
|
+
' "exploitability": trivial|moderate|hard|theoretical, "file", "line", "evidence", "repro", "fix" }',
|
|
77
|
+
'',
|
|
78
|
+
'Rules that decide whether your finding survives review:',
|
|
79
|
+
'- `evidence` must quote the actual offending code, not describe it.',
|
|
80
|
+
'- `repro` must be concrete steps or a command. A finding you cannot reproduce is `speculative` at best.',
|
|
81
|
+
'- Do not pad. Five confirmed findings beat forty guesses — unverifiable findings get refuted and count against you.',
|
|
82
|
+
'- Only claim `confirmed` when you have actually reproduced it.',
|
|
83
|
+
].join('\n');
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ── Stage 2: verification briefs ─────────────────────────────────────────────
|
|
87
|
+
// The adversarial-verification trick: the verifier is told to REFUTE, not to
|
|
88
|
+
// confirm. Default-to-refuted kills the false positives that otherwise waste
|
|
89
|
+
// GOD mode's entire next round.
|
|
90
|
+
|
|
91
|
+
function verifyBrief(finding) {
|
|
92
|
+
return [
|
|
93
|
+
`Adversarially verify this finding. Your job is to REFUTE it.`,
|
|
94
|
+
'',
|
|
95
|
+
`Title: ${finding.title}`,
|
|
96
|
+
`Claimed severity: ${finding.severity} (${finding.confidence}, ${finding.exploitability})`,
|
|
97
|
+
`Location: ${finding.file || 'unknown'}:${finding.line ?? '?'}`,
|
|
98
|
+
`Evidence: ${finding.evidence || '(none given)'}`,
|
|
99
|
+
`Claimed repro: ${finding.repro || '(none given)'}`,
|
|
100
|
+
'',
|
|
101
|
+
'Do this:',
|
|
102
|
+
'1. Read the actual code at that location. Does the quoted evidence match reality?',
|
|
103
|
+
'2. Check whether guards, framework behaviour, or callers already neutralise it.',
|
|
104
|
+
'3. Try to reproduce it. If you cannot, say so plainly.',
|
|
105
|
+
'',
|
|
106
|
+
'Return JSON: { "verdict": "confirmed"|"refuted"|"needs_context", "why": "...", "repro": "..."|null }',
|
|
107
|
+
'',
|
|
108
|
+
'Default to "refuted" when uncertain. A finding that cannot be demonstrated is not a finding.',
|
|
109
|
+
'Use "needs_context" only when verification genuinely requires runtime access or secrets you do not have.',
|
|
110
|
+
].join('\n');
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// ── Stage 3: grade the returns ───────────────────────────────────────────────
|
|
114
|
+
|
|
115
|
+
function normalizeFindings(rawList = [], { agent, round }) {
|
|
116
|
+
const out = [];
|
|
117
|
+
for (const raw of rawList) {
|
|
118
|
+
if (!raw || !raw.title) continue;
|
|
119
|
+
out.push(makeFinding({ ...raw, agent, round }));
|
|
120
|
+
}
|
|
121
|
+
return out;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
// Apply verification verdicts back onto the findings. A refuted finding stays in
|
|
125
|
+
// the record (so the same false positive is not re-litigated next round) but its
|
|
126
|
+
// effective risk drops to zero.
|
|
127
|
+
function applyVerdicts(findings = [], verdicts = {}) {
|
|
128
|
+
return findings.map(f => {
|
|
129
|
+
const v = verdicts[f.id];
|
|
130
|
+
if (!v) return f;
|
|
131
|
+
const next = { ...f, verdict: v.verdict || f.verdict };
|
|
132
|
+
if (v.repro && !next.repro) next.repro = String(v.repro).slice(0, 2000);
|
|
133
|
+
if (v.why) next.evidence = `${next.evidence}\n[verification] ${v.why}`.slice(0, 2000);
|
|
134
|
+
// Confirming a finding raises certainty; refuting zeroes it out via effectiveRisk.
|
|
135
|
+
if (next.verdict === VERDICT.CONFIRMED) next.confidence = 'confirmed';
|
|
136
|
+
return makeFinding(next);
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Which findings are worth spending a verification pass on. Verifying an
|
|
141
|
+
// already-confirmed or clearly-trivial item is wasted tokens.
|
|
142
|
+
function selectForVerification(findings = [], { max = 12, minRisk = 1.0 } = {}) {
|
|
143
|
+
return dedupe(findings)
|
|
144
|
+
.filter(f => f.verdict === VERDICT.UNVERIFIED && f.risk >= minRisk)
|
|
145
|
+
.slice(0, max);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// What GOD mode actually has to fix: open, verified-or-plausible, worst first.
|
|
149
|
+
function actionable(findings = [], { minRisk = 2.0 } = {}) {
|
|
150
|
+
return dedupe(findings)
|
|
151
|
+
.filter(f => f.verdict !== VERDICT.REFUTED)
|
|
152
|
+
.filter(f => effectiveRisk(f) >= minRisk)
|
|
153
|
+
.sort((a, b) => effectiveRisk(b) - effectiveRisk(a));
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// ── Sweep plan ───────────────────────────────────────────────────────────────
|
|
157
|
+
// The orchestrator asks for a plan, dispatches the agents, and feeds results back.
|
|
158
|
+
|
|
159
|
+
function planSweep({ scope, flags = [], round = 1, knownFindingIds = [] } = {}) {
|
|
160
|
+
const crew = selectCrew(flags);
|
|
161
|
+
return {
|
|
162
|
+
round,
|
|
163
|
+
crew,
|
|
164
|
+
briefs: crew.map(agent => ({ agent, brief: huntBrief({ agent, scope, round, knownFindingIds }) })),
|
|
165
|
+
estimatedTokens: crew.length * 12000, // rough: one agent sweep ≈ 12k
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
module.exports = {
|
|
170
|
+
CREW, PRESETS,
|
|
171
|
+
selectCrew, planSweep,
|
|
172
|
+
huntBrief, verifyBrief,
|
|
173
|
+
normalizeFindings, applyVerdicts,
|
|
174
|
+
selectForVerification, actionable,
|
|
175
|
+
};
|