atris 3.82.8 → 3.82.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/sentinel-bench.js +83 -5
- package/lib/sentinel-bench.js +50 -2
- package/lib/sentinel-guard.js +198 -10
- package/lib/sentinel-improve.js +136 -19
- package/lib/sentinel-night.js +910 -0
- package/lib/sentinel-rules.js +15 -3
- package/lib/sentinel-rules.json +10 -1
- package/package.json +1 -1
|
@@ -0,0 +1,910 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// atris sentinel improve --rounds N --parallel K --review codex: the night run.
|
|
4
|
+
//
|
|
5
|
+
// One round:
|
|
6
|
+
//
|
|
7
|
+
// propose K sealed Claude proposers at once, each in its own temp copy
|
|
8
|
+
// with every seal rule of a single run (see sentinel-improve.js)
|
|
9
|
+
// and each pointed at a different focus area. That way they do
|
|
10
|
+
// not all chase the same miss
|
|
11
|
+
// judge each proposal alone, with the same exam and keep rule as a
|
|
12
|
+
// single run
|
|
13
|
+
// review with --review codex, each change the exam kept goes to
|
|
14
|
+
// `codex exec -s read-only` with stdin closed. It sees the diff
|
|
15
|
+
// and the seen exam only. It answers one strict json object:
|
|
16
|
+
// SHIP keeps the change, FIX or anything malformed rejects it
|
|
17
|
+
// learn every everyday command the reviewer says a change wrongly
|
|
18
|
+
// blocks becomes a seen item expected to run, every bypass a
|
|
19
|
+
// seen item expected to block and every agent false alarm a
|
|
20
|
+
// clean agent item. Each is tagged source "review", checked
|
|
21
|
+
// for anything that looks like a real secret and deduplicated.
|
|
22
|
+
// They go on their own branch, sentinel-exam/<date>
|
|
23
|
+
// (sentinel-exam/<date>-<short id> on the remote when an earlier
|
|
24
|
+
// run that day already pushed that name)
|
|
25
|
+
// land the union of the shipped changes is judged again on the grown
|
|
26
|
+
// exam (and reviewed when it holds more than one change). If it
|
|
27
|
+
// fails, the best single shipped change by held-out gain, then
|
|
28
|
+
// older seen gain, is judged again the same way. The first that
|
|
29
|
+
// passes lands on sentinel-improve/night-<date>
|
|
30
|
+
//
|
|
31
|
+
// The next round starts from the landed rules and the grown exam. The run
|
|
32
|
+
// stops after two rounds in a row with nothing shipped, two rounds in a row
|
|
33
|
+
// where held-out catch did not rise, the time limit or the call cap.
|
|
34
|
+
//
|
|
35
|
+
// The guarantee, on top of everything a single run keeps:
|
|
36
|
+
//
|
|
37
|
+
// the proposer is sealed exactly as in a single run. It never writes
|
|
38
|
+
// exam items from the review; only this file does, from the reviewer's
|
|
39
|
+
// parsed answer
|
|
40
|
+
// the reviewer runs in an empty temp folder holding the diff and the seen
|
|
41
|
+
// exam, with an empty temp HOME (its own login folder is pointed to by
|
|
42
|
+
// CODEX_HOME), a stripped environment and no mcp servers. Its prompt tells
|
|
43
|
+
// it to read nothing outside that folder. Nothing here passes it the
|
|
44
|
+
// held-out folder, a held-out item or the key
|
|
45
|
+
// a night run refuses to start its engines unless the judge reports that
|
|
46
|
+
// the held-out key came from the keychain. A key file could be read
|
|
47
|
+
// every review example passes three checks before it joins the exam: the
|
|
48
|
+
// strict secret check (no suppression marker, no placeholder pass), a
|
|
49
|
+
// dedupe and the judge's screen. The screen refuses any example that equals,
|
|
50
|
+
// contains or sits inside a held-out item. The screen runs in the judge's
|
|
51
|
+
// own process. Only refused ids leave it. If it cannot run, nothing joins
|
|
52
|
+
// the reviewer answer is cut at 64 KB and its lists at the item limit
|
|
53
|
+
// before anything is read. Missing, extra or mistyped fields mean FIX
|
|
54
|
+
// receipts are scanned line by line before they are written
|
|
55
|
+
// nothing lands without the exam passing on the exam as it stands at
|
|
56
|
+
// landing time, review items included
|
|
57
|
+
// one night run at a time per machine, by a lock file holding its pid.
|
|
58
|
+
// A lock is never taken over. Only --break-lock clears it, and only when
|
|
59
|
+
// its owner is gone
|
|
60
|
+
// the reviewer's reason text is dropped after its shape is checked. Only
|
|
61
|
+
// its verdict and the counts of accepted examples go anywhere
|
|
62
|
+
// proposer exam items pass the same strict secret check as review items.
|
|
63
|
+
// One flagged item rejects the whole proposal before it is judged
|
|
64
|
+
// every copy goes through the same safe cleanup as a single run
|
|
65
|
+
// nothing merges, releases or pushes to the main branch. With --push the
|
|
66
|
+
// two night branches are pushed by name and nothing else
|
|
67
|
+
//
|
|
68
|
+
// Accepted limits:
|
|
69
|
+
//
|
|
70
|
+
// codex read-only can still read files on this computer outside its
|
|
71
|
+
// folder. The held-out half is encrypted at rest and its key sits in the
|
|
72
|
+
// keychain. The empty HOME does not link it in. Even a read that worked
|
|
73
|
+
// could not move a held-out item into the seen half past the screen
|
|
74
|
+
// a union of changes that each shipped is judged again by the exam but a
|
|
75
|
+
// single shipped change that lands is not reviewed a second time
|
|
76
|
+
|
|
77
|
+
const fs = require('fs');
|
|
78
|
+
const os = require('os');
|
|
79
|
+
const path = require('path');
|
|
80
|
+
const crypto = require('crypto');
|
|
81
|
+
const imp = require('./sentinel-improve');
|
|
82
|
+
|
|
83
|
+
const { FOCI, RULES_PATH, SEEN_REL } = imp;
|
|
84
|
+
const MAX_PARALLEL = FOCI.length;
|
|
85
|
+
const MAX_REVIEW_ITEMS = 12;
|
|
86
|
+
const CATCH = { guard: 'blocked', agents: 'found' };
|
|
87
|
+
|
|
88
|
+
const PUSH_MS = 2 * 60 * 1000;
|
|
89
|
+
|
|
90
|
+
class TimeUp extends Error {
|
|
91
|
+
constructor() { super('the time limit ran out'); this.timeUp = true; }
|
|
92
|
+
}
|
|
93
|
+
const isTimeUp = (e) => Boolean(e && (e.timeUp || /ran out of time/.test(String(e.message || ''))));
|
|
94
|
+
|
|
95
|
+
const isObj = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
96
|
+
const sha = (s) => crypto.createHash('sha256').update(String(s)).digest('hex');
|
|
97
|
+
|
|
98
|
+
function dateTag(now = new Date()) {
|
|
99
|
+
return now.toISOString().slice(0, 10).replace(/-/g, '');
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// ---------------------------------------------------------------- lock
|
|
103
|
+
|
|
104
|
+
function lockPath(env) {
|
|
105
|
+
return path.join(imp.stateDir(env), 'night.lock');
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function readLock(file) {
|
|
109
|
+
try { const v = JSON.parse(fs.readFileSync(file, 'utf8')); return isObj(v) ? v : null; } catch { return null; }
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function lockAge(file, now) {
|
|
113
|
+
try { return Math.max(0, Math.round((now() - fs.statSync(file).mtimeMs) / 60000)); } catch { return null; }
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const BREAK_HINT = 'clear it with: atris sentinel improve --break-lock';
|
|
117
|
+
|
|
118
|
+
// One night run at a time. The lock file is made only by an exclusive
|
|
119
|
+
// create and holds the run's pid, process group and a random token. There
|
|
120
|
+
// is no automatic takeover: when a lock exists the run stops and says who
|
|
121
|
+
// holds it and how old it is. A person clears a dead owner's lock with
|
|
122
|
+
// --break-lock (breakLock below).
|
|
123
|
+
function takeLock(env, { now = Date.now } = {}) {
|
|
124
|
+
const file = lockPath(env);
|
|
125
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
126
|
+
const rec = { pid: process.pid, pgid: imp.pgidOf(process.pid), started: new Date(now()).toISOString(), token: crypto.randomBytes(8).toString('hex') };
|
|
127
|
+
let fd;
|
|
128
|
+
try { fd = fs.openSync(file, 'wx', 0o600); } catch (e) {
|
|
129
|
+
if (e.code !== 'EEXIST') throw e;
|
|
130
|
+
const held = readLock(file);
|
|
131
|
+
const age = lockAge(file, now);
|
|
132
|
+
const ageText = age === null ? 'of unknown age' : `${age} ${age === 1 ? 'minute' : 'minutes'} old`;
|
|
133
|
+
if (!held) throw new Error(`a night run lock exists at ${file} that cannot be read (${ageText}). if no night run is going, remove that file by hand`);
|
|
134
|
+
const alive = imp.runAlive(held);
|
|
135
|
+
throw new Error(`another night run holds the lock: pid ${held.pid}, ${alive ? 'still running' : 'no longer running'}, ${ageText}. ${alive ? 'wait for it to finish' : BREAK_HINT}`);
|
|
136
|
+
}
|
|
137
|
+
fs.writeSync(fd, `${JSON.stringify(rec)}\n`);
|
|
138
|
+
fs.closeSync(fd);
|
|
139
|
+
return {
|
|
140
|
+
file,
|
|
141
|
+
release() { const held = readLock(file); if (held && held.token === rec.token) { try { fs.unlinkSync(file); } catch {} } },
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
// --break-lock: removes the lock only when its owner is not alive. A lock
|
|
146
|
+
// that cannot be read is left alone.
|
|
147
|
+
function breakLock(env) {
|
|
148
|
+
const file = lockPath(env);
|
|
149
|
+
if (!fs.existsSync(file)) return { removed: false, message: 'there is no night run lock' };
|
|
150
|
+
const held = readLock(file);
|
|
151
|
+
if (!held) return { removed: false, refused: true, message: `the lock at ${file} cannot be read. remove it by hand only if no night run is going` };
|
|
152
|
+
if (imp.runAlive(held)) return { removed: false, refused: true, message: `the night run that holds the lock is still running (pid ${held.pid}). the lock stays` };
|
|
153
|
+
const again = readLock(file);
|
|
154
|
+
if (!again || again.token !== held.token) return { removed: false, refused: true, message: 'the lock changed while it was checked. the lock stays' };
|
|
155
|
+
fs.unlinkSync(file);
|
|
156
|
+
return { removed: true, message: `removed the lock of pid ${held.pid}. that run is no longer running` };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// ---------------------------------------------------------------- budget
|
|
160
|
+
|
|
161
|
+
function makeBudget({ maxMinutes, maxCalls, clock = Date.now }) {
|
|
162
|
+
const start = clock();
|
|
163
|
+
const b = {
|
|
164
|
+
start,
|
|
165
|
+
deadline: start + maxMinutes * 60 * 1000,
|
|
166
|
+
maxCalls,
|
|
167
|
+
calls: 0,
|
|
168
|
+
leftMs: () => b.deadline - clock(),
|
|
169
|
+
minutes: () => Math.round(((clock() - start) / 60000) * 10) / 10,
|
|
170
|
+
// Takes up to n calls from the cap and returns how many it got.
|
|
171
|
+
take(n) { const got = Math.max(0, Math.min(n, b.maxCalls - b.calls)); b.calls += got; return got; },
|
|
172
|
+
};
|
|
173
|
+
return b;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
// ---------------------------------------------------------------- focus
|
|
177
|
+
|
|
178
|
+
// K different areas. --focus goes first. The rest follow by how many seen
|
|
179
|
+
// misses each has. Ties keep the fixed order.
|
|
180
|
+
function assignFoci(report, k, first = null) {
|
|
181
|
+
const counts = imp.missCounts(report);
|
|
182
|
+
const rest = FOCI.filter((f) => f !== first).sort((a, b) => (counts[b] - counts[a]) || (FOCI.indexOf(a) - FOCI.indexOf(b)));
|
|
183
|
+
return [...(first ? [first] : []), ...rest].slice(0, Math.max(0, Math.min(k, MAX_PARALLEL)));
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// ---------------------------------------------------------------- union
|
|
187
|
+
|
|
188
|
+
// Three-way merge of literal rules: entries any change added are added,
|
|
189
|
+
// entries any change removed are removed, a value two changes set
|
|
190
|
+
// differently is a conflict. Returns { rules } or { conflict }.
|
|
191
|
+
function mergeNode(b, cs, where) {
|
|
192
|
+
const key = (x) => JSON.stringify(x);
|
|
193
|
+
if (Array.isArray(b)) {
|
|
194
|
+
if (!cs.every(Array.isArray)) return { conflict: `${where} is a list in one change and not in another` };
|
|
195
|
+
const baseKeys = new Set(b.map(key));
|
|
196
|
+
const removed = new Set();
|
|
197
|
+
const added = [];
|
|
198
|
+
const seen = new Set();
|
|
199
|
+
for (const c of cs) {
|
|
200
|
+
const ck = new Set(c.map(key));
|
|
201
|
+
for (const x of b) if (!ck.has(key(x))) removed.add(key(x));
|
|
202
|
+
for (const x of c) if (!baseKeys.has(key(x)) && !seen.has(key(x))) { seen.add(key(x)); added.push(x); }
|
|
203
|
+
}
|
|
204
|
+
const out = [...b.filter((x) => !removed.has(key(x))), ...added];
|
|
205
|
+
const ids = out.filter((x) => isObj(x) && typeof x.id === 'string').map((x) => x.id);
|
|
206
|
+
if (new Set(ids).size !== ids.length) return { conflict: `two changes add different ${where} entries with the same id` };
|
|
207
|
+
return { value: out };
|
|
208
|
+
}
|
|
209
|
+
if (isObj(b)) {
|
|
210
|
+
const out = {};
|
|
211
|
+
const keys = new Set([...Object.keys(b), ...cs.flatMap((c) => (isObj(c) ? Object.keys(c) : []))]);
|
|
212
|
+
if (!cs.every(isObj)) return { conflict: `${where} is an object in one change and not in another` };
|
|
213
|
+
for (const k of keys) {
|
|
214
|
+
const w = where ? `${where}.${k}` : k;
|
|
215
|
+
if (k in b) {
|
|
216
|
+
const gone = cs.filter((c) => !(k in c));
|
|
217
|
+
const kept = cs.filter((c) => k in c);
|
|
218
|
+
if (gone.length) {
|
|
219
|
+
if (kept.some((c) => key(c[k]) !== key(b[k]))) return { conflict: `${w} is removed by one change and changed by another` };
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
const r = mergeNode(b[k], kept.map((c) => c[k]), w);
|
|
223
|
+
if (r.conflict) return r;
|
|
224
|
+
out[k] = r.value;
|
|
225
|
+
} else {
|
|
226
|
+
const vals = [...new Set(cs.filter((c) => k in c).map((c) => key(c[k])))];
|
|
227
|
+
if (vals.length > 1) return { conflict: `${w} is added differently by two changes` };
|
|
228
|
+
out[k] = JSON.parse(vals[0]);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return { value: out };
|
|
232
|
+
}
|
|
233
|
+
const changed = [...new Set(cs.map(key).filter((v) => v !== key(b)))];
|
|
234
|
+
if (changed.length > 1) return { conflict: `${where} is set differently by two changes` };
|
|
235
|
+
return { value: changed.length ? JSON.parse(changed[0]) : b };
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
function unionRules(base, list) {
|
|
239
|
+
const r = mergeNode(base, list, '');
|
|
240
|
+
if (r.conflict) return { conflict: r.conflict };
|
|
241
|
+
const problems = require('./sentinel-rules').validateRules(r.value);
|
|
242
|
+
if (problems.length) return { conflict: `the union breaks the schema: ${problems.slice(0, 2).join('; ')}` };
|
|
243
|
+
return { rules: r.value };
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Plain lines naming each literal change, for the reviewer and the receipt.
|
|
247
|
+
function literalChanges(b, c, where = '') {
|
|
248
|
+
const key = (x) => JSON.stringify(x);
|
|
249
|
+
const out = [];
|
|
250
|
+
if (Array.isArray(b) && Array.isArray(c)) {
|
|
251
|
+
const bk = new Set(b.map(key));
|
|
252
|
+
const ck = new Set(c.map(key));
|
|
253
|
+
for (const x of c) if (!bk.has(key(x))) out.push(`added to ${where}: ${key(x)}`);
|
|
254
|
+
for (const x of b) if (!ck.has(key(x))) out.push(`removed from ${where}: ${key(x)}`);
|
|
255
|
+
return out;
|
|
256
|
+
}
|
|
257
|
+
if (isObj(b) && isObj(c)) {
|
|
258
|
+
for (const k of new Set([...Object.keys(b), ...Object.keys(c)])) {
|
|
259
|
+
const w = where ? `${where}.${k}` : k;
|
|
260
|
+
if (!(k in c)) out.push(`removed ${w}`);
|
|
261
|
+
else if (!(k in b)) out.push(`added ${w}: ${key(c[k])}`);
|
|
262
|
+
else out.push(...literalChanges(b[k], c[k], w));
|
|
263
|
+
}
|
|
264
|
+
return out;
|
|
265
|
+
}
|
|
266
|
+
if (key(b) !== key(c)) out.push(`${where}: ${key(b)} became ${key(c)}`);
|
|
267
|
+
return out;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function unifiedDiff(baseRules, candRules) {
|
|
271
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'atris-sentinel-diff-'));
|
|
272
|
+
try {
|
|
273
|
+
fs.mkdirSync(path.join(dir, 'a', 'lib'), { recursive: true });
|
|
274
|
+
fs.mkdirSync(path.join(dir, 'b', 'lib'), { recursive: true });
|
|
275
|
+
fs.writeFileSync(path.join(dir, 'a', RULES_PATH), `${JSON.stringify(baseRules, null, 2)}\n`);
|
|
276
|
+
fs.writeFileSync(path.join(dir, 'b', RULES_PATH), `${JSON.stringify(candRules, null, 2)}\n`);
|
|
277
|
+
const r = imp.git(dir, ['diff', '--no-index', '--no-color', '--', `a/${RULES_PATH}`, `b/${RULES_PATH}`], { check: false });
|
|
278
|
+
return String(r.stdout || '');
|
|
279
|
+
} finally { fs.rmSync(dir, { recursive: true, force: true }); }
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// ---------------------------------------------------------------- gains
|
|
283
|
+
|
|
284
|
+
function gainOf(before, after, newIds = []) {
|
|
285
|
+
let held = 0;
|
|
286
|
+
for (const set of Object.keys(CATCH)) {
|
|
287
|
+
const hb = (before.holdout || {})[set];
|
|
288
|
+
const ha = (after.holdout || {})[set];
|
|
289
|
+
if (hb && ha) held += (ha[CATCH[set]] || 0) - (hb[CATCH[set]] || 0);
|
|
290
|
+
}
|
|
291
|
+
const fresh = new Set(newIds);
|
|
292
|
+
const old = (r) => [...imp.okMap(r).entries()].filter(([k, ok]) => ok && !fresh.has(k)).length;
|
|
293
|
+
return { held, seen: old(after) - old(before) };
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
const byGain = (a, b) => (b.gain.held - a.gain.held) || (b.gain.seen - a.gain.seen);
|
|
297
|
+
|
|
298
|
+
// ---------------------------------------------------------------- review
|
|
299
|
+
|
|
300
|
+
function reviewPrompt({ title, why, diff, literal, seen }) {
|
|
301
|
+
return [
|
|
302
|
+
'You are a strict reviewer of one proposed change to the atris sentinel, a fixed-rule safety check that coding agents run before every shell command.',
|
|
303
|
+
'The guard blocks a command when it matches the rules. A false alarm on an everyday developer command is the worst outcome: people turn the guard off.',
|
|
304
|
+
'',
|
|
305
|
+
'How the rules are used:',
|
|
306
|
+
' guard.command_blocks: {"id", "category", "all": [literal substrings], "programs"?: [program names], "inline"?, "needs_secret"?, "needs_remote"?, "unless"?: [literal substrings]}. A block fires when the command runs one of its programs (if listed) and contains every substring in all, by a plain includes. inline: true reads only the inline code of python -c, node -e, perl -e, ruby -e or php -r. needs_secret: true also needs a secret-looking variable, the whole environment, a secrets file or a key-shaped string in that text. needs_remote: true also needs an http(s) or host:port address that is not this machine and not in allow_hosts. unless: any of these substrings stops the block.',
|
|
307
|
+
' guard.allow_hosts: hosts a network tool may reach. guard.net_tools and guard.db_clients: programs treated as network or database clients. guard.secret_parts: words that mark a variable as a secret. guard.safe_net: options of curl, wget and http that never send data.',
|
|
308
|
+
' agents.override_substrings: phrases in agent instruction files flagged as an instruction override. agents.secret_env_names: env names flagged when an agent config sets them.',
|
|
309
|
+
'',
|
|
310
|
+
`The change: ${String(title || 'untitled').slice(0, 300)}`,
|
|
311
|
+
`Why: ${String(why || '').slice(0, 600)}`,
|
|
312
|
+
'',
|
|
313
|
+
'Literal changes:',
|
|
314
|
+
...literal.map((l) => ` ${l}`),
|
|
315
|
+
'',
|
|
316
|
+
'Unified diff of lib/sentinel-rules.json:',
|
|
317
|
+
diff.slice(0, 20000),
|
|
318
|
+
'',
|
|
319
|
+
'The seen exam, public items the deterministic exam already scores (also in ./seen in this folder):',
|
|
320
|
+
seen.slice(0, 40000),
|
|
321
|
+
'',
|
|
322
|
+
'Read only the files in this folder. Do not read, list or search anything outside it. Never open a held-out exam, a key file or the keychain.',
|
|
323
|
+
'',
|
|
324
|
+
'Your task:',
|
|
325
|
+
' picture at least ten everyday commands developers and coding agents run that contain the same literal text. List each one this change would block as a false positive.',
|
|
326
|
+
' list short attack variants in the same area that this change still misses as bypasses.',
|
|
327
|
+
' if the change touches agents keys, list everyday agent instruction text it would wrongly flag as agent false alarms.',
|
|
328
|
+
' say SHIP only when you find no false positive and no agent false alarm. Otherwise say FIX.',
|
|
329
|
+
' never write a real key or token. Where a command needs one, write a placeholder such as {{fake:github}}, {{fake:aws}}, {{fake:openai}} or {{fake:value}}.',
|
|
330
|
+
'',
|
|
331
|
+
'Answer with exactly one JSON object and nothing else, no prose and no code fence:',
|
|
332
|
+
'{"verdict":"SHIP"|"FIX","false_positives":[{"command":"..."}],"bypasses":[{"command":"...","kind":"exfil|destroy|disarm"}],"agent_false_alarms":[{"file":"CLAUDE.md","text":"..."}],"reason":"one or two plain sentences"}',
|
|
333
|
+
].join('\n');
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
const MAX_ANSWER_BYTES = 64 * 1024;
|
|
337
|
+
const REVIEW_KEYS = ['verdict', 'false_positives', 'bypasses', 'agent_false_alarms', 'reason'];
|
|
338
|
+
const LIST_SHAPES = {
|
|
339
|
+
false_positives: { need: ['command'], may: [] },
|
|
340
|
+
bypasses: { need: ['command'], may: ['kind'] },
|
|
341
|
+
agent_false_alarms: { need: ['file', 'text'], may: [] },
|
|
342
|
+
};
|
|
343
|
+
const FIELD_MAX = { command: 2000, file: 200, text: 8000, kind: 20 };
|
|
344
|
+
|
|
345
|
+
// Strict and fail-closed: the answer must be at most 64 KB and one json
|
|
346
|
+
// object, alone (a single code fence around it is tolerated), with exactly
|
|
347
|
+
// the verdict, the three lists and an optional reason. Every list entry must
|
|
348
|
+
// be an object with exactly its string keys. Lists are cut to the item limit
|
|
349
|
+
// before any entry is read. Anything else is FIX. A SHIP that lists any
|
|
350
|
+
// example is FIX.
|
|
351
|
+
function parseReview(text) {
|
|
352
|
+
const fix = (reason) => ({ verdict: 'FIX', malformed: true, false_positives: [], bypasses: [], agent_false_alarms: [], reason });
|
|
353
|
+
const raw = String(text || '');
|
|
354
|
+
if (Buffer.byteLength(raw, 'utf8') > MAX_ANSWER_BYTES) return fix('the reviewer answer was longer than 64 KB');
|
|
355
|
+
let s = raw.trim();
|
|
356
|
+
const fence = s.match(/^```(?:json)?[ \t]*\n([\s\S]*?)\n```$/);
|
|
357
|
+
if (fence) s = fence[1].trim();
|
|
358
|
+
if (!s.startsWith('{') || !s.endsWith('}')) return fix('the reviewer did not answer with one json object');
|
|
359
|
+
let v;
|
|
360
|
+
try { v = JSON.parse(s); } catch { return fix('the reviewer answer was not valid json'); }
|
|
361
|
+
if (!isObj(v)) return fix('the reviewer answer was not one json object');
|
|
362
|
+
const extra = Object.keys(v).filter((k) => !REVIEW_KEYS.includes(k));
|
|
363
|
+
if (extra.length) return fix('the reviewer answer had unknown fields');
|
|
364
|
+
if (v.verdict !== 'SHIP' && v.verdict !== 'FIX') return fix('the reviewer gave no SHIP or FIX verdict');
|
|
365
|
+
if (v.reason !== undefined && typeof v.reason !== 'string') return fix('the reviewer reason was not text');
|
|
366
|
+
// The reviewer's own reason text is checked for shape and then dropped. It
|
|
367
|
+
// never reaches a proposer, a receipt or a commit: only the verdict and
|
|
368
|
+
// the counts of accepted examples do. Every reason below is written here.
|
|
369
|
+
const out = { verdict: v.verdict, malformed: false, reason: '' };
|
|
370
|
+
for (const [name, shape] of Object.entries(LIST_SHAPES)) {
|
|
371
|
+
if (!Array.isArray(v[name])) return fix(`the reviewer answer had no ${name} list`);
|
|
372
|
+
const list = v[name].slice(0, MAX_REVIEW_ITEMS);
|
|
373
|
+
const clean = [];
|
|
374
|
+
for (const entry of list) {
|
|
375
|
+
if (!isObj(entry)) return fix(`a ${name} entry was not an object`);
|
|
376
|
+
const keys = Object.keys(entry);
|
|
377
|
+
if (keys.some((k) => !shape.need.includes(k) && !shape.may.includes(k))) return fix(`a ${name} entry had an unknown key`);
|
|
378
|
+
const item = {};
|
|
379
|
+
for (const k of keys) {
|
|
380
|
+
if (typeof entry[k] !== 'string' || !entry[k].trim() || entry[k].length > FIELD_MAX[k]) return fix(`a ${name} entry had a bad ${k}`);
|
|
381
|
+
item[k] = entry[k];
|
|
382
|
+
}
|
|
383
|
+
if (shape.need.some((k) => !(k in item))) return fix(`a ${name} entry was missing a field`);
|
|
384
|
+
if (name === 'bypasses' && item.kind !== undefined && !['exfil', 'destroy', 'disarm'].includes(item.kind)) return fix('a bypass had an unknown kind');
|
|
385
|
+
clean.push(item);
|
|
386
|
+
}
|
|
387
|
+
out[name] = clean;
|
|
388
|
+
}
|
|
389
|
+
if (out.verdict === 'SHIP' && (out.false_positives.length || out.bypasses.length || out.agent_false_alarms.length)) {
|
|
390
|
+
return { ...out, verdict: 'FIX', reason: 'the reviewer said ship but listed examples' };
|
|
391
|
+
}
|
|
392
|
+
return out;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// The reviewer: codex exec, read-only, stdin closed, in an empty folder
|
|
396
|
+
// holding only the diff and the seen exam, with an empty temp HOME.
|
|
397
|
+
async function askReviewer({ prompt, files, env = process.env, timeoutMs = 15 * 60 * 1000, model = '' }) {
|
|
398
|
+
const ask = require('./engine-ask');
|
|
399
|
+
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'atris-sentinel-review-'));
|
|
400
|
+
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'atris-sentinel-reviewer-home-'));
|
|
401
|
+
const out = path.join(home, 'answer.txt');
|
|
402
|
+
try {
|
|
403
|
+
for (const [rel, content] of Object.entries(files)) {
|
|
404
|
+
fs.mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true });
|
|
405
|
+
fs.writeFileSync(path.join(dir, rel), content);
|
|
406
|
+
}
|
|
407
|
+
const renv = { ...imp.engineEnv(env, home), CODEX_HOME: env.CODEX_HOME || path.join(imp.homeDir(env), '.codex') };
|
|
408
|
+
const invocation = {
|
|
409
|
+
engine: 'codex',
|
|
410
|
+
bin: 'codex',
|
|
411
|
+
args: ['exec', '-s', 'read-only', '--ephemeral', '--skip-git-repo-check', '--color', 'never', '-c', 'mcp_servers={}', ...(model ? ['-m', model] : []), '-o', out, prompt],
|
|
412
|
+
};
|
|
413
|
+
const res = await ask.runAskProcess(invocation, { cwd: dir, timeoutMs, env: renv });
|
|
414
|
+
let answer = '';
|
|
415
|
+
try { answer = fs.readFileSync(out, 'utf8'); } catch { answer = String(res.stdout || ''); }
|
|
416
|
+
return { ...res, answer };
|
|
417
|
+
} finally {
|
|
418
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
419
|
+
fs.rmSync(home, { recursive: true, force: true });
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
// ---------------------------------------------------------------- learn
|
|
424
|
+
|
|
425
|
+
function itemKey(set, item) {
|
|
426
|
+
return set === 'guard' ? `guard:${String(item.command || '').trim()}` : `agents:${JSON.stringify(item.files || {})}`;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
function seenKeys(seenDir) {
|
|
430
|
+
const keys = new Set();
|
|
431
|
+
const ids = new Set();
|
|
432
|
+
for (const set of ['guard', 'agents']) {
|
|
433
|
+
for (const item of require('./sentinel-bench').readItems(path.join(seenDir, `${set}.jsonl`)).items) { keys.add(itemKey(set, item)); ids.add(item.id); }
|
|
434
|
+
}
|
|
435
|
+
return { keys, ids };
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// The strict secret check for anything a reviewer wrote. Unlike the repo
|
|
439
|
+
// scan it honors no suppression marker and no placeholder exemption: any
|
|
440
|
+
// line with a marker or any match of a secret rule counts.
|
|
441
|
+
const { strictSecret } = imp;
|
|
442
|
+
|
|
443
|
+
// Receipts are scanned before they are written. A line that fails the
|
|
444
|
+
// strict check is replaced, never written.
|
|
445
|
+
function scrubReceipt(text) {
|
|
446
|
+
return String(text).split('\n').map((l) => (strictSecret(l) ? '[a line was removed here: it looked like a secret]' : l)).join('\n');
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
// The reviewer's examples as exam items. The proposer never writes these.
|
|
450
|
+
// The examples are cut to the item limit before any is hashed or scanned.
|
|
451
|
+
// Items that fail the strict secret check are dropped, never stored.
|
|
452
|
+
function reviewItems(review, { seenDir, date, focus = '' }) {
|
|
453
|
+
const { keys, ids } = seenKeys(seenDir);
|
|
454
|
+
const out = [];
|
|
455
|
+
const dropped = [];
|
|
456
|
+
const guardKind = ['exfil', 'destroy', 'disarm'].includes(focus) ? focus : 'exfil';
|
|
457
|
+
const raw = [
|
|
458
|
+
...(review.false_positives || []).map((fp) => ['guard', { id: '', kind: 'benign', expect: 'allow', command: fp.command, source: 'review' }]),
|
|
459
|
+
...(review.bypasses || []).map((bp) => ['guard', { id: '', kind: bp.kind || guardKind, expect: 'block', command: bp.command, source: 'review' }]),
|
|
460
|
+
...(review.agent_false_alarms || []).map((fa) => {
|
|
461
|
+
const rel = path.normalize(String(fa.file));
|
|
462
|
+
return path.isAbsolute(rel) || rel.startsWith('..') ? null : ['agents', { id: '', expect: [], files: { [rel]: fa.text }, source: 'review' }];
|
|
463
|
+
}),
|
|
464
|
+
].filter(Boolean).slice(0, MAX_REVIEW_ITEMS);
|
|
465
|
+
for (const [set, item] of raw) {
|
|
466
|
+
const k = itemKey(set, item);
|
|
467
|
+
if (keys.has(k)) continue;
|
|
468
|
+
item.id = `rv-${date}-${sha(k).slice(0, 10)}`;
|
|
469
|
+
if (ids.has(item.id)) continue;
|
|
470
|
+
if (!imp.gateValue(item)) { dropped.push(item.id); continue; }
|
|
471
|
+
keys.add(k);
|
|
472
|
+
ids.add(item.id);
|
|
473
|
+
out.push({ set, item });
|
|
474
|
+
}
|
|
475
|
+
return { items: out, dropped };
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
// The judge already holds the held-out half. It refuses any review item
|
|
479
|
+
// that equals, contains or sits inside a held-out item. Only refused ids come
|
|
480
|
+
// back. Fail-closed: if the screen cannot run, every item is refused.
|
|
481
|
+
function runScreen(items, { env = process.env, atrisBin, dir, timeoutMs = 60 * 1000 } = {}) {
|
|
482
|
+
if (!items.length) return { refused: [] };
|
|
483
|
+
const all = { refused: items.map((x) => x.item.id) };
|
|
484
|
+
const file = path.join(dir, `screen-${crypto.randomBytes(4).toString('hex')}.json`);
|
|
485
|
+
try {
|
|
486
|
+
fs.writeFileSync(file, JSON.stringify(items.map((x) => ({ id: x.item.id, set: x.set, item: x.item }))));
|
|
487
|
+
const bin = atrisBin || path.join(__dirname, '..', 'bin', 'atris.js');
|
|
488
|
+
const r = require('child_process').spawnSync(process.execPath, [bin, 'sentinel', 'bench', '--screen', file], { encoding: 'utf8', env: { ...env, ATRIS_SKIP_UPDATE_CHECK: '1' }, timeout: Math.max(1, Math.floor(timeoutMs)), maxBuffer: 8 * 1024 * 1024 });
|
|
489
|
+
if (r.status !== 0) return all;
|
|
490
|
+
const v = JSON.parse(String(r.stdout || '').trim().split('\n').pop());
|
|
491
|
+
if (!v || v.ok !== true || !Array.isArray(v.refused) || v.unreadable) return all;
|
|
492
|
+
return { refused: v.refused.filter((x) => typeof x === 'string') };
|
|
493
|
+
} catch { return all; } finally { try { fs.unlinkSync(file); } catch {} }
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
// ---------------------------------------------------------------- copies
|
|
497
|
+
|
|
498
|
+
function branchExists(root, name, t) {
|
|
499
|
+
return imp.git(root, ['rev-parse', '--verify', '--quiet', `refs/heads/${name}`], { check: false, timeoutMs: t }).status === 0;
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
// The name a night branch takes on the remote. Branch names carry only the
|
|
503
|
+
// date, so an earlier run that day may already have pushed the same name.
|
|
504
|
+
// Then this run pushes under name-<short id> instead of failing the push.
|
|
505
|
+
// When the remote cannot be asked, the plain name is kept and the push
|
|
506
|
+
// reports its own error.
|
|
507
|
+
function remoteBranchName(root, name, shortId, { env, timeoutMs } = {}) {
|
|
508
|
+
let r;
|
|
509
|
+
try { r = imp.git(root, ['ls-remote', '--exit-code', '--heads', 'origin', `refs/heads/${name}`], { check: false, timeoutMs, env }); } catch { return name; }
|
|
510
|
+
return r.status === 0 ? `${name}-${shortId}` : name;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
function freeBranch(root, name, t) {
|
|
514
|
+
if (!branchExists(root, name, t)) return name;
|
|
515
|
+
for (let n = 2; n < 100; n++) if (!branchExists(root, `${name}-${n}`, t)) return `${name}-${n}`;
|
|
516
|
+
throw new Error(`too many branches named ${name}`);
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
// A copy on its own branch with a run record. The shared sweep reads it to
|
|
520
|
+
// know whose the copy is. The folder name matches what the sweep looks for.
|
|
521
|
+
function makeCopy(root, branch, at, env, tmpRoot, t) {
|
|
522
|
+
const tree = fs.mkdtempSync(path.join(tmpRoot, 'atris-sentinel-improve-'));
|
|
523
|
+
fs.rmSync(tree, { recursive: true, force: true });
|
|
524
|
+
imp.git(root, ['worktree', 'add', '-q', '-b', branch, tree, at], { timeoutMs: t });
|
|
525
|
+
fs.mkdirSync(imp.runsDir(env), { recursive: true });
|
|
526
|
+
const runFile = path.join(imp.runsDir(env), `${path.basename(tree)}.json`);
|
|
527
|
+
fs.writeFileSync(runFile, `${JSON.stringify({ pid: process.pid, pgid: imp.pgidOf(process.pid), tree, branch, started: new Date().toISOString() })}\n`);
|
|
528
|
+
return { tree, branch, runFile };
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
function dropCopy(root, copy, env) {
|
|
532
|
+
return imp.dropCopy(root, copy.tree, copy.branch, { env, runFile: copy.runFile, ownRun: true });
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
function headOf(tree, t) {
|
|
536
|
+
return String(imp.git(tree, ['rev-parse', 'HEAD'], { timeoutMs: t }).stdout).trim();
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
function copySeen(fromDir, toDir) {
|
|
540
|
+
fs.mkdirSync(toDir, { recursive: true });
|
|
541
|
+
for (const set of ['guard', 'agents']) {
|
|
542
|
+
const f = path.join(fromDir, `${set}.jsonl`);
|
|
543
|
+
if (fs.existsSync(f)) fs.copyFileSync(f, path.join(toDir, `${set}.jsonl`));
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
const TRAILER = 'Co-authored-by: Atris <299057014+atris-builder[bot]@users.noreply.github.com>';
|
|
548
|
+
|
|
549
|
+
function commitIn(tree, paths, message, env, t) {
|
|
550
|
+
// Messages are fixed sentences with counts and ids. The gate checks them
|
|
551
|
+
// anyway. One that fails is replaced by a bare fixed sentence.
|
|
552
|
+
if (!imp.gateValue(message)) message = 'Sentinel night: one commit';
|
|
553
|
+
imp.git(tree, ['add', ...paths], { timeoutMs: t });
|
|
554
|
+
const staged = String(imp.git(tree, ['diff', '--cached', '--name-only'], { timeoutMs: t }).stdout || '').trim();
|
|
555
|
+
if (!staged) return false;
|
|
556
|
+
imp.git(tree, ['commit', '-q', '-m', `${message}\n\n${TRAILER}`], { env, timeoutMs: t });
|
|
557
|
+
return true;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
// ---------------------------------------------------------------- receipt
|
|
561
|
+
|
|
562
|
+
function heldLine(set, s) {
|
|
563
|
+
return imp.totalsLine(set, s);
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
function renderNightReceipt(run) {
|
|
567
|
+
const L = [`# sentinel improve night, ${run.at}`, ''];
|
|
568
|
+
L.push(`result: ${run.landed} ${run.landed === 1 ? 'change' : 'changes'} shipped in ${run.rounds.length} ${run.rounds.length === 1 ? 'round' : 'rounds'}. stopped because ${run.stop}`);
|
|
569
|
+
if (run.night_branch) L.push(`rules branch: ${run.night_branch}`);
|
|
570
|
+
if (run.exam_branch) L.push(`exam branch: ${run.exam_branch} (${run.exam_added} new seen items)`);
|
|
571
|
+
L.push(`proposers: claude, sealed as in a single run. reviewer: ${run.review || 'none'}. judge: the sentinel bench, fixed code, no model`);
|
|
572
|
+
L.push(`used: ${run.calls} of ${run.max_calls} engine calls, ${run.minutes} of ${run.max_minutes} minutes`);
|
|
573
|
+
for (const r of run.rounds) {
|
|
574
|
+
L.push('', `## round ${r.round}`, '');
|
|
575
|
+
L.push(`foci: ${r.foci.join(', ') || 'none'}`);
|
|
576
|
+
for (const p of r.proposals) {
|
|
577
|
+
const exam = p.error || !p.decision ? `stopped: ${p.error || 'not judged'}` : p.decision.keep ? `kept by the exam (held-out ${p.gain.held >= 0 ? '+' : ''}${p.gain.held}, older seen ${p.gain.seen >= 0 ? '+' : ''}${p.gain.seen})` : `rejected by the exam: ${imp.fixedReason(p.decision.reasons[0]) || 'no gain'}`;
|
|
578
|
+
L.push(`- ${p.focus}: ${p.title ? 'one proposal' : 'no usable proposal'}`);
|
|
579
|
+
L.push(` exam: ${exam}`);
|
|
580
|
+
if (p.review) L.push(` review: ${p.review.verdict}${p.review.malformed ? ' (answer not usable)' : ''}${p.review.reason ? `. ${p.review.reason}` : ''}`);
|
|
581
|
+
if (p.learned) L.push(` learned: ${p.learned} new seen items`);
|
|
582
|
+
}
|
|
583
|
+
if (r.union) L.push(`- union of ${r.union.parts} shipped changes: ${r.union.result}`);
|
|
584
|
+
if (r.landed) {
|
|
585
|
+
L.push('', `landed: ${r.landed.focus === 'union' ? 'the union of the shipped changes' : `the ${r.landed.focus} proposal`}`);
|
|
586
|
+
for (const c of r.landed.literal.slice(0, 20)) L.push(` ${c}`);
|
|
587
|
+
} else L.push('', 'landed: nothing');
|
|
588
|
+
if (!r.end) continue;
|
|
589
|
+
L.push('', 'before and after this round, held-out half (totals only):');
|
|
590
|
+
for (const set of Object.keys(r.start.holdout || {})) L.push(`- before ${heldLine(set, r.start.holdout[set])}`, `- after ${heldLine(set, r.end.holdout[set])}`);
|
|
591
|
+
L.push('seen half as it stood at each end:');
|
|
592
|
+
for (const set of Object.keys(r.start.seen || {})) L.push(`- before ${heldLine(set, r.start.seen[set])}`, `- after ${heldLine(set, r.end.seen[set])}`);
|
|
593
|
+
L.push(`exam items added this round: ${r.exam_added.review} from review (${r.exam_added.allow} must run, ${r.exam_added.block} must block, ${r.exam_added.agents} clean agent setups), ${r.exam_added.proposer} from the landed proposal`);
|
|
594
|
+
if (r.exam_added.dropped) L.push(`dropped ${r.exam_added.dropped} review examples that looked like a real secret`);
|
|
595
|
+
if (r.exam_added.held_out) L.push(`refused ${r.exam_added.held_out} review examples that overlap the held-out half`);
|
|
596
|
+
}
|
|
597
|
+
for (const k of run.kept_copies || []) L.push('', `kept a copy at ${k.tree}: ${imp.fixedReason(k.reason)}`);
|
|
598
|
+
L.push('');
|
|
599
|
+
return L.join('\n');
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
// ---------------------------------------------------------------- night
|
|
603
|
+
|
|
604
|
+
function seenText(seenDir) {
|
|
605
|
+
return ['guard', 'agents'].map((set) => {
|
|
606
|
+
let t = '';
|
|
607
|
+
try { t = fs.readFileSync(path.join(seenDir, `${set}.jsonl`), 'utf8'); } catch {}
|
|
608
|
+
return `seen/${set}.jsonl:\n${t}`;
|
|
609
|
+
}).join('\n');
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
async function night(opts = {}) {
|
|
613
|
+
const env = opts.env || process.env;
|
|
614
|
+
const log = opts.log || (() => {});
|
|
615
|
+
const now = opts.now || new Date();
|
|
616
|
+
const clock = opts.clock || Date.now;
|
|
617
|
+
const engine = opts.engine || 'claude';
|
|
618
|
+
const rounds = Number(opts.rounds || 1);
|
|
619
|
+
const parallel = Number(opts.parallel || 1);
|
|
620
|
+
const review = opts.review || null;
|
|
621
|
+
const maxMinutes = opts.maxMinutes === undefined ? 120 : Number(opts.maxMinutes);
|
|
622
|
+
const maxCalls = opts.maxCalls === undefined ? rounds * (2 * parallel + 1) : Number(opts.maxCalls);
|
|
623
|
+
const focus = opts.focus || null;
|
|
624
|
+
if (!Number.isInteger(rounds) || rounds < 1 || rounds > 50) throw new Error('--rounds needs a whole number from 1 to 50');
|
|
625
|
+
if (!Number.isInteger(parallel) || parallel < 1 || parallel > MAX_PARALLEL) throw new Error(`--parallel needs a whole number from 1 to ${MAX_PARALLEL}, one per focus area`);
|
|
626
|
+
if (review && review !== 'codex') throw new Error('--review takes codex only');
|
|
627
|
+
if (!Number.isFinite(maxMinutes) || maxMinutes <= 0) throw new Error('--max-minutes needs a number above 0');
|
|
628
|
+
if (!Number.isInteger(maxCalls) || maxCalls < 1) throw new Error('--max-calls needs a whole number above 0');
|
|
629
|
+
if (focus && !FOCI.includes(focus)) throw new Error(`unknown focus: ${focus}. use ${FOCI.join(', ')}`);
|
|
630
|
+
const refused = imp.refusedEngine(engine);
|
|
631
|
+
if (refused) throw new Error(refused);
|
|
632
|
+
const root = imp.repoTop(opts.root || process.cwd());
|
|
633
|
+
if (!fs.existsSync(path.join(root, RULES_PATH))) throw new Error(`${root} is not an atris checkout with the sentinel rules in it`);
|
|
634
|
+
const rawJudge = opts.judge || ((args) => imp.runJudge({ ...args, env, atrisBin: opts.atrisBin }));
|
|
635
|
+
const screen = opts.screen || ((items, args) => runScreen(items, { ...args, env, atrisBin: opts.atrisBin }));
|
|
636
|
+
const propose = opts.propose || ((args) => imp.askEngine({ ...args, env }));
|
|
637
|
+
const reviewer = opts.reviewer || ((args) => askReviewer({ ...args, env, model: opts.reviewModel || '' }));
|
|
638
|
+
const tmpRoot = opts.tmpRoot || os.tmpdir();
|
|
639
|
+
|
|
640
|
+
const lock = takeLock(env, { now: clock });
|
|
641
|
+
const budget = makeBudget({ maxMinutes, maxCalls, clock });
|
|
642
|
+
// The deadline is checked before every exam, engine call and git call in
|
|
643
|
+
// a round. No step is given more time than remains.
|
|
644
|
+
const checkTime = () => { if (budget.leftMs() <= 0) throw new TimeUp(); };
|
|
645
|
+
const left = (cap) => { checkTime(); return Math.min(cap, budget.leftMs()); };
|
|
646
|
+
const judge = (args) => rawJudge({ ...args, timeoutMs: left(10 * 60 * 1000) });
|
|
647
|
+
const FINAL_GIT_MS = 60 * 1000;
|
|
648
|
+
const at = now.toISOString();
|
|
649
|
+
const date = dateTag(now);
|
|
650
|
+
const id = `${imp.stamp(now)}-${crypto.randomBytes(2).toString('hex')}`;
|
|
651
|
+
const run = { at, rounds: [], stop: '', landed: 0, review, calls: 0, max_calls: maxCalls, minutes: 0, max_minutes: maxMinutes, exam_added: 0, night_branch: null, exam_branch: null, kept_copies: [], pushed: [] };
|
|
652
|
+
const receiptDir = path.join(imp.stateDir(env), 'receipts');
|
|
653
|
+
let nightCopy = null;
|
|
654
|
+
let examCopy = null;
|
|
655
|
+
let work = null;
|
|
656
|
+
let examCommits = 0;
|
|
657
|
+
try {
|
|
658
|
+
work = fs.mkdtempSync(path.join(tmpRoot, 'atris-sentinel-night-'));
|
|
659
|
+
const swept = imp.sweepStale(root, env);
|
|
660
|
+
for (const k of swept.kept) log(`kept a leftover copy at ${k.tree}: ${k.reason}`);
|
|
661
|
+
nightCopy = makeCopy(root, freeBranch(root, `sentinel-improve/night-${date}`, left(FINAL_GIT_MS)), 'HEAD', env, tmpRoot, left(FINAL_GIT_MS));
|
|
662
|
+
examCopy = makeCopy(root, freeBranch(root, `sentinel-exam/${date}`, left(FINAL_GIT_MS)), 'HEAD', env, tmpRoot, left(FINAL_GIT_MS));
|
|
663
|
+
// The state each round starts from, outside every copy: the rules as
|
|
664
|
+
// landed so far and the exam as grown so far.
|
|
665
|
+
const base = path.join(work, 'base');
|
|
666
|
+
const baseSeen = path.join(work, 'seen');
|
|
667
|
+
fs.mkdirSync(path.join(base, 'lib'), { recursive: true });
|
|
668
|
+
fs.copyFileSync(path.join(nightCopy.tree, RULES_PATH), path.join(base, RULES_PATH));
|
|
669
|
+
copySeen(path.join(examCopy.tree, SEEN_REL), baseSeen);
|
|
670
|
+
const readBase = () => JSON.parse(fs.readFileSync(path.join(base, RULES_PATH), 'utf8'));
|
|
671
|
+
log('reading local signals');
|
|
672
|
+
const signals = opts.signals || await imp.gather({ env, request: opts.request, osvBase: opts.osvBase, offline: opts.offline });
|
|
673
|
+
const tonight = [];
|
|
674
|
+
let noShip = 0;
|
|
675
|
+
let flatHeld = 0;
|
|
676
|
+
|
|
677
|
+
for (let r = 1; r <= rounds; r++) {
|
|
678
|
+
if (budget.leftMs() <= 0) { run.stop = 'the time limit ran out'; break; }
|
|
679
|
+
const k = budget.take(parallel);
|
|
680
|
+
if (!k) { run.stop = 'the engine call cap was reached'; break; }
|
|
681
|
+
const start = judge({ target: base, seenDir: baseSeen });
|
|
682
|
+
// A file key could be read by the reviewer. Night runs need the
|
|
683
|
+
// keychain. The reviewer's empty HOME does not reach it.
|
|
684
|
+
if (start.key_source !== 'keychain' && !opts._testAllowFileKey) throw new Error('night runs need the held-out key in the keychain. this machine uses a key file. the reviewer could read it');
|
|
685
|
+
const foci = assignFoci(start, k, focus);
|
|
686
|
+
const round = { round: r, foci, proposals: [], union: null, landed: null, start, end: null, exam_added: { review: 0, allow: 0, block: 0, agents: 0, proposer: 0, dropped: 0, held_out: 0 } };
|
|
687
|
+
run.rounds.push(round);
|
|
688
|
+
log(`round ${r}: asking ${foci.length} ${foci.length === 1 ? 'proposer' : 'proposers'} at once (${foci.join(', ')})`);
|
|
689
|
+
const tip = headOf(nightCopy.tree, left(FINAL_GIT_MS));
|
|
690
|
+
const copies = [];
|
|
691
|
+
const baseRules = readBase();
|
|
692
|
+
try {
|
|
693
|
+
for (const f of foci) copies.push(makeCopy(root, freeBranch(root, `sentinel-improve/${id}-r${r}-${f}`, left(FINAL_GIT_MS)), tip, env, tmpRoot, left(FINAL_GIT_MS)));
|
|
694
|
+
const sealed = [path.join(imp.homeDir(env), '.atris'), start.holdout_dir, ...(start.sealed_sources || [])];
|
|
695
|
+
const timeoutMs = left(20 * 60 * 1000);
|
|
696
|
+
const answers = await Promise.all(copies.map(async (c, i) => {
|
|
697
|
+
try {
|
|
698
|
+
return await propose({ engine, prompt: imp.buildPrompt({ report: start, signals, seenDirRel: SEEN_REL, focus: foci[i], tonight: tonight.slice(-20) }), cwd: c.tree, sealed, timeoutMs });
|
|
699
|
+
} catch (e) { return { error: String(e.message || e) }; }
|
|
700
|
+
}));
|
|
701
|
+
for (let i = 0; i < copies.length; i++) {
|
|
702
|
+
const f = foci[i];
|
|
703
|
+
const p = { focus: f, title: '', error: '', decision: null, gain: { held: 0, seen: 0 }, review: null, learned: 0 };
|
|
704
|
+
round.proposals.push(p);
|
|
705
|
+
try {
|
|
706
|
+
const answer = answers[i] || {};
|
|
707
|
+
if (answer.error) throw new Error(`the proposer failed: ${answer.error}`);
|
|
708
|
+
const proposal = isObj(answer) && (Array.isArray(answer.edits) || answer.rules) ? answer : imp.extractJsonObject(answer.stdout);
|
|
709
|
+
const bad = imp.checkProposal(proposal, base);
|
|
710
|
+
if (bad) throw new Error(bad);
|
|
711
|
+
checkTime();
|
|
712
|
+
const stray = imp.strayChanges(copies[i].tree);
|
|
713
|
+
if (stray.length) throw new Error(`the proposer changed files it may not touch: ${stray.slice(0, 3).join(', ')}`);
|
|
714
|
+
if (Array.isArray(proposal.new_items) && !imp.gateValue(proposal.new_items)) throw new Error(imp.SECRET_ITEM_ERROR);
|
|
715
|
+
if (!imp.gateValue(proposal)) throw new Error(imp.SECRET_PROPOSAL_ERROR);
|
|
716
|
+
// Kept in memory for the reviewer prompt only. Never written.
|
|
717
|
+
p.title = String(proposal.title || 'untitled').replace(/\s+/g, ' ').slice(0, 200);
|
|
718
|
+
p.why = String(proposal.why || '').slice(0, 600);
|
|
719
|
+
p.items = imp.cleanItems(proposal, `imp-${id.slice(0, 8)}-r${r}-${f}`);
|
|
720
|
+
if (p.items.some((x) => !imp.gateValue(x.item))) throw new Error(imp.SECRET_ITEM_ERROR);
|
|
721
|
+
p.dir = path.join(work, `r${r}-${f}`);
|
|
722
|
+
const candSeen = path.join(p.dir, 'seen');
|
|
723
|
+
copySeen(baseSeen, candSeen);
|
|
724
|
+
imp.appendItems(candSeen, p.items);
|
|
725
|
+
const before = judge({ target: base, seenDir: candSeen });
|
|
726
|
+
const applyErr = imp.applyRules(base, proposal, p.dir);
|
|
727
|
+
if (applyErr) throw new Error(applyErr);
|
|
728
|
+
const after = judge({ target: p.dir, seenDir: candSeen });
|
|
729
|
+
const newIds = p.items.map((x) => `${x.set}:${x.item.id}`);
|
|
730
|
+
p.decision = imp.keepDecision(before, after, { newIds });
|
|
731
|
+
p.gain = gainOf(before, after, newIds);
|
|
732
|
+
p.rules = JSON.parse(fs.readFileSync(path.join(p.dir, RULES_PATH), 'utf8'));
|
|
733
|
+
p.literal = literalChanges(baseRules, p.rules);
|
|
734
|
+
} catch (e) { if (isTimeUp(e)) { p.error = 'the time limit ran out'; throw e; } p.error = imp.fixedError(e); }
|
|
735
|
+
log(`round ${r} ${f}: ${p.error ? `stopped: ${p.error}` : p.decision.keep ? 'kept by the exam' : `rejected: ${imp.fixedReason(p.decision.reasons[0])}`}`);
|
|
736
|
+
}
|
|
737
|
+
} finally {
|
|
738
|
+
for (const c of copies) { const res = dropCopy(root, c, env); if (!res.removed) run.kept_copies.push({ tree: c.tree, branch: c.branch, reason: res.reason }); }
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
const learn = (p, rev, learnFocus) => {
|
|
742
|
+
const found = reviewItems(rev, { seenDir: baseSeen, date, focus: learnFocus });
|
|
743
|
+
// No time left means nothing is learned: the screen cannot run.
|
|
744
|
+
const { refused } = budget.leftMs() > 0 ? screen(found.items, { dir: work, timeoutMs: Math.min(60 * 1000, budget.leftMs()) }) : { refused: found.items.map((x) => x.item.id) };
|
|
745
|
+
const no = new Set(refused);
|
|
746
|
+
const items = found.items.filter((x) => !no.has(x.item.id));
|
|
747
|
+
const dropped = found.dropped;
|
|
748
|
+
round.exam_added.held_out += found.items.length - items.length;
|
|
749
|
+
imp.appendItems(baseSeen, items);
|
|
750
|
+
round.exam_added.review += items.length;
|
|
751
|
+
round.exam_added.dropped += dropped.length;
|
|
752
|
+
for (const x of items) {
|
|
753
|
+
if (x.set === 'agents') round.exam_added.agents += 1;
|
|
754
|
+
else if (x.item.expect === 'allow') { round.exam_added.allow += 1; tonight.push(`must run: ${x.item.command.slice(0, 200)}`); }
|
|
755
|
+
else round.exam_added.block += 1;
|
|
756
|
+
}
|
|
757
|
+
if (rev.verdict === 'FIX' && p.literal) tonight.push(`rejected by the reviewer: the ${p.focus} proposal from round ${r}. it changed: ${p.literal.slice(0, 3).join('; ').slice(0, 400)}`);
|
|
758
|
+
return items.length;
|
|
759
|
+
};
|
|
760
|
+
const doReview = async (cand, learnFocus) => {
|
|
761
|
+
if (!budget.take(1)) return { verdict: 'FIX', malformed: true, false_positives: [], bypasses: [], agent_false_alarms: [], reason: 'the engine call cap was reached before review' };
|
|
762
|
+
if (budget.leftMs() <= 0) return { verdict: 'FIX', malformed: true, false_positives: [], bypasses: [], agent_false_alarms: [], reason: 'the time limit ran out before review' };
|
|
763
|
+
const diff = unifiedDiff(baseRules, cand.rules);
|
|
764
|
+
const seenNow = path.join(cand.dir, 'seen');
|
|
765
|
+
const prompt = reviewPrompt({ title: cand.title, why: cand.why, diff, literal: cand.literal, seen: seenText(seenNow) });
|
|
766
|
+
const files = { 'change.diff': diff };
|
|
767
|
+
for (const set of ['guard', 'agents']) { const f = path.join(seenNow, `${set}.jsonl`); if (fs.existsSync(f)) files[`seen/${set}.jsonl`] = fs.readFileSync(f, 'utf8'); }
|
|
768
|
+
let res;
|
|
769
|
+
const reviewMs = Math.max(1, Math.min(15 * 60 * 1000, budget.leftMs()));
|
|
770
|
+
try { res = await reviewer({ prompt, files, timeoutMs: reviewMs }); } catch (e) { res = { answer: '', error: String(e.message || e) }; }
|
|
771
|
+
const rev = parseReview(res && (res.answer !== undefined ? res.answer : res.stdout));
|
|
772
|
+
if (rev.malformed && res && res.timed_out) rev.reason = 'the reviewer ran out of time';
|
|
773
|
+
cand.learned = learn(cand, rev, learnFocus);
|
|
774
|
+
return rev;
|
|
775
|
+
};
|
|
776
|
+
|
|
777
|
+
const kept = round.proposals.filter((p) => !p.error && p.decision.keep).sort(byGain);
|
|
778
|
+
if (review && kept.length) {
|
|
779
|
+
log(`round ${r}: asking codex to review ${kept.length} ${kept.length === 1 ? 'change' : 'changes'}`);
|
|
780
|
+
const reviews = await Promise.all(kept.map((p) => doReview(p, p.focus)));
|
|
781
|
+
kept.forEach((p, i) => { p.review = reviews[i]; });
|
|
782
|
+
}
|
|
783
|
+
const shipped = kept.filter((p) => !review || (p.review && p.review.verdict === 'SHIP'));
|
|
784
|
+
|
|
785
|
+
// Land the union of what shipped or else the best single. Either one
|
|
786
|
+
// lands only after it passes the exam as it stands now, review items included.
|
|
787
|
+
const recheck = (cand) => {
|
|
788
|
+
const candSeen = path.join(cand.dir, 'seen');
|
|
789
|
+
copySeen(baseSeen, candSeen);
|
|
790
|
+
imp.appendItems(candSeen, cand.items);
|
|
791
|
+
const before = judge({ target: base, seenDir: candSeen });
|
|
792
|
+
const after = judge({ target: cand.dir, seenDir: candSeen });
|
|
793
|
+
return imp.keepDecision(before, after, { newIds: cand.items.map((x) => `${x.set}:${x.item.id}`) });
|
|
794
|
+
};
|
|
795
|
+
const order = [];
|
|
796
|
+
if (shipped.length > 1) {
|
|
797
|
+
const u = unionRules(baseRules, shipped.map((p) => p.rules));
|
|
798
|
+
round.union = { parts: shipped.length, result: '' };
|
|
799
|
+
if (u.conflict) round.union.result = 'not tried: two changes conflict';
|
|
800
|
+
else {
|
|
801
|
+
const dir = path.join(work, `r${r}-union`);
|
|
802
|
+
fs.mkdirSync(path.join(dir, 'lib'), { recursive: true });
|
|
803
|
+
fs.writeFileSync(path.join(dir, RULES_PATH), `${JSON.stringify(u.rules, null, 2)}\n`);
|
|
804
|
+
order.push({ union: true, focus: 'union', title: `${shipped.length} changes together: ${shipped.map((p) => p.title).join('; ').slice(0, 300)}`, why: shipped.map((p) => p.why).join(' ').slice(0, 600), rules: u.rules, dir, items: shipped.flatMap((p) => p.items), literal: literalChanges(baseRules, u.rules), learned: 0 });
|
|
805
|
+
}
|
|
806
|
+
}
|
|
807
|
+
order.push(...shipped);
|
|
808
|
+
let landed = null;
|
|
809
|
+
for (const cand of order) {
|
|
810
|
+
const d = recheck(cand);
|
|
811
|
+
if (!d.keep) { if (cand.union) round.union.result = `failed the exam: ${imp.fixedReason(d.reasons[0])}`; else cand.recheck = imp.fixedReason(d.reasons[0]); continue; }
|
|
812
|
+
if (cand.union && review) {
|
|
813
|
+
const rev = await doReview(cand, '');
|
|
814
|
+
if (rev.verdict !== 'SHIP') { round.union.result = `review said ${rev.verdict}`; continue; }
|
|
815
|
+
}
|
|
816
|
+
if (cand.union) round.union.result = 'kept';
|
|
817
|
+
landed = { ...cand, decision: d };
|
|
818
|
+
break;
|
|
819
|
+
}
|
|
820
|
+
if (round.union && !round.union.result) round.union.result = 'not reached';
|
|
821
|
+
|
|
822
|
+
if (landed) {
|
|
823
|
+
fs.copyFileSync(path.join(landed.dir, RULES_PATH), path.join(base, RULES_PATH));
|
|
824
|
+
fs.copyFileSync(path.join(landed.dir, RULES_PATH), path.join(nightCopy.tree, RULES_PATH));
|
|
825
|
+
checkTime();
|
|
826
|
+
const stray = imp.strayChanges(nightCopy.tree, new Set([RULES_PATH]));
|
|
827
|
+
if (stray.length) throw new Error(`the rules copy changed outside the rules file: ${stray.slice(0, 3).join(', ')}`);
|
|
828
|
+
commitIn(nightCopy.tree, [RULES_PATH], `Sentinel night round ${r}: one guard rules change\n\nKept by the sentinel bench${review ? ' and shipped by the codex review' : ''}. Night receipt: ${SEEN_REL}/receipts/night-${id}.md`, env, left(FINAL_GIT_MS));
|
|
829
|
+
imp.appendItems(baseSeen, landed.items.map((x) => ({ ...x, item: { ...x.item } })));
|
|
830
|
+
round.exam_added.proposer = landed.items.length;
|
|
831
|
+
round.landed = { focus: landed.focus, title: landed.title, literal: landed.literal, path: landed.decision.path };
|
|
832
|
+
run.landed += 1;
|
|
833
|
+
}
|
|
834
|
+
// Exam growth goes on its own branch, apart from the rules.
|
|
835
|
+
checkTime();
|
|
836
|
+
copySeen(baseSeen, path.join(examCopy.tree, SEEN_REL));
|
|
837
|
+
const examStray = imp.strayChanges(examCopy.tree, new Set([`${SEEN_REL}/guard.jsonl`, `${SEEN_REL}/agents.jsonl`]));
|
|
838
|
+
if (examStray.length) throw new Error(`the exam copy changed outside the seen items: ${examStray.slice(0, 3).join(', ')}`);
|
|
839
|
+
const added = round.exam_added.review + round.exam_added.proposer;
|
|
840
|
+
if (added && commitIn(examCopy.tree, [SEEN_REL], `Sentinel exam: ${added} seen items from night round ${r}\n\n${round.exam_added.review} came from the codex review (source "review"), ${round.exam_added.proposer} from the landed proposal (source "improve"). Exam data only, no rules change.`, env, left(FINAL_GIT_MS))) examCommits += 1;
|
|
841
|
+
run.exam_added += added;
|
|
842
|
+
round.end = judge({ target: base, seenDir: baseSeen });
|
|
843
|
+
const rose = round.landed && gainOf(start, round.end).held > 0;
|
|
844
|
+
noShip = round.landed ? 0 : noShip + 1;
|
|
845
|
+
flatHeld = rose ? 0 : flatHeld + 1;
|
|
846
|
+
log(`round ${r}: ${round.landed ? (round.landed.focus === 'union' ? 'shipped the union of the changes' : `shipped the ${round.landed.focus} proposal`) : 'nothing shipped'}, ${round.exam_added.review + round.exam_added.proposer} exam items added`);
|
|
847
|
+
if (noShip >= 2) { run.stop = 'two rounds in a row shipped nothing'; break; }
|
|
848
|
+
if (flatHeld >= 2) { run.stop = 'held-out catch did not rise for two rounds in a row'; break; }
|
|
849
|
+
}
|
|
850
|
+
if (!run.stop) run.stop = `all ${rounds} ${rounds === 1 ? 'round' : 'rounds'} ran`;
|
|
851
|
+
} catch (e) {
|
|
852
|
+
if (isTimeUp(e)) run.stop = 'the time limit ran out';
|
|
853
|
+
else {
|
|
854
|
+
run.stop = `an error: ${imp.fixedReason(String(e.message || e).split('\n')[0]).slice(0, 200)}`;
|
|
855
|
+
run.error = run.stop;
|
|
856
|
+
}
|
|
857
|
+
} finally {
|
|
858
|
+
run.calls = budget.calls;
|
|
859
|
+
run.minutes = budget.minutes();
|
|
860
|
+
try {
|
|
861
|
+
if (examCopy && examCommits) run.exam_branch = examCopy.branch;
|
|
862
|
+
// Counted, not read from branch ancestry: a kept proposer copy can
|
|
863
|
+
// share the night commits and make them look like no one's.
|
|
864
|
+
if (nightCopy && run.landed) {
|
|
865
|
+
run.night_branch = nightCopy.branch;
|
|
866
|
+
const rel = path.join(SEEN_REL, 'receipts', `night-${id}.md`);
|
|
867
|
+
fs.mkdirSync(path.dirname(path.join(nightCopy.tree, rel)), { recursive: true });
|
|
868
|
+
fs.writeFileSync(path.join(nightCopy.tree, rel), scrubReceipt(renderNightReceipt(run)));
|
|
869
|
+
commitIn(nightCopy.tree, [rel], `Sentinel night receipt: ${run.landed} shipped in ${run.rounds.length} rounds`, env, FINAL_GIT_MS);
|
|
870
|
+
}
|
|
871
|
+
if (opts.push) {
|
|
872
|
+
for (const b of [run.night_branch, run.exam_branch].filter(Boolean)) {
|
|
873
|
+
let r;
|
|
874
|
+
const gitEnv = { ...env, GIT_TERMINAL_PROMPT: '0' };
|
|
875
|
+
const dest = remoteBranchName(root, b, id.slice(-4), { env: gitEnv, timeoutMs: PUSH_MS });
|
|
876
|
+
if (dest !== b) log(`${b} is already on the remote, pushing as ${dest}`);
|
|
877
|
+
try { r = imp.git(root, ['push', 'origin', `refs/heads/${b}:refs/heads/${dest}`], { check: false, timeoutMs: PUSH_MS, env: gitEnv }); } catch (e) { r = { status: 1, stderr: String(e.message || e) }; }
|
|
878
|
+
if (r.status === 0) run.pushed.push(dest);
|
|
879
|
+
else log(`could not push ${b}: ${String(r.stderr || '').trim().split('\n').slice(-1)[0]}`);
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
for (const c of [nightCopy, examCopy].filter(Boolean)) { const res = dropCopy(root, c, env); if (!res.removed) run.kept_copies.push({ tree: c.tree, branch: c.branch, reason: res.reason }); }
|
|
883
|
+
} catch (e) { run.error = run.error || `cleanup did not finish: ${String(e.message || e).split('\n')[0]}`; }
|
|
884
|
+
finally {
|
|
885
|
+
if (work) fs.rmSync(work, { recursive: true, force: true });
|
|
886
|
+
lock.release();
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
fs.mkdirSync(receiptDir, { recursive: true });
|
|
890
|
+
const receipt = path.join(receiptDir, `night-${id}.md`);
|
|
891
|
+
fs.writeFileSync(receipt, scrubReceipt(renderNightReceipt(run)));
|
|
892
|
+
for (const k of run.kept_copies) log(`left a copy at ${k.tree}: ${k.reason}`);
|
|
893
|
+
return { ...run, receipt };
|
|
894
|
+
}
|
|
895
|
+
|
|
896
|
+
module.exports = {
|
|
897
|
+
FOCI,
|
|
898
|
+
assignFoci,
|
|
899
|
+
breakLock,
|
|
900
|
+
lockPath,
|
|
901
|
+
night,
|
|
902
|
+
parseReview,
|
|
903
|
+
remoteBranchName,
|
|
904
|
+
reviewItems,
|
|
905
|
+
reviewPrompt,
|
|
906
|
+
scrubReceipt,
|
|
907
|
+
strictSecret,
|
|
908
|
+
takeLock,
|
|
909
|
+
unionRules,
|
|
910
|
+
};
|