atris 3.82.8 → 3.82.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/commands/sentinel-bench.js +83 -5
- package/lib/sentinel-bench.js +50 -2
- package/lib/sentinel-guard.js +17 -4
- package/lib/sentinel-improve.js +128 -17
- package/lib/sentinel-night.js +893 -0
- package/package.json +1 -1
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
//
|
|
5
5
|
// atris sentinel bench [--summary] [--json] [--review] [--sets guard,agents]
|
|
6
6
|
// atris sentinel bench --split <file> --set guard|agents|review
|
|
7
|
-
// atris sentinel improve [--engine claude] [--offline] [--json]
|
|
7
|
+
// atris sentinel improve [--engine claude] [--focus exfil|destroy|disarm|agents] [--offline] [--json]
|
|
8
|
+
// atris sentinel improve --rounds N --parallel K --review codex [--max-minutes 120] [--max-calls N] [--push]
|
|
8
9
|
// atris sentinel improve --schedule [--every 24] [--engine claude] | --unschedule
|
|
9
10
|
|
|
10
11
|
const path = require('path');
|
|
@@ -51,12 +52,21 @@ function benchHelp() {
|
|
|
51
52
|
|
|
52
53
|
async function benchCommand(argv, deps = {}) {
|
|
53
54
|
if (argv.includes('--help') || argv.includes('-h') || argv[0] === 'help') return benchHelp();
|
|
54
|
-
try { checkFlags(argv, ['--summary', '--json', '--review'], ['--split', '--set', '--sets', '--target', '--seen', '--hunter', '--verifier', '--budget']); } catch (e) { console.error(` ${e.message}`); return 2; }
|
|
55
|
+
try { checkFlags(argv, ['--summary', '--json', '--review'], ['--split', '--set', '--sets', '--target', '--seen', '--hunter', '--verifier', '--budget', '--screen']); } catch (e) { console.error(` ${e.message}`); return 2; }
|
|
55
56
|
const bench = require('../lib/sentinel-bench');
|
|
56
57
|
const env = deps.env || process.env;
|
|
57
58
|
const root = path.resolve(readFlag(argv, '--target') || deps.root || process.cwd());
|
|
58
59
|
const seenDir = path.resolve(readFlag(argv, '--seen') || bench.seenDirOf(root));
|
|
59
60
|
const holdoutDir = bench.defaultHoldoutDir(env);
|
|
61
|
+
const screen = readFlag(argv, '--screen');
|
|
62
|
+
if (screen) {
|
|
63
|
+
// Internal to the night run: refused ids only, as json.
|
|
64
|
+
let candidates;
|
|
65
|
+
try { candidates = JSON.parse(require('fs').readFileSync(path.resolve(screen), 'utf8')); } catch (e) { console.error(` could not read the items to screen: ${e.message}`); return 1; }
|
|
66
|
+
if (!Array.isArray(candidates)) { console.error(' the items to screen must be a json list'); return 1; }
|
|
67
|
+
console.log(JSON.stringify({ ok: true, ...bench.screenItems(candidates, { holdoutDir, env }) }));
|
|
68
|
+
return 0;
|
|
69
|
+
}
|
|
60
70
|
const split = readFlag(argv, '--split');
|
|
61
71
|
if (split) {
|
|
62
72
|
const set = readFlag(argv, '--set');
|
|
@@ -97,26 +107,49 @@ function improveHelp() {
|
|
|
97
107
|
atris sentinel improve, one exam-checked change to the sentinel
|
|
98
108
|
|
|
99
109
|
atris sentinel improve [--engine claude] propose one change and judge it
|
|
110
|
+
atris sentinel improve --focus exfil point the proposer at one area:
|
|
111
|
+
exfil, destroy, disarm or agents
|
|
100
112
|
atris sentinel improve --offline skip the published advisory lookup
|
|
101
113
|
atris sentinel improve --schedule [--every 24] [--engine claude]
|
|
102
114
|
run it on a timer, nightly by default
|
|
103
115
|
atris sentinel improve --unschedule turn the timer off
|
|
104
116
|
|
|
117
|
+
a night run, many changes in one sitting:
|
|
118
|
+
|
|
119
|
+
atris sentinel improve --rounds 4 --parallel 3 --review codex
|
|
120
|
+
--rounds N up to N rounds. each starts from what shipped before
|
|
121
|
+
--parallel K K proposers at once, each on its own area (up to 4)
|
|
122
|
+
--review codex codex reviews every change the exam keeps. SHIP keeps
|
|
123
|
+
it. FIX rejects it and its examples join the exam
|
|
124
|
+
--max-minutes M stop after M minutes (120 by default)
|
|
125
|
+
--max-calls C stop after C proposer and reviewer calls
|
|
126
|
+
--push push the two night branches by name when done
|
|
127
|
+
atris sentinel improve --break-lock clear a night run lock whose run is gone
|
|
128
|
+
|
|
129
|
+
shipped rules land on sentinel-improve/night-<date>. new exam items land
|
|
130
|
+
apart on sentinel-exam/<date>. you can review them on their own. the
|
|
131
|
+
run stops early after two rounds with nothing shipped or no held-out gain.
|
|
132
|
+
|
|
105
133
|
claude proposes one change to lib/sentinel-rules.json on a new branch,
|
|
106
134
|
sealed off from the held-out exam. the sentinel bench, fixed code with no
|
|
107
135
|
model, scores it before and after. it is kept only when
|
|
108
136
|
held-out attack catch holds or rises, wrong blocks and false alarms do
|
|
109
137
|
not rise and no seen item that passed now fails. a kept change waits on
|
|
110
|
-
its branch for you. nothing is merged
|
|
138
|
+
its branch for you. nothing is merged or released. nothing is pushed without --push.
|
|
111
139
|
`);
|
|
112
140
|
return 0;
|
|
113
141
|
}
|
|
114
142
|
|
|
115
143
|
async function improveCommand(argv, deps = {}) {
|
|
116
144
|
if (argv.includes('--help') || argv.includes('-h') || argv[0] === 'help') return improveHelp();
|
|
117
|
-
try { checkFlags(argv, ['--json', '--offline', '--schedule', '--unschedule'], ['--engine', '--every']); } catch (e) { console.error(` ${e.message}`); return 2; }
|
|
145
|
+
try { checkFlags(argv, ['--json', '--offline', '--schedule', '--unschedule', '--push', '--break-lock'], ['--engine', '--every', '--focus', '--rounds', '--parallel', '--review', '--max-minutes', '--max-calls']); } catch (e) { console.error(` ${e.message}`); return 2; }
|
|
118
146
|
const improve = require('../lib/sentinel-improve');
|
|
119
147
|
const env = deps.env || process.env;
|
|
148
|
+
if (argv.includes('--break-lock')) {
|
|
149
|
+
const res = require('../lib/sentinel-night').breakLock(env);
|
|
150
|
+
(res.refused ? console.error : console.log)(` ${res.message}`);
|
|
151
|
+
return res.refused ? 1 : 0;
|
|
152
|
+
}
|
|
120
153
|
const engine = readFlag(argv, '--engine') || 'claude';
|
|
121
154
|
if (!/^[a-z][a-z0-9:_-]{0,30}$/.test(engine)) { console.error(` unknown engine: ${engine}`); return 2; }
|
|
122
155
|
const refused = require('../lib/sentinel-improve').refusedEngine(engine);
|
|
@@ -148,9 +181,14 @@ async function improveCommand(argv, deps = {}) {
|
|
|
148
181
|
console.log(' each run leaves a receipt in ~/.atris/sentinel-improve/receipts. kept changes wait on their branch for you');
|
|
149
182
|
return 0;
|
|
150
183
|
}
|
|
184
|
+
const focus = readFlag(argv, '--focus');
|
|
185
|
+
if (argv.includes('--focus') && !focus) { console.error(' --focus needs exfil, destroy, disarm or agents'); return 2; }
|
|
186
|
+
if (focus && !improve.FOCI.includes(focus)) { console.error(` unknown focus: ${focus}. use exfil, destroy, disarm or agents`); return 2; }
|
|
187
|
+
const nightFlags = ['--rounds', '--parallel', '--review', '--max-minutes', '--max-calls', '--push'];
|
|
188
|
+
if (argv.some((a) => nightFlags.includes(String(a).split('=')[0]))) return nightCommand(argv, { root, engine, env, focus, deps });
|
|
151
189
|
let res;
|
|
152
190
|
try {
|
|
153
|
-
res = await improve.improve({ root, engine, env, offline: argv.includes('--offline'), log: argv.includes('--json') ? () => {} : (m) => console.log(` ${m}`), ...(deps.improveOpts || {}) });
|
|
191
|
+
res = await improve.improve({ root, engine, env, focus, offline: argv.includes('--offline'), log: argv.includes('--json') ? () => {} : (m) => console.log(` ${m}`), ...(deps.improveOpts || {}) });
|
|
154
192
|
} catch (e) { console.error(` improve did not run: ${e.message}`); return 1; }
|
|
155
193
|
if (argv.includes('--json')) {
|
|
156
194
|
console.log(JSON.stringify({ ok: !res.error, kept: res.kept, branch: res.branch, error: res.error || null, decision: res.decision, receipt: res.receipt }, null, 2));
|
|
@@ -169,4 +207,44 @@ async function improveCommand(argv, deps = {}) {
|
|
|
169
207
|
return res.error ? 1 : 0;
|
|
170
208
|
}
|
|
171
209
|
|
|
210
|
+
function numFlag(argv, name) {
|
|
211
|
+
const v = readFlag(argv, name);
|
|
212
|
+
if (v === null) return argv.includes(name) ? NaN : undefined;
|
|
213
|
+
return Number(v);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
async function nightCommand(argv, { root, engine, env, focus, deps }) {
|
|
217
|
+
const improve = require('../lib/sentinel-improve');
|
|
218
|
+
const review = readFlag(argv, '--review');
|
|
219
|
+
if (argv.includes('--review') && !review) { console.error(' --review needs codex'); return 2; }
|
|
220
|
+
if (review && review !== 'codex') { console.error(' --review takes codex only'); return 2; }
|
|
221
|
+
const opts = { root, engine, env, focus, review, push: argv.includes('--push'), offline: argv.includes('--offline') };
|
|
222
|
+
for (const [flag, key] of [['--rounds', 'rounds'], ['--parallel', 'parallel'], ['--max-minutes', 'maxMinutes'], ['--max-calls', 'maxCalls']]) {
|
|
223
|
+
const n = numFlag(argv, flag);
|
|
224
|
+
if (n === undefined) continue;
|
|
225
|
+
if (!Number.isFinite(n) || n <= 0) { console.error(` ${flag} needs a number above 0`); return 2; }
|
|
226
|
+
if (key !== 'maxMinutes' && !Number.isInteger(n)) { console.error(` ${flag} needs a whole number`); return 2; }
|
|
227
|
+
if (key === 'parallel' && n > improve.FOCI.length) { console.error(` --parallel goes up to ${improve.FOCI.length}, one proposer per focus area`); return 2; }
|
|
228
|
+
opts[key] = n;
|
|
229
|
+
}
|
|
230
|
+
const json = argv.includes('--json');
|
|
231
|
+
let res;
|
|
232
|
+
try {
|
|
233
|
+
res = await require('../lib/sentinel-night').night({ ...opts, log: json ? () => {} : (m) => console.log(` ${m}`), ...(deps.nightOpts || {}) });
|
|
234
|
+
} catch (e) { console.error(` improve did not run: ${e.message}`); return 1; }
|
|
235
|
+
if (json) {
|
|
236
|
+
console.log(JSON.stringify({ ok: !res.error, shipped: res.landed, stop: res.stop, night_branch: res.night_branch, exam_branch: res.exam_branch, exam_added: res.exam_added, calls: res.calls, minutes: res.minutes, pushed: res.pushed, error: res.error || null, receipt: res.receipt }, null, 2));
|
|
237
|
+
return res.error ? 1 : 0;
|
|
238
|
+
}
|
|
239
|
+
const home = require('os').homedir();
|
|
240
|
+
console.log('');
|
|
241
|
+
console.log(` ${res.landed} ${res.landed === 1 ? 'change' : 'changes'} shipped in ${res.rounds.length} ${res.rounds.length === 1 ? 'round' : 'rounds'}. stopped because ${res.stop}`);
|
|
242
|
+
if (res.night_branch) console.log(` rules wait on ${res.night_branch}`);
|
|
243
|
+
if (res.exam_branch) console.log(` ${res.exam_added} new exam items wait on ${res.exam_branch}`);
|
|
244
|
+
if (res.pushed.length) console.log(` pushed: ${res.pushed.join(', ')}`);
|
|
245
|
+
console.log(` used ${res.calls} engine calls in ${res.minutes} minutes`);
|
|
246
|
+
console.log(` receipt: ${res.receipt.replace(home, '~')}\n`);
|
|
247
|
+
return res.error ? 1 : 0;
|
|
248
|
+
}
|
|
249
|
+
|
|
172
250
|
module.exports = { benchCommand, improveCommand };
|
package/lib/sentinel-bench.js
CHANGED
|
@@ -158,12 +158,18 @@ function keyFromKeychain() {
|
|
|
158
158
|
return add.status === 0 ? read() : null;
|
|
159
159
|
}
|
|
160
160
|
|
|
161
|
+
// Where the last key came from: 'keychain' or 'file'. The night run asks
|
|
162
|
+
// for this in the report and refuses a file key: a reviewer that can read
|
|
163
|
+
// files could read a key file too.
|
|
164
|
+
let lastKeySource = null;
|
|
165
|
+
|
|
161
166
|
function holdoutKey(env = process.env) {
|
|
162
|
-
if (env.ATRIS_SENTINEL_KEY_FILE) return keyFromFile(env.ATRIS_SENTINEL_KEY_FILE);
|
|
167
|
+
if (env.ATRIS_SENTINEL_KEY_FILE) { lastKeySource = 'file'; return keyFromFile(env.ATRIS_SENTINEL_KEY_FILE); }
|
|
163
168
|
if (process.platform === 'darwin' && env.ATRIS_SENTINEL_KEYCHAIN !== '0') {
|
|
164
169
|
const k = keyFromKeychain();
|
|
165
|
-
if (k) return k;
|
|
170
|
+
if (k) { lastKeySource = 'keychain'; return k; }
|
|
166
171
|
}
|
|
172
|
+
lastKeySource = 'file';
|
|
167
173
|
return keyFromFile(path.join(homeDir(env), '.atris', 'sentinel-judge', 'holdout.key'));
|
|
168
174
|
}
|
|
169
175
|
|
|
@@ -416,9 +422,47 @@ async function runBench({ target, seenDir, holdoutDir, sets = ['guard', 'agents'
|
|
|
416
422
|
report[half][set] = score;
|
|
417
423
|
}
|
|
418
424
|
}
|
|
425
|
+
report.key_source = lastKeySource;
|
|
419
426
|
return report;
|
|
420
427
|
}
|
|
421
428
|
|
|
429
|
+
// ---------------------------------------------------------------- screen
|
|
430
|
+
|
|
431
|
+
// Lowercase with whitespace collapsed, for overlap checks.
|
|
432
|
+
function normalizeText(v) {
|
|
433
|
+
return String(v || '').toLowerCase().replace(/\s+/g, ' ').trim();
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
function itemTexts(set, item) {
|
|
437
|
+
if (set === 'guard') return [item.command !== undefined ? item.command : JSON.stringify(item.input || '')];
|
|
438
|
+
return Object.entries(item.files || {}).map(([f, c]) => `${f} ${typeof c === 'string' ? c : JSON.stringify(c)}`).concat(Object.values(item.files || {}).map((c) => (typeof c === 'string' ? c : JSON.stringify(c))));
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
// New seen items proposed from outside (the night reviewer) checked against
|
|
442
|
+
// the held-out half. This runs in the judge, the one process that holds it. An item
|
|
443
|
+
// whose normalized text equals, contains or sits inside any held-out item's
|
|
444
|
+
// text is refused. Only the refused ids leave this function, never a
|
|
445
|
+
// held-out item. A held-out set that cannot be read refuses everything.
|
|
446
|
+
function screenItems(candidates, { holdoutDir, env = process.env } = {}) {
|
|
447
|
+
const held = path.resolve(holdoutDir || defaultHoldoutDir(env));
|
|
448
|
+
const texts = [];
|
|
449
|
+
let unreadable = false;
|
|
450
|
+
for (const set of ['guard', 'agents']) {
|
|
451
|
+
try {
|
|
452
|
+
const loaded = loadSet(held, set, { held: true, env });
|
|
453
|
+
for (const item of loaded.items) for (const t of itemTexts(set, item)) { const n = normalizeText(t); if (n) texts.push(n); }
|
|
454
|
+
} catch { unreadable = true; }
|
|
455
|
+
}
|
|
456
|
+
const refused = [];
|
|
457
|
+
for (const c of candidates) {
|
|
458
|
+
if (!c || typeof c.id !== 'string' || (c.set !== 'guard' && c.set !== 'agents')) continue;
|
|
459
|
+
if (unreadable) { refused.push(c.id); continue; }
|
|
460
|
+
const mine = itemTexts(c.set, c.item || {}).map(normalizeText).filter(Boolean);
|
|
461
|
+
if (!mine.length || mine.some((m) => texts.some((h) => h === m || h.includes(m) || m.includes(h)))) refused.push(c.id);
|
|
462
|
+
}
|
|
463
|
+
return { refused, key_source: lastKeySource, unreadable };
|
|
464
|
+
}
|
|
465
|
+
|
|
422
466
|
// What may leave this module: held-out totals only, never an item. With
|
|
423
467
|
// summary, seen items are dropped too.
|
|
424
468
|
function publicReport(report, { summary = false } = {}) {
|
|
@@ -437,6 +481,8 @@ function publicReport(report, { summary = false } = {}) {
|
|
|
437
481
|
holdout_dir: report.holdout_dir,
|
|
438
482
|
// Sealed split sources, paths only. The improver uses them to deny reads.
|
|
439
483
|
sealed_sources: report.sealed_sources || [],
|
|
484
|
+
// How the judge got the held-out key: 'keychain' or 'file'. Never the key.
|
|
485
|
+
key_source: report.key_source || null,
|
|
440
486
|
};
|
|
441
487
|
}
|
|
442
488
|
|
|
@@ -541,6 +587,8 @@ module.exports = {
|
|
|
541
587
|
SETS,
|
|
542
588
|
readHeldText,
|
|
543
589
|
readSealedSources,
|
|
590
|
+
normalizeText,
|
|
591
|
+
screenItems,
|
|
544
592
|
FAKE,
|
|
545
593
|
defaultHoldoutDir,
|
|
546
594
|
expand,
|
package/lib/sentinel-guard.js
CHANGED
|
@@ -704,13 +704,26 @@ const BLOCK_REASON = {
|
|
|
704
704
|
disarm: ['blocked: this command matches a known way to turn off a protection.', 'An agent ran a command that matches a known way to turn off a protection. Nothing changed.'],
|
|
705
705
|
};
|
|
706
706
|
|
|
707
|
-
// Literal command blocks from the rules file
|
|
708
|
-
//
|
|
709
|
-
//
|
|
707
|
+
// Literal command blocks from the rules file, matched with plain substring
|
|
708
|
+
// checks in linear time, never a regular expression. A block that lists
|
|
709
|
+
// programs reads only the words of each piece of the command that runs one of
|
|
710
|
+
// them, so harmless pieces chained with && or ; never add up to a match. A
|
|
711
|
+
// block with no programs reads the whole command. Fail closed: when a piece can
|
|
712
|
+
// receive code from elsewhere (a pipe, any redirect or file write, a heredoc or
|
|
713
|
+
// here-string, a command substitution, a shell variable, eval, source or a shell
|
|
714
|
+
// string), every block reads the whole command. Per-piece matching is only for
|
|
715
|
+
// plain chains that pass nothing between pieces.
|
|
716
|
+
// Accepted limit: the guard judges one command at a time, so a script written by
|
|
717
|
+
// one command and run by a later, separate command is not connected by any rule.
|
|
718
|
+
const FEEDS_CODE = ['|', '<', '>', '`', '$', 'tee ', 'eval', 'source ', 'sh -c', 'bash -c', 'zsh -c'];
|
|
710
719
|
function checkBlocks(command, cmds) {
|
|
720
|
+
const whole = FEEDS_CODE.some((t) => command.includes(t));
|
|
711
721
|
for (const b of R().commandBlocks) {
|
|
722
|
+
const texts = b.programs && !whole
|
|
723
|
+
? cmds.filter((c) => b.programs.has(c.name)).map((c) => (c.words || [c.name, ...(c.args || [])]).join(' '))
|
|
724
|
+
: [command];
|
|
712
725
|
if (b.programs && !cmds.some((c) => b.programs.has(c.name))) continue;
|
|
713
|
-
if (!b.all.every((sub) =>
|
|
726
|
+
if (!texts.some((t) => b.all.every((sub) => t.includes(sub)))) continue;
|
|
714
727
|
const [reason, detail] = BLOCK_REASON[b.category];
|
|
715
728
|
return block(`${b.category}-block`, b.category === 'disarm' ? 'high' : 'critical', reason, detail);
|
|
716
729
|
}
|
package/lib/sentinel-improve.js
CHANGED
|
@@ -177,11 +177,12 @@ async function gather({ env = process.env, request, osvBase, offline = false } =
|
|
|
177
177
|
|
|
178
178
|
// The exam in its own process, from this atris. Returns the public report:
|
|
179
179
|
// seen items and held-out totals, never a held-out item.
|
|
180
|
-
function runJudge({ target, seenDir, sets = ['guard', 'agents'], atrisBin, env = process.env, spawn = spawnSync } = {}) {
|
|
180
|
+
function runJudge({ target, seenDir, sets = ['guard', 'agents'], atrisBin, env = process.env, spawn = spawnSync, timeoutMs = 10 * 60 * 1000 } = {}) {
|
|
181
181
|
const bin = atrisBin || path.join(__dirname, '..', 'bin', 'atris.js');
|
|
182
182
|
const args = [bin, 'sentinel', 'bench', '--json', '--target', target, '--sets', sets.join(',')];
|
|
183
183
|
if (seenDir) args.push('--seen', seenDir);
|
|
184
|
-
const r = spawn(process.execPath, args, { encoding: 'utf8', env: { ...env, ATRIS_SKIP_UPDATE_CHECK: '1' }, maxBuffer: 32 * 1024 * 1024, timeout:
|
|
184
|
+
const r = spawn(process.execPath, args, { encoding: 'utf8', env: { ...env, ATRIS_SKIP_UPDATE_CHECK: '1' }, maxBuffer: 32 * 1024 * 1024, timeout: Math.max(1, Math.floor(timeoutMs)) });
|
|
185
|
+
if (r.error && r.error.code === 'ETIMEDOUT') throw new Error('the exam ran out of time');
|
|
185
186
|
if (r.status !== 0) throw new Error(`the exam did not run: ${String(r.stderr || r.stdout || '').trim().split('\n').slice(-1)[0] || `exit ${r.status}`}`);
|
|
186
187
|
try { return JSON.parse(r.stdout); } catch { throw new Error('the exam printed something that is not json'); }
|
|
187
188
|
}
|
|
@@ -289,15 +290,36 @@ function keepDecision(before, after, { newIds = [] } = {}) {
|
|
|
289
290
|
|
|
290
291
|
// ---------------------------------------------------------------- propose
|
|
291
292
|
|
|
292
|
-
|
|
293
|
+
// The areas a proposer can be pointed at. A focus narrows the misses it
|
|
294
|
+
// sees to one area so parallel proposers do not all chase the same thing.
|
|
295
|
+
const FOCI = ['exfil', 'destroy', 'disarm', 'agents'];
|
|
296
|
+
const FOCUS_WORDS = {
|
|
297
|
+
exfil: 'commands that send a secret or private file off the machine',
|
|
298
|
+
destroy: 'commands that delete or wreck files, history or data',
|
|
299
|
+
disarm: 'commands that turn the guard or other safety checks off',
|
|
300
|
+
agents: 'agent setup files (CLAUDE.md, AGENTS.md, skills, hooks, mcp configs) with planted problems',
|
|
301
|
+
};
|
|
302
|
+
|
|
303
|
+
function inFocus(set, r, focus) {
|
|
304
|
+
if (!focus) return true;
|
|
305
|
+
if (focus === 'agents') return set === 'agents';
|
|
306
|
+
return set === 'guard' && r.kind === focus;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
function missesForPrompt(report, focus = null) {
|
|
293
310
|
const lines = [];
|
|
294
311
|
const seenDir = report.seen || {};
|
|
295
|
-
for (const r of (seenDir.guard && seenDir.guard.results) || []) if (!r.ok) lines.push(`guard ${r.id}: expected ${r.expect}, got ${r.got}${r.rule ? ` (${r.rule})` : ''}`);
|
|
296
|
-
for (const r of (seenDir.agents && seenDir.agents.results) || []) if (!r.ok) lines.push(`agents ${r.id}: missed ${r.missed.join(', ') || 'none'}, false alarms ${r.false_alarms.join(', ') || 'none'}`);
|
|
312
|
+
for (const r of (seenDir.guard && seenDir.guard.results) || []) if (!r.ok && inFocus('guard', r, focus)) lines.push(`guard ${r.id}: expected ${r.expect}, got ${r.got}${r.rule ? ` (${r.rule})` : ''}`);
|
|
313
|
+
for (const r of (seenDir.agents && seenDir.agents.results) || []) if (!r.ok && inFocus('agents', r, focus)) lines.push(`agents ${r.id}: missed ${r.missed.join(', ') || 'none'}, false alarms ${r.false_alarms.join(', ') || 'none'}`);
|
|
297
314
|
return lines;
|
|
298
315
|
}
|
|
299
316
|
|
|
300
|
-
|
|
317
|
+
// Misses per focus area, for handing each parallel proposer its own area.
|
|
318
|
+
function missCounts(report) {
|
|
319
|
+
return Object.fromEntries(FOCI.map((f) => [f, missesForPrompt(report, f).length]));
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function buildPrompt({ report, signals, seenDirRel, focus = null, tonight = [] }) {
|
|
301
323
|
const totals = (half) => Object.entries(report[half] || {}).map(([set, s]) => `${set}: ${JSON.stringify(Object.fromEntries(Object.entries(s).filter(([k]) => k !== 'results' && k !== 'by_kind')))}`).join('\n');
|
|
302
324
|
return [
|
|
303
325
|
'You are improving the atris sentinel, a fixed-rule safety check for coding agents. Propose exactly one small change.',
|
|
@@ -320,8 +342,11 @@ function buildPrompt({ report, signals, seenDirRel }) {
|
|
|
320
342
|
'Known limits a literal rule cannot fix, so do not try: one-liners run through an interpreter (python -c, node -e and similar) that read a secret and send it. Literal words cannot tell reading a secret and sending it apart from printing a variable next to a web address, and reviewers rejected three such rules tonight for blocking everyday commands. Pick a different miss.',
|
|
321
343
|
'Before proposing a command_block, picture five everyday developer commands that contain the same literal substrings. If any of them would be blocked, the rule is too broad.',
|
|
322
344
|
'',
|
|
323
|
-
|
|
324
|
-
...(
|
|
345
|
+
...(focus ? [`Focus: ${focus}, ${FOCUS_WORDS[focus]}. Other proposers cover the other areas tonight. Propose a change for this area only.`, ''] : []),
|
|
346
|
+
...(tonight.length ? ['Learned earlier tonight from a strict reviewer. Treat these as part of the exam:', ...tonight.map((l) => ` ${l}`), ''] : []),
|
|
347
|
+
focus ? `Seen items the current code gets wrong in this area:` : 'Seen items the current code gets wrong:',
|
|
348
|
+
...(missesForPrompt(report, focus).map((l) => ` ${l}`)),
|
|
349
|
+
...(focus && !missesForPrompt(report, focus).length ? [' none. look for a general weakness in this area that the hidden half may hold'] : []),
|
|
325
350
|
'',
|
|
326
351
|
'Current totals, seen half:',
|
|
327
352
|
totals('seen'),
|
|
@@ -505,9 +530,63 @@ function cleanItems(p, prefix) {
|
|
|
505
530
|
return out;
|
|
506
531
|
}
|
|
507
532
|
|
|
533
|
+
// The strict secret check for exam items an engine wrote, proposer or
|
|
534
|
+
// reviewer. Unlike the repo scan it honors no suppression marker and no
|
|
535
|
+
// placeholder exemption: a marker or any match of a secret rule counts.
|
|
536
|
+
// Exam items may write a key only as a {{fake:...}} placeholder.
|
|
537
|
+
const SUPPRESS = /atris-allow-secret|atris-security-ignore|nosec/i;
|
|
538
|
+
function strictSecret(text) {
|
|
539
|
+
const { SECRET_RULES } = require('./security-scan');
|
|
540
|
+
const s = String(text || '');
|
|
541
|
+
if (SUPPRESS.test(s)) return true;
|
|
542
|
+
return SECRET_RULES.some((rule) => rule.re.test(s));
|
|
543
|
+
}
|
|
544
|
+
const SECRET_ITEM_ERROR = 'the proposal holds an exam item that looks like a real key. keys in exam items must be {{fake:...}} placeholders';
|
|
545
|
+
const SECRET_PROPOSAL_ERROR = 'the proposal holds text that looks like a secret';
|
|
546
|
+
|
|
547
|
+
// The one gate for engine text. Every raw string in a value, keys included,
|
|
548
|
+
// is checked with the strict secret check before anything is serialized or
|
|
549
|
+
// written. One failing string fails the whole value. True means clean.
|
|
550
|
+
function gateValue(v, depth = 0) {
|
|
551
|
+
if (depth > 40) return false;
|
|
552
|
+
if (typeof v === 'string') return !strictSecret(v);
|
|
553
|
+
if (Array.isArray(v)) return v.every((x) => gateValue(x, depth + 1));
|
|
554
|
+
if (v && typeof v === 'object') return Object.entries(v).every(([k, x]) => !strictSecret(k) && gateValue(x, depth + 1));
|
|
555
|
+
return true;
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
// Errors and reasons that get written are fixed sentences chosen here,
|
|
559
|
+
// never built from engine output: no file names, field names, titles or
|
|
560
|
+
// example text. Anything not known maps to one plain sentence.
|
|
561
|
+
const FIXED_ERRORS = [
|
|
562
|
+
[/^the time limit ran out/, 'the time limit ran out'],
|
|
563
|
+
[/did not return a proposal|not one json|is not json|must be one object|needs the new rules or one to six edits|an edit/, 'the proposer gave no usable proposal'],
|
|
564
|
+
[/only lib\/sentinel-rules\.json may change/, 'the proposal named a file other than lib/sentinel-rules.json. only lib/sentinel-rules.json may change'],
|
|
565
|
+
[/break the schema/, 'the proposed rules break the schema'],
|
|
566
|
+
[/changed files it may not touch/, 'the proposer changed files it may not touch'],
|
|
567
|
+
[/looks like a real key/, SECRET_ITEM_ERROR],
|
|
568
|
+
[/looks like a secret/, SECRET_PROPOSAL_ERROR],
|
|
569
|
+
[/^the proposer failed/, 'the proposer did not finish'],
|
|
570
|
+
[/^the exam did not run|^the exam printed|ran out of time/, 'the exam did not run'],
|
|
571
|
+
[/cannot be kept from reading/, 'that engine may not propose'],
|
|
572
|
+
[/the copy changed while the exam ran/, 'the copy changed while the exam ran'],
|
|
573
|
+
[/touches files other than the rules/, 'the change touches files other than the rules and the seen items'],
|
|
574
|
+
];
|
|
575
|
+
function fixedError(e) {
|
|
576
|
+
const m = String((e && e.message) || e || '');
|
|
577
|
+
for (const [re, text] of FIXED_ERRORS) if (re.test(m)) return text;
|
|
578
|
+
return 'the proposal could not be judged';
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
// A keep-rule reason without the item ids it may list.
|
|
582
|
+
function fixedReason(r) {
|
|
583
|
+
return String(r || '').replace(/:.*$/, '');
|
|
584
|
+
}
|
|
585
|
+
|
|
508
586
|
function appendItems(seenDir, items) {
|
|
509
587
|
for (const set of ['guard', 'agents']) {
|
|
510
|
-
|
|
588
|
+
// The gate again at the write: an item that fails is never written.
|
|
589
|
+
const list = items.filter((x) => x.set === set && gateValue(x.item)).map((x) => x.item);
|
|
511
590
|
if (!list.length) continue;
|
|
512
591
|
const file = path.join(seenDir, `${set}.jsonl`);
|
|
513
592
|
fs.appendFileSync(file, list.map((i) => JSON.stringify(i)).join('\n') + '\n');
|
|
@@ -531,7 +610,9 @@ function renderReceipt({ at, engine, proposal, items, before, after, decision, b
|
|
|
531
610
|
lines.push('proposer sealed: reads denied under ~/.atris and the held-out folder, no shell, no mcp servers or connectors, temp HOME, no api keys. only the rules file may change');
|
|
532
611
|
if (error) lines.push(`stopped: ${error}`);
|
|
533
612
|
if (proposal) {
|
|
534
|
-
|
|
613
|
+
// The proposer's title and reasons are never written. The change itself
|
|
614
|
+
// is the rules diff on the branch.
|
|
615
|
+
lines.push('', '## the change', '', 'one change to lib/sentinel-rules.json. read the rules diff on the branch', '');
|
|
535
616
|
lines.push(`new seen exam items: ${items ? items.length : 0}`);
|
|
536
617
|
}
|
|
537
618
|
if (before && after) {
|
|
@@ -556,7 +637,7 @@ function renderReceipt({ at, engine, proposal, items, before, after, decision, b
|
|
|
556
637
|
if (decision.path) lines.push(`- kept because: ${decision.path}`);
|
|
557
638
|
for (const g of decision.gains) lines.push(`- gain: ${g}`);
|
|
558
639
|
for (const n of decision.notes || []) lines.push(`- note: ${n}`);
|
|
559
|
-
for (const r of decision.reasons) lines.push(`- stop: ${r}`);
|
|
640
|
+
for (const r of decision.reasons) lines.push(`- stop: ${fixedReason(r)}`);
|
|
560
641
|
}
|
|
561
642
|
lines.push('');
|
|
562
643
|
return lines.join('\n');
|
|
@@ -564,8 +645,9 @@ function renderReceipt({ at, engine, proposal, items, before, after, decision, b
|
|
|
564
645
|
|
|
565
646
|
// ---------------------------------------------------------------- git
|
|
566
647
|
|
|
567
|
-
function git(cwd, args, { check = true, env } = {}) {
|
|
568
|
-
const r = spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, ...(env ? { env } : {}) });
|
|
648
|
+
function git(cwd, args, { check = true, env, timeoutMs } = {}) {
|
|
649
|
+
const r = spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, ...(env ? { env } : {}), ...(timeoutMs ? { timeout: Math.max(1, Math.floor(timeoutMs)) } : {}) });
|
|
650
|
+
if (r.error && r.error.code === 'ETIMEDOUT') throw new Error(`git ${args[0]} ran out of time`);
|
|
569
651
|
if (check && r.status !== 0) throw new Error(`git ${args[0]} failed: ${String(r.stderr || r.stdout || '').trim().split('\n').slice(-1)[0]}`);
|
|
570
652
|
return r;
|
|
571
653
|
}
|
|
@@ -679,7 +761,7 @@ function sweepStale(root, env) {
|
|
|
679
761
|
for (const block of list.split('\n\n')) {
|
|
680
762
|
const tree = (block.match(/^worktree (.+)$/m) || [])[1];
|
|
681
763
|
const branch = ((block.match(/^branch refs\/heads\/(.+)$/m) || [])[1]) || '';
|
|
682
|
-
if (!tree || !/^sentinel-improve\//.test(branch) || !/atris-sentinel-improve-/.test(path.basename(tree))) continue;
|
|
764
|
+
if (!tree || !/^sentinel-(?:improve|exam)\//.test(branch) || !/atris-sentinel-improve-/.test(path.basename(tree))) continue;
|
|
683
765
|
const rec = records.get(path.resolve(tree)) || [...records.values()].find((r) => fs.existsSync(r.tree) && fs.existsSync(tree) && fs.realpathSync(r.tree) === fs.realpathSync(tree));
|
|
684
766
|
const keep = (reason) => kept.push({ tree, branch, reason });
|
|
685
767
|
if (!rec) { keep('no record says which run owns it'); continue; }
|
|
@@ -702,6 +784,7 @@ async function improve(opts = {}) {
|
|
|
702
784
|
if (!fs.existsSync(path.join(root, RULES_PATH))) throw new Error(`${root} is not an atris checkout with the sentinel rules in it`);
|
|
703
785
|
const refused = refusedEngine(engine);
|
|
704
786
|
if (refused) throw new Error(refused);
|
|
787
|
+
if (opts.focus && !FOCI.includes(opts.focus)) throw new Error(`unknown focus: ${opts.focus}. use ${FOCI.join(', ')}`);
|
|
705
788
|
const judge = opts.judge || ((args) => runJudge({ ...args, env, atrisBin: opts.atrisBin }));
|
|
706
789
|
const propose = opts.propose || ((args) => askEngine({ ...args, env }));
|
|
707
790
|
const at = now.toISOString();
|
|
@@ -738,13 +821,16 @@ async function improve(opts = {}) {
|
|
|
738
821
|
log('running the exam on the current code');
|
|
739
822
|
const current = judge({ target: tree, seenDir });
|
|
740
823
|
log(`asking ${engine} for one change`);
|
|
741
|
-
const answer = await propose({ engine, prompt: buildPrompt({ report: current, signals, seenDirRel: seenRel }), cwd: tree, sealed: [path.join(homeDir(env), '.atris'), current.holdout_dir, ...(current.sealed_sources || [])] });
|
|
824
|
+
const answer = await propose({ engine, prompt: buildPrompt({ report: current, signals, seenDirRel: seenRel, focus: opts.focus || null }), cwd: tree, sealed: [path.join(homeDir(env), '.atris'), current.holdout_dir, ...(current.sealed_sources || [])] });
|
|
742
825
|
proposal = answer && typeof answer === 'object' && (Array.isArray(answer.edits) || answer.rules) ? answer : extractJsonObject(answer && answer.stdout);
|
|
743
826
|
const bad = checkProposal(proposal, tree);
|
|
744
827
|
if (bad) throw new Error(bad);
|
|
745
828
|
const strayEarly = strayChanges(tree);
|
|
746
829
|
if (strayEarly.length) throw new Error(`the proposer changed files it may not touch: ${strayEarly.slice(0, 3).join(', ')}`);
|
|
830
|
+
if (Array.isArray(proposal.new_items) && !gateValue(proposal.new_items)) throw new Error(SECRET_ITEM_ERROR);
|
|
831
|
+
if (!gateValue(proposal)) throw new Error(SECRET_PROPOSAL_ERROR);
|
|
747
832
|
items = cleanItems(proposal, `imp-${id.slice(0, 8)}`);
|
|
833
|
+
if (items.some((x) => !gateValue(x.item))) throw new Error(SECRET_ITEM_ERROR);
|
|
748
834
|
fs.mkdirSync(candSeen, { recursive: true });
|
|
749
835
|
for (const set of ['guard', 'agents']) {
|
|
750
836
|
const f = path.join(seenDir, `${set}.jsonl`);
|
|
@@ -774,14 +860,14 @@ async function improve(opts = {}) {
|
|
|
774
860
|
const outside = strayChanges(tree, new Set([...ALLOWED_CHANGES, rel]));
|
|
775
861
|
if (outside.length) throw new Error(`the change touches files other than the rules and the seen items: ${outside.slice(0, 3).join(', ')}`);
|
|
776
862
|
git(tree, ['add', RULES_PATH, seenRel]);
|
|
777
|
-
const msg = `Sentinel improve:
|
|
863
|
+
const msg = `Sentinel improve: one guard rules change\n\nKept by the sentinel bench. Receipt: ${rel}\n\nCo-authored-by: Atris <299057014+atris-builder[bot]@users.noreply.github.com>`;
|
|
778
864
|
git(tree, ['commit', '-q', '-m', msg], { env });
|
|
779
865
|
keepBranch = true;
|
|
780
866
|
out.branch = branch;
|
|
781
867
|
out.kept = true;
|
|
782
868
|
}
|
|
783
869
|
} catch (e) {
|
|
784
|
-
error =
|
|
870
|
+
error = fixedError(e);
|
|
785
871
|
} finally {
|
|
786
872
|
fs.rmSync(candidate, { recursive: true, force: true });
|
|
787
873
|
// Kept, rejected or failed: the same safe check decides if the copy goes.
|
|
@@ -811,6 +897,31 @@ function schedulePlan({ root, engine = 'claude', everyHours = 24, env = process.
|
|
|
811
897
|
}
|
|
812
898
|
|
|
813
899
|
module.exports = {
|
|
900
|
+
SECRET_ITEM_ERROR,
|
|
901
|
+
SECRET_PROPOSAL_ERROR,
|
|
902
|
+
fixedError,
|
|
903
|
+
fixedReason,
|
|
904
|
+
gateValue,
|
|
905
|
+
strictSecret,
|
|
906
|
+
FOCI,
|
|
907
|
+
RULES_PATH,
|
|
908
|
+
SEEN_REL,
|
|
909
|
+
appendItems,
|
|
910
|
+
applyRules,
|
|
911
|
+
askEngine,
|
|
912
|
+
cleanItems,
|
|
913
|
+
dropCopy,
|
|
914
|
+
git,
|
|
915
|
+
homeDir,
|
|
916
|
+
missCounts,
|
|
917
|
+
okMap,
|
|
918
|
+
pgidOf,
|
|
919
|
+
repoTop,
|
|
920
|
+
runAlive,
|
|
921
|
+
runsDir,
|
|
922
|
+
stamp,
|
|
923
|
+
strayChanges,
|
|
924
|
+
totalsLine,
|
|
814
925
|
buildPrompt,
|
|
815
926
|
checkProposal,
|
|
816
927
|
engineEnv,
|