atris 3.82.7 → 3.82.9

package/lib/sentinel-improve.js

@@ -177,11 +177,12 @@ async function gather({ env = process.env, request, osvBase, offline = false } =
 
 // The exam in its own process, from this atris. Returns the public report:
 // seen items and held-out totals, never a held-out item.
-function runJudge({ target, seenDir, sets = ['guard', 'agents'], atrisBin, env = process.env, spawn = spawnSync } = {}) {
+function runJudge({ target, seenDir, sets = ['guard', 'agents'], atrisBin, env = process.env, spawn = spawnSync, timeoutMs = 10 * 60 * 1000 } = {}) {
   const bin = atrisBin || path.join(__dirname, '..', 'bin', 'atris.js');
   const args = [bin, 'sentinel', 'bench', '--json', '--target', target, '--sets', sets.join(',')];
   if (seenDir) args.push('--seen', seenDir);
-  const r = spawn(process.execPath, args, { encoding: 'utf8', env: { ...env, ATRIS_SKIP_UPDATE_CHECK: '1' }, maxBuffer: 32 * 1024 * 1024, timeout: 10 * 60 * 1000 });
+  const r = spawn(process.execPath, args, { encoding: 'utf8', env: { ...env, ATRIS_SKIP_UPDATE_CHECK: '1' }, maxBuffer: 32 * 1024 * 1024, timeout: Math.max(1, Math.floor(timeoutMs)) });
+  if (r.error && r.error.code === 'ETIMEDOUT') throw new Error('the exam ran out of time');
   if (r.status !== 0) throw new Error(`the exam did not run: ${String(r.stderr || r.stdout || '').trim().split('\n').slice(-1)[0] || `exit ${r.status}`}`);
   try { return JSON.parse(r.stdout); } catch { throw new Error('the exam printed something that is not json'); }
 }
@@ -289,15 +290,36 @@ function keepDecision(before, after, { newIds = [] } = {}) {
 
 // ---------------------------------------------------------------- propose
 
-function missesForPrompt(report) {
+// The areas a proposer can be pointed at. A focus narrows the misses it
+// sees to one area so parallel proposers do not all chase the same thing.
+const FOCI = ['exfil', 'destroy', 'disarm', 'agents'];
+const FOCUS_WORDS = {
+  exfil: 'commands that send a secret or private file off the machine',
+  destroy: 'commands that delete or wreck files, history or data',
+  disarm: 'commands that turn the guard or other safety checks off',
+  agents: 'agent setup files (CLAUDE.md, AGENTS.md, skills, hooks, mcp configs) with planted problems',
+};
+
+function inFocus(set, r, focus) {
+  if (!focus) return true;
+  if (focus === 'agents') return set === 'agents';
+  return set === 'guard' && r.kind === focus;
+}
+
+function missesForPrompt(report, focus = null) {
   const lines = [];
   const seenDir = report.seen || {};
-  for (const r of (seenDir.guard && seenDir.guard.results) || []) if (!r.ok) lines.push(`guard ${r.id}: expected ${r.expect}, got ${r.got}${r.rule ? ` (${r.rule})` : ''}`);
-  for (const r of (seenDir.agents && seenDir.agents.results) || []) if (!r.ok) lines.push(`agents ${r.id}: missed ${r.missed.join(', ') || 'none'}, false alarms ${r.false_alarms.join(', ') || 'none'}`);
+  for (const r of (seenDir.guard && seenDir.guard.results) || []) if (!r.ok && inFocus('guard', r, focus)) lines.push(`guard ${r.id}: expected ${r.expect}, got ${r.got}${r.rule ? ` (${r.rule})` : ''}`);
+  for (const r of (seenDir.agents && seenDir.agents.results) || []) if (!r.ok && inFocus('agents', r, focus)) lines.push(`agents ${r.id}: missed ${r.missed.join(', ') || 'none'}, false alarms ${r.false_alarms.join(', ') || 'none'}`);
   return lines;
 }
 
-function buildPrompt({ report, signals, seenDirRel }) {
+// Misses per focus area, for handing each parallel proposer its own area.
+function missCounts(report) {
+  return Object.fromEntries(FOCI.map((f) => [f, missesForPrompt(report, f).length]));
+}
+
+function buildPrompt({ report, signals, seenDirRel, focus = null, tonight = [] }) {
   const totals = (half) => Object.entries(report[half] || {}).map(([set, s]) => `${set}: ${JSON.stringify(Object.fromEntries(Object.entries(s).filter(([k]) => k !== 'results' && k !== 'by_kind')))}`).join('\n');
   return [
     'You are improving the atris sentinel, a fixed-rule safety check for coding agents. Propose exactly one small change.',
@@ -317,8 +339,14 @@ function buildPrompt({ report, signals, seenDirRel }) {
     'The new items you add are scored and reported but never count toward keeping this change.',
     'So fix a general weakness, not one string. A rule that only matches the seen item will not move the hidden half.',
     '',
-    'Seen items the current code gets wrong:',
-    ...(missesForPrompt(report).map((l) => `  ${l}`)),
+    'Known limits a literal rule cannot fix, so do not try: one-liners run through an interpreter (python -c, node -e and similar) that read a secret and send it. Literal words cannot tell reading a secret and sending it apart from printing a variable next to a web address, and reviewers rejected three such rules tonight for blocking everyday commands. Pick a different miss.',
+    'Before proposing a command_block, picture five everyday developer commands that contain the same literal substrings. If any of them would be blocked, the rule is too broad.',
+    '',
+    ...(focus ? [`Focus: ${focus}, ${FOCUS_WORDS[focus]}. Other proposers cover the other areas tonight. Propose a change for this area only.`, ''] : []),
+    ...(tonight.length ? ['Learned earlier tonight from a strict reviewer. Treat these as part of the exam:', ...tonight.map((l) => `  ${l}`), ''] : []),
+    focus ? `Seen items the current code gets wrong in this area:` : 'Seen items the current code gets wrong:',
+    ...(missesForPrompt(report, focus).map((l) => `  ${l}`)),
+    ...(focus && !missesForPrompt(report, focus).length ? ['  none. look for a general weakness in this area that the hidden half may hold'] : []),
     '',
     'Current totals, seen half:',
     totals('seen'),
@@ -502,9 +530,63 @@ function cleanItems(p, prefix) {
   return out;
 }
 
+// The strict secret check for exam items an engine wrote, proposer or
+// reviewer. Unlike the repo scan it honors no suppression marker and no
+// placeholder exemption: a marker or any match of a secret rule counts.
+// Exam items may write a key only as a {{fake:...}} placeholder.
+const SUPPRESS = /atris-allow-secret|atris-security-ignore|nosec/i;
+function strictSecret(text) {
+  const { SECRET_RULES } = require('./security-scan');
+  const s = String(text || '');
+  if (SUPPRESS.test(s)) return true;
+  return SECRET_RULES.some((rule) => rule.re.test(s));
+}
+const SECRET_ITEM_ERROR = 'the proposal holds an exam item that looks like a real key. keys in exam items must be {{fake:...}} placeholders';
+const SECRET_PROPOSAL_ERROR = 'the proposal holds text that looks like a secret';
+
+// The one gate for engine text. Every raw string in a value, keys included,
+// is checked with the strict secret check before anything is serialized or
+// written. One failing string fails the whole value. True means clean.
+function gateValue(v, depth = 0) {
+  if (depth > 40) return false;
+  if (typeof v === 'string') return !strictSecret(v);
+  if (Array.isArray(v)) return v.every((x) => gateValue(x, depth + 1));
+  if (v && typeof v === 'object') return Object.entries(v).every(([k, x]) => !strictSecret(k) && gateValue(x, depth + 1));
+  return true;
+}
+
+// Errors and reasons that get written are fixed sentences chosen here,
+// never built from engine output: no file names, field names, titles or
+// example text. Anything not known maps to one plain sentence.
+const FIXED_ERRORS = [
+  [/^the time limit ran out/, 'the time limit ran out'],
+  [/did not return a proposal|not one json|is not json|must be one object|needs the new rules or one to six edits|an edit/, 'the proposer gave no usable proposal'],
+  [/only lib\/sentinel-rules\.json may change/, 'the proposal named a file other than lib/sentinel-rules.json. only lib/sentinel-rules.json may change'],
+  [/break the schema/, 'the proposed rules break the schema'],
+  [/changed files it may not touch/, 'the proposer changed files it may not touch'],
+  [/looks like a real key/, SECRET_ITEM_ERROR],
+  [/looks like a secret/, SECRET_PROPOSAL_ERROR],
+  [/^the proposer failed/, 'the proposer did not finish'],
+  [/^the exam did not run|^the exam printed|ran out of time/, 'the exam did not run'],
+  [/cannot be kept from reading/, 'that engine may not propose'],
+  [/the copy changed while the exam ran/, 'the copy changed while the exam ran'],
+  [/touches files other than the rules/, 'the change touches files other than the rules and the seen items'],
+];
+function fixedError(e) {
+  const m = String((e && e.message) || e || '');
+  for (const [re, text] of FIXED_ERRORS) if (re.test(m)) return text;
+  return 'the proposal could not be judged';
+}
+
+// A keep-rule reason without the item ids it may list.
+function fixedReason(r) {
+  return String(r || '').replace(/:.*$/, '');
+}
+
 function appendItems(seenDir, items) {
   for (const set of ['guard', 'agents']) {
-    const list = items.filter((x) => x.set === set).map((x) => x.item);
+    // The gate again at the write: an item that fails is never written.
+    const list = items.filter((x) => x.set === set && gateValue(x.item)).map((x) => x.item);
     if (!list.length) continue;
     const file = path.join(seenDir, `${set}.jsonl`);
     fs.appendFileSync(file, list.map((i) => JSON.stringify(i)).join('\n') + '\n');
@@ -528,7 +610,9 @@ function renderReceipt({ at, engine, proposal, items, before, after, decision, b
   lines.push('proposer sealed: reads denied under ~/.atris and the held-out folder, no shell, no mcp servers or connectors, temp HOME, no api keys. only the rules file may change');
   if (error) lines.push(`stopped: ${error}`);
   if (proposal) {
-    lines.push('', '## the change', '', `${String(proposal.title || 'untitled').slice(0, 300)}`, '', `${String(proposal.why || '').slice(0, 600)}`, '', `file: ${proposal.file}`);
+    // The proposer's title and reasons are never written. The change itself
+    // is the rules diff on the branch.
+    lines.push('', '## the change', '', 'one change to lib/sentinel-rules.json. read the rules diff on the branch', '');
     lines.push(`new seen exam items: ${items ? items.length : 0}`);
   }
   if (before && after) {
@@ -553,7 +637,7 @@ function renderReceipt({ at, engine, proposal, items, before, after, decision, b
     if (decision.path) lines.push(`- kept because: ${decision.path}`);
     for (const g of decision.gains) lines.push(`- gain: ${g}`);
     for (const n of decision.notes || []) lines.push(`- note: ${n}`);
-    for (const r of decision.reasons) lines.push(`- stop: ${r}`);
+    for (const r of decision.reasons) lines.push(`- stop: ${fixedReason(r)}`);
   }
   lines.push('');
   return lines.join('\n');
@@ -561,8 +645,9 @@ function renderReceipt({ at, engine, proposal, items, before, after, decision, b
 
 // ---------------------------------------------------------------- git
 
-function git(cwd, args, { check = true, env } = {}) {
-  const r = spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, ...(env ? { env } : {}) });
+function git(cwd, args, { check = true, env, timeoutMs } = {}) {
+  const r = spawnSync('git', args, { cwd, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, ...(env ? { env } : {}), ...(timeoutMs ? { timeout: Math.max(1, Math.floor(timeoutMs)) } : {}) });
+  if (r.error && r.error.code === 'ETIMEDOUT') throw new Error(`git ${args[0]} ran out of time`);
   if (check && r.status !== 0) throw new Error(`git ${args[0]} failed: ${String(r.stderr || r.stdout || '').trim().split('\n').slice(-1)[0]}`);
   return r;
 }
@@ -615,8 +700,9 @@ function foreignFiles(tree) {
 }
 
 // Files the run itself may leave in a copy besides tracked, committed ones.
-// Today nothing: the candidate and the receipt draft live outside the copy.
-const KNOWN_OUTPUT = new Set();
+// The candidate and the receipt draft live outside the copy. Running the exam
+// records the engine choice in the copy's own state file, which is ours.
+const KNOWN_OUTPUT = new Set(['.atris/state/engines.json']);
 
 // Live processes this run started. A run is alive by definition. For its
 // own copy the check is that every child it spawned has exited. Fail-closed:
@@ -675,7 +761,7 @@ function sweepStale(root, env) {
   for (const block of list.split('\n\n')) {
     const tree = (block.match(/^worktree (.+)$/m) || [])[1];
     const branch = ((block.match(/^branch refs\/heads\/(.+)$/m) || [])[1]) || '';
-    if (!tree || !/^sentinel-improve\//.test(branch) || !/atris-sentinel-improve-/.test(path.basename(tree))) continue;
+    if (!tree || !/^sentinel-(?:improve|exam)\//.test(branch) || !/atris-sentinel-improve-/.test(path.basename(tree))) continue;
     const rec = records.get(path.resolve(tree)) || [...records.values()].find((r) => fs.existsSync(r.tree) && fs.existsSync(tree) && fs.realpathSync(r.tree) === fs.realpathSync(tree));
     const keep = (reason) => kept.push({ tree, branch, reason });
     if (!rec) { keep('no record says which run owns it'); continue; }
@@ -698,6 +784,7 @@ async function improve(opts = {}) {
   if (!fs.existsSync(path.join(root, RULES_PATH))) throw new Error(`${root} is not an atris checkout with the sentinel rules in it`);
   const refused = refusedEngine(engine);
   if (refused) throw new Error(refused);
+  if (opts.focus && !FOCI.includes(opts.focus)) throw new Error(`unknown focus: ${opts.focus}. use ${FOCI.join(', ')}`);
   const judge = opts.judge || ((args) => runJudge({ ...args, env, atrisBin: opts.atrisBin }));
   const propose = opts.propose || ((args) => askEngine({ ...args, env }));
   const at = now.toISOString();
@@ -734,13 +821,16 @@ async function improve(opts = {}) {
     log('running the exam on the current code');
     const current = judge({ target: tree, seenDir });
     log(`asking ${engine} for one change`);
-    const answer = await propose({ engine, prompt: buildPrompt({ report: current, signals, seenDirRel: seenRel }), cwd: tree, sealed: [path.join(homeDir(env), '.atris'), current.holdout_dir, ...(current.sealed_sources || [])] });
+    const answer = await propose({ engine, prompt: buildPrompt({ report: current, signals, seenDirRel: seenRel, focus: opts.focus || null }), cwd: tree, sealed: [path.join(homeDir(env), '.atris'), current.holdout_dir, ...(current.sealed_sources || [])] });
     proposal = answer && typeof answer === 'object' && (Array.isArray(answer.edits) || answer.rules) ? answer : extractJsonObject(answer && answer.stdout);
     const bad = checkProposal(proposal, tree);
     if (bad) throw new Error(bad);
     const strayEarly = strayChanges(tree);
     if (strayEarly.length) throw new Error(`the proposer changed files it may not touch: ${strayEarly.slice(0, 3).join(', ')}`);
+    if (Array.isArray(proposal.new_items) && !gateValue(proposal.new_items)) throw new Error(SECRET_ITEM_ERROR);
+    if (!gateValue(proposal)) throw new Error(SECRET_PROPOSAL_ERROR);
     items = cleanItems(proposal, `imp-${id.slice(0, 8)}`);
+    if (items.some((x) => !gateValue(x.item))) throw new Error(SECRET_ITEM_ERROR);
     fs.mkdirSync(candSeen, { recursive: true });
     for (const set of ['guard', 'agents']) {
       const f = path.join(seenDir, `${set}.jsonl`);
@@ -770,14 +860,14 @@ async function improve(opts = {}) {
       const outside = strayChanges(tree, new Set([...ALLOWED_CHANGES, rel]));
       if (outside.length) throw new Error(`the change touches files other than the rules and the seen items: ${outside.slice(0, 3).join(', ')}`);
       git(tree, ['add', RULES_PATH, seenRel]);
-      const msg = `Sentinel improve: ${String(proposal.title || 'one guard change').replace(/\s+/g, ' ').slice(0, 72)}\n\nKept by the sentinel bench. Receipt: ${rel}\n\nCo-authored-by: Atris <299057014+atris-builder[bot]@users.noreply.github.com>`;
+      const msg = `Sentinel improve: one guard rules change\n\nKept by the sentinel bench. Receipt: ${rel}\n\nCo-authored-by: Atris <299057014+atris-builder[bot]@users.noreply.github.com>`;
       git(tree, ['commit', '-q', '-m', msg], { env });
       keepBranch = true;
       out.branch = branch;
       out.kept = true;
     }
   } catch (e) {
-    error = String(e.message || e).split('\n')[0].slice(0, 300);
+    error = fixedError(e);
   } finally {
     fs.rmSync(candidate, { recursive: true, force: true });
     // Kept, rejected or failed: the same safe check decides if the copy goes.
@@ -807,6 +897,31 @@ function schedulePlan({ root, engine = 'claude', everyHours = 24, env = process.
 }
 
 module.exports = {
+  SECRET_ITEM_ERROR,
+  SECRET_PROPOSAL_ERROR,
+  fixedError,
+  fixedReason,
+  gateValue,
+  strictSecret,
+  FOCI,
+  RULES_PATH,
+  SEEN_REL,
+  appendItems,
+  applyRules,
+  askEngine,
+  cleanItems,
+  dropCopy,
+  git,
+  homeDir,
+  missCounts,
+  okMap,
+  pgidOf,
+  repoTop,
+  runAlive,
+  runsDir,
+  stamp,
+  strayChanges,
+  totalsLine,
   buildPrompt,
   checkProposal,
   engineEnv,