atris 3.82.4 → 3.82.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/atris.js +3 -0
- package/commands/security-review.js +165 -17
- package/commands/sentinel-bench.js +172 -0
- package/commands/sentinel-fix.js +243 -0
- package/lib/security-audit.js +529 -13
- package/lib/sentinel-agents.js +75 -41
- package/lib/sentinel-bench.js +556 -0
- package/lib/sentinel-cloud.js +12 -6
- package/lib/sentinel-events.js +112 -1
- package/lib/sentinel-fix.js +777 -0
- package/lib/sentinel-guard-install.js +10 -4
- package/lib/sentinel-guard.js +168 -89
- package/lib/sentinel-improve.js +823 -0
- package/lib/sentinel-rules.js +108 -0
- package/lib/sentinel-rules.json +237 -0
- package/lib/sentinel-watch.js +12 -6
- package/package.json +1 -1
package/bin/atris.js
CHANGED
|
@@ -634,6 +634,9 @@ function showHelpAll() {
|
|
|
634
634
|
console.log(' slop - Deterministic slop detector: frontend/prose tells, dead code (slop dead --exports)');
|
|
635
635
|
console.log(' sentinel - Security audit: engines hunt for holes, a different model verifies each one');
|
|
636
636
|
console.log(' free checks: sentinel agents, sentinel watch, sentinel guard install');
|
|
637
|
+
console.log(' exam and self-improvement: sentinel bench, sentinel improve');
|
|
638
|
+
console.log(' fix a confirmed finding on its own branch: sentinel fix <id>');
|
|
639
|
+
console.log(' re-check old findings against today\'s code: sentinel recheck');
|
|
637
640
|
console.log(' voice - Scan binary voice tells or judge reply shape and plainness');
|
|
638
641
|
console.log(' recover - Explain stopped missions; --apply performs safe recovery only');
|
|
639
642
|
console.log(' task - Local agent task plane (atomic claims, TODO import)');
|
|
@@ -457,13 +457,15 @@ function renderAuditLanding({ dir, meta, records }) {
|
|
|
457
457
|
if (hunters < 1) L.push(' this budget is too small to send a hunter; try --budget 8 or more');
|
|
458
458
|
else L.push(needed <= hunters ? ' one run can read every area' : ` about ${Math.ceil(needed / hunters)} runs to read every area`);
|
|
459
459
|
} else {
|
|
460
|
-
|
|
460
|
+
const waiting = require('../lib/security-audit').waitingOf(meta);
|
|
461
|
+
L.push(` ${confirmed.length + waiting.length} confirmed${sevText ? ` (${sevText})` : ''} · ${needs.length} need a fact from you · ${rejected.length} disproved`);
|
|
461
462
|
L.push(` read ${cov.fresh} areas this run, ${cov.carried} carried unchanged, ${cov.deferred || 0} left for the next run`);
|
|
462
463
|
L.push(` found by ${meta.engines.hunter}, checked by ${meta.engines.verifier}${meta.engines.hunter === meta.engines.verifier ? ' (same engine)' : ''} · ${meta.invocations.used} engine calls of ${meta.budget}${meta.credits_used ? ` · ${meta.credits_used} credits used` : ''}`);
|
|
463
464
|
for (const r of confirmed.slice(0, 6)) {
|
|
464
465
|
const sink = r.trace[r.trace.length - 1];
|
|
465
466
|
L.push(` ${r.severity.overall.padEnd(8)} ${r.title} (${sink.file}:${sink.line})`);
|
|
466
467
|
}
|
|
468
|
+
for (const w of waiting.slice(0, 6)) L.push(` ${w.severity.padEnd(8)} ${w.title} (last seen at ${w.location}, still open: ${w.why})`);
|
|
467
469
|
if (meta.run_status === 'incomplete') L.push(` incomplete: ${String(meta.incomplete_reason).replace(/_/g, ' ')}`);
|
|
468
470
|
}
|
|
469
471
|
L.push(` report: ${tildePath(path.join(dir, 'REPORT.md'))}`, '');
|
|
@@ -478,11 +480,28 @@ function askYesNo(question, { input = process.stdin, output = process.stdout } =
|
|
|
478
480
|
});
|
|
479
481
|
}
|
|
480
482
|
|
|
481
|
-
async function uploadAfterRun(dir, { token, quiet, cloud }) {
|
|
483
|
+
async function uploadAfterRun(dir, { token, quiet, cloud, root = null, env = process.env }) {
|
|
482
484
|
const res = await cloud.uploadRun(dir, { token });
|
|
483
485
|
const say = quiet ? (m) => console.error(m) : (m) => console.log(m);
|
|
484
486
|
if (res.ok) say(' saved the summary to your atris account (no source code was sent)');
|
|
485
487
|
else say(` warning: could not save the summary to your atris account: ${res.reason}. the review itself is done`);
|
|
488
|
+
if (res.ok && res.body) await sendFixedEvents(dir, res.body, { token, root, env, say });
|
|
489
|
+
return res;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
// Fixed findings also go to the events feed. The overview counts them as
|
|
493
|
+
// fixed today. A failure here never undoes the saved review.
|
|
494
|
+
async function sendFixedEvents(dir, body, { token, root, env, say }) {
|
|
495
|
+
if (!body.findings.some((f) => f.status === 'fixed')) return null;
|
|
496
|
+
let meta = null;
|
|
497
|
+
try { meta = JSON.parse(fs.readFileSync(path.join(dir, 'run-metadata.json'), 'utf8')); } catch {}
|
|
498
|
+
const target = root || (meta && meta.target) || process.cwd();
|
|
499
|
+
let res;
|
|
500
|
+
try {
|
|
501
|
+
res = await require('../lib/sentinel-events').reportReviewFixes(target, body, { token, env, foundAt: meta && meta.prior_finished_at });
|
|
502
|
+
} catch (e) { res = { ok: false, reason: e.message }; }
|
|
503
|
+
if (res.ok && res.fixed) say(` marked ${res.fixed} finding${res.fixed === 1 ? '' : 's'} fixed on your dashboard`);
|
|
504
|
+
else if (!res.ok) say(` warning: could not mark the fixed findings on your dashboard: ${res.reason}. run atris sentinel upload to try again`);
|
|
486
505
|
return res;
|
|
487
506
|
}
|
|
488
507
|
|
|
@@ -502,6 +521,7 @@ async function auditUploadCommand(argv, deps) {
|
|
|
502
521
|
const res = await cloud.uploadRun(dir, { token });
|
|
503
522
|
if (!res.ok) { console.error(` could not save ${path.basename(dir)}: ${res.reason}`); return 1; }
|
|
504
523
|
console.log(` saved ${path.basename(dir)} to your atris account (no source code was sent)`);
|
|
524
|
+
if (res.body) await sendFixedEvents(dir, res.body, { token, root: null, env: deps.env || process.env, say: (m) => console.log(m) });
|
|
505
525
|
return 0;
|
|
506
526
|
}
|
|
507
527
|
|
|
@@ -598,6 +618,119 @@ async function auditScheduleCommand(argv, deps) {
|
|
|
598
618
|
return 0;
|
|
599
619
|
}
|
|
600
620
|
|
|
621
|
+
// The quote-and-confirm step every run on atris models goes through: price
|
|
622
|
+
// the estimate and set the cap. Then a yes, --yes or a scheduled approval.
|
|
623
|
+
function spendConfirmer({ cloud, token, spend, capOverride, opts, deps, tell }) {
|
|
624
|
+
return async (estimate) => {
|
|
625
|
+
const quote = await cloud.quoteEstimate(estimate, { token });
|
|
626
|
+
spend.max = capOverride || quote.credits;
|
|
627
|
+
spend.limits = quote.limits;
|
|
628
|
+
tell(`\n ${cloud.describeQuote(quote)}\n`);
|
|
629
|
+
if (capOverride) tell(` this scheduled run may spend at most ${capOverride} credits, as approved`);
|
|
630
|
+
if (opts.yes) return true;
|
|
631
|
+
const input = deps.input || process.stdin;
|
|
632
|
+
if (!input.isTTY && !deps.input) { console.error(' nothing was spent. add --yes to run without this question'); return false; }
|
|
633
|
+
const yes = await (deps.ask || askYesNo)(' spend these credits? (y/N) ', { input, output: opts.json ? process.stderr : process.stdout });
|
|
634
|
+
if (!yes) tell(' nothing was spent');
|
|
635
|
+
return yes;
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
function parseRecheckArgs(argv) {
|
|
640
|
+
const opts = { json: false, verifier: null, engine: null, yes: false, upload: true, timeoutMs: null, budget: null, target: null };
|
|
641
|
+
for (let i = 0; i < argv.length; i++) {
|
|
642
|
+
const arg = argv[i];
|
|
643
|
+
const [flag, inline] = arg.startsWith('--') && arg.includes('=') ? [arg.slice(0, arg.indexOf('=')), arg.slice(arg.indexOf('=') + 1)] : [arg, null];
|
|
644
|
+
const value = () => {
|
|
645
|
+
if (inline !== null) return inline;
|
|
646
|
+
if (!argv[i + 1] || argv[i + 1].startsWith('--')) throw new Error(`${flag} needs a value`);
|
|
647
|
+
return argv[++i];
|
|
648
|
+
};
|
|
649
|
+
if (flag === '--json') opts.json = true;
|
|
650
|
+
else if (flag === '--verifier') opts.verifier = value();
|
|
651
|
+
else if (flag === '--engine') opts.engine = value();
|
|
652
|
+
else if (flag === '--yes' || flag === '-y') opts.yes = true;
|
|
653
|
+
else if (flag === '--no-upload') opts.upload = false;
|
|
654
|
+
else if (flag === '--timeout') opts.timeoutMs = Number(value()) * 1000;
|
|
655
|
+
else if (flag === '--budget') opts.budget = Number(value());
|
|
656
|
+
else if (arg.startsWith('-')) throw new Error(`unknown flag: ${arg}`);
|
|
657
|
+
else if (opts.target) throw new Error('give one folder to re-check');
|
|
658
|
+
else opts.target = arg;
|
|
659
|
+
}
|
|
660
|
+
if (opts.engine !== null && !['atris', 'local'].includes(opts.engine)) throw new Error('--engine must be atris or local');
|
|
661
|
+
if (opts.budget !== null && (!Number.isInteger(opts.budget) || opts.budget < 1 || opts.budget > 500)) throw new Error('--budget must be a whole number from 1 to 500');
|
|
662
|
+
if (opts.timeoutMs !== null && (!Number.isFinite(opts.timeoutMs) || opts.timeoutMs < 10000)) throw new Error('--timeout is in seconds, at least 10');
|
|
663
|
+
return opts;
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
const OUTCOME_WORDS = { fixed: 'fixed', open: 'still open', moved: 'still open, moved', unsure: 'unsure, kept open', held: 'kept open' };
|
|
667
|
+
|
|
668
|
+
function renderRecheckLanding(result) {
|
|
669
|
+
const { meta, records, dir } = result;
|
|
670
|
+
const a = meta.audit;
|
|
671
|
+
const byFp = new Map(records.map((r) => [r.fingerprint, r]));
|
|
672
|
+
const L = ['', ` ◉ sentinel re-check · ${meta.repo}`, ''];
|
|
673
|
+
L.push(` re-checked ${a.checked} open finding${a.checked === 1 ? '' : 's'} from ${path.basename(meta.prior_run)}`);
|
|
674
|
+
L.push(` ${a.fixed} fixed · ${a.open} still open · ${a.unchanged} unchanged with no call`);
|
|
675
|
+
for (const n of a.notes) {
|
|
676
|
+
const r = byFp.get(n.fingerprint);
|
|
677
|
+
const held = (a.held || []).find((h) => h.fingerprint === n.fingerprint);
|
|
678
|
+
const title = r ? r.title : held ? held.title : n.fingerprint;
|
|
679
|
+
const sev = r && r.severity ? r.severity.overall : held ? held.severity : '';
|
|
680
|
+
L.push(` ${(OUTCOME_WORDS[n.outcome] || n.outcome).padEnd(18)} ${sev ? `${sev} ` : ''}${title}`);
|
|
681
|
+
L.push(` ${''.padEnd(18)} ${n.why}`);
|
|
682
|
+
}
|
|
683
|
+
L.push(` checked by ${meta.engines.verifier} · ${meta.invocations.used} engine call${meta.invocations.used === 1 ? '' : 's'}${meta.credits_used ? ` · ${meta.credits_used} credits used` : ''}`);
|
|
684
|
+
if (meta.run_status === 'incomplete') L.push(' some findings got no answer. they stay open and wait for the next run');
|
|
685
|
+
L.push(` report: ${tildePath(path.join(dir, 'REPORT.md'))}`, '');
|
|
686
|
+
return L.join('\n');
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
// atris sentinel recheck [path]: only the auditor, on the last review's open
|
|
690
|
+
// findings. Same engine rules, same price question and cap on atris models,
|
|
691
|
+
// same plain-sentence upload.
|
|
692
|
+
async function auditRecheckCommand(argv, deps = {}) {
|
|
693
|
+
const audit = require('../lib/security-audit');
|
|
694
|
+
const cloud = deps.cloud || require('../lib/sentinel-cloud');
|
|
695
|
+
let opts;
|
|
696
|
+
try { opts = parseRecheckArgs(argv); } catch (e) { console.error(`sentinel recheck: ${e.message}`); return 2; }
|
|
697
|
+
const root = path.resolve(opts.target || deps.root || process.cwd());
|
|
698
|
+
if (!fs.existsSync(root) || !fs.statSync(root).isDirectory()) { console.error(`sentinel recheck: ${root} is not a folder`); return 2; }
|
|
699
|
+
const token = deps.token !== undefined ? deps.token : cloud.authToken();
|
|
700
|
+
let engines;
|
|
701
|
+
try { engines = cloud.resolveEngines({ engine: opts.engine, verifier: opts.verifier, token, env: deps.env || process.env }); } catch (e) { console.error(`sentinel recheck: ${e.message}`); return 2; }
|
|
702
|
+
const verifier = engines.verifier || undefined;
|
|
703
|
+
const hosted = audit.isHostedEngine(verifier);
|
|
704
|
+
if (hosted && !token) { console.error('sentinel recheck: re-checks on atris models need a login; run atris login first'); return 2; }
|
|
705
|
+
const tell = opts.json ? (m) => console.error(m) : (m) => console.log(m);
|
|
706
|
+
if (hosted && engines.chosen === 'no_local_engine') tell(' no codex or claude found on this machine. this re-check runs on atris models');
|
|
707
|
+
const spend = hosted ? cloud.spendGuard({ quoteCall: deps.quoteCall }) : null;
|
|
708
|
+
let runJobs = deps.runJobs;
|
|
709
|
+
if (!runJobs && hosted) runJobs = cloud.atrisJobRunner({ token, spend });
|
|
710
|
+
const confirmSpend = hosted ? spendConfirmer({ cloud, token, spend, capOverride: null, opts, deps, tell }) : null;
|
|
711
|
+
let result;
|
|
712
|
+
try {
|
|
713
|
+
result = await audit.runRecheck({ root, verifier, runJobs, confirmSpend, budget: opts.budget ?? undefined, timeoutMs: opts.timeoutMs || undefined, log: opts.json ? () => {} : (m) => console.log(` · ${m}`) });
|
|
714
|
+
} catch (e) {
|
|
715
|
+
console.error(`sentinel recheck: ${e.message}`);
|
|
716
|
+
return 2;
|
|
717
|
+
}
|
|
718
|
+
if (result.nothing) {
|
|
719
|
+
if (opts.json) console.log(JSON.stringify({ ok: true, nothing: result.nothing }, null, 2));
|
|
720
|
+
else console.log(`\n ${result.nothing}. nothing to do\n`);
|
|
721
|
+
return 0;
|
|
722
|
+
}
|
|
723
|
+
if (result.declined) return 1;
|
|
724
|
+
const fail = audit.shouldFailRun(result.dir, result.records);
|
|
725
|
+
if (opts.json) console.log(JSON.stringify({ ok: !fail, dir: result.dir, meta: result.meta, findings: result.records }, null, 2));
|
|
726
|
+
else console.log(renderRecheckLanding(result));
|
|
727
|
+
if (spend && spend.blocked) tell(` stopped at the approved limit of ${spend.max} credits. ${spend.blocked} model call${spend.blocked === 1 ? '' : 's'} did not run. those findings stay open`);
|
|
728
|
+
if (opts.upload && token) {
|
|
729
|
+
try { await uploadAfterRun(result.dir, { token, quiet: opts.json, cloud, root, env: deps.env || process.env }); } catch (e) { tell(` warning: could not save the summary to your atris account: ${e.message}. the re-check itself is done`); }
|
|
730
|
+
}
|
|
731
|
+
return fail ? 1 : 0;
|
|
732
|
+
}
|
|
733
|
+
|
|
601
734
|
async function auditCommand(argv, deps = {}) {
|
|
602
735
|
if (argv[0] === 'help' || argv.includes('-h') || argv.includes('--help')) return printAuditHelp();
|
|
603
736
|
const audit = require('../lib/security-audit');
|
|
@@ -614,7 +747,13 @@ async function auditCommand(argv, deps = {}) {
|
|
|
614
747
|
const run = { agents: powers.agentsCommand, guard: powers.guardCommand, watch: powers.watchCommand, flush: powers.flushCommand }[argv[0]];
|
|
615
748
|
return run(argv.slice(1), deps);
|
|
616
749
|
}
|
|
750
|
+
if (argv[0] === 'bench' || argv[0] === 'improve') {
|
|
751
|
+
const sb = require('./sentinel-bench');
|
|
752
|
+
return (argv[0] === 'bench' ? sb.benchCommand : sb.improveCommand)(argv.slice(1), deps);
|
|
753
|
+
}
|
|
754
|
+
if (argv[0] === 'fix') return require('./sentinel-fix').fixCommand(argv.slice(1), deps);
|
|
617
755
|
if (argv[0] === 'upload') return auditUploadCommand(argv.slice(1), deps);
|
|
756
|
+
if (argv[0] === 'recheck') return auditRecheckCommand(argv.slice(1), deps);
|
|
618
757
|
if (argv[0] === 'schedule') return auditScheduleCommand(argv.slice(1), deps);
|
|
619
758
|
let opts;
|
|
620
759
|
try { opts = parseAuditArgs(argv); } catch (e) { console.error(`security-review audit: ${e.message}`); return 2; }
|
|
@@ -647,19 +786,7 @@ async function auditCommand(argv, deps = {}) {
|
|
|
647
786
|
let runJobs = deps.runJobs;
|
|
648
787
|
const spend = hosted ? cloud.spendGuard({ quoteCall: deps.quoteCall }) : null;
|
|
649
788
|
if (!runJobs && hosted) runJobs = cloud.atrisJobRunner({ token, spend });
|
|
650
|
-
const confirmSpend =
|
|
651
|
-
const quote = await cloud.quoteEstimate(estimate, { token });
|
|
652
|
-
spend.max = capOverride || quote.credits;
|
|
653
|
-
spend.limits = quote.limits;
|
|
654
|
-
tell(`\n ${cloud.describeQuote(quote)}\n`);
|
|
655
|
-
if (capOverride) tell(` this scheduled run may spend at most ${capOverride} credits, as approved`);
|
|
656
|
-
if (opts.yes) return true;
|
|
657
|
-
const input = deps.input || process.stdin;
|
|
658
|
-
if (!input.isTTY && !deps.input) { console.error(' nothing was spent. add --yes to run without this question'); return false; }
|
|
659
|
-
const yes = await (deps.ask || askYesNo)(' spend these credits? (y/N) ', { input, output: opts.json ? process.stderr : process.stdout });
|
|
660
|
-
if (!yes) tell(' nothing was spent');
|
|
661
|
-
return yes;
|
|
662
|
-
};
|
|
789
|
+
const confirmSpend = spendConfirmer({ cloud, token, spend, capOverride, opts, deps, tell });
|
|
663
790
|
let result;
|
|
664
791
|
try {
|
|
665
792
|
result = await audit.runAudit({
|
|
@@ -672,12 +799,12 @@ async function auditCommand(argv, deps = {}) {
|
|
|
672
799
|
return 2;
|
|
673
800
|
}
|
|
674
801
|
if (result.declined) return 1;
|
|
675
|
-
const fail = audit.
|
|
802
|
+
const fail = !opts.planOnly && audit.shouldFailRun(result.dir, result.records);
|
|
676
803
|
if (opts.json) console.log(JSON.stringify({ ok: !fail, dir: result.dir, meta: result.meta, findings: result.records }, null, 2));
|
|
677
804
|
else console.log(renderAuditLanding(result));
|
|
678
805
|
if (spend && spend.blocked) tell(` stopped at the approved limit of ${spend.max} credits. ${spend.blocked} model call${spend.blocked === 1 ? '' : 's'} did not run and wait for the next run`);
|
|
679
806
|
if (opts.upload && token && !opts.planOnly) {
|
|
680
|
-
try { await uploadAfterRun(result.dir, { token, quiet: opts.json, cloud }); } catch (e) { tell(` warning: could not save the summary to your atris account: ${e.message}. the review itself is done`); }
|
|
807
|
+
try { await uploadAfterRun(result.dir, { token, quiet: opts.json, cloud, root, env: deps.env || process.env }); } catch (e) { tell(` warning: could not save the summary to your atris account: ${e.message}. the review itself is done`); }
|
|
681
808
|
}
|
|
682
809
|
return fail ? 1 : 0;
|
|
683
810
|
}
|
|
@@ -714,6 +841,15 @@ function printAuditHelp() {
|
|
|
714
841
|
atris sentinel --no-upload keep this run on this machine only
|
|
715
842
|
atris sentinel upload [run-folder] save a past run (default: the last one)
|
|
716
843
|
|
|
844
|
+
re-checking old findings (every review does this first):
|
|
845
|
+
|
|
846
|
+
atris sentinel recheck [path] only re-check the last review's open findings
|
|
847
|
+
atris sentinel recheck --engine atris re-check on atris models, price shown first
|
|
848
|
+
|
|
849
|
+
a finding whose code did not change stays open without a call. a changed
|
|
850
|
+
one goes to the verifier, never the engine that found it. it is marked
|
|
851
|
+
fixed only with a yes and the line that now closes it. unsure stays open.
|
|
852
|
+
|
|
717
853
|
only the summary is saved: plain sentences, file:line locations and counts.
|
|
718
854
|
source code and code lines never leave in an upload.
|
|
719
855
|
|
|
@@ -729,6 +865,18 @@ function printAuditHelp() {
|
|
|
729
865
|
results go to your atris dashboard when you are logged in. add --no-upload
|
|
730
866
|
to keep them on this machine. secret values and file contents never leave.
|
|
731
867
|
|
|
868
|
+
the exam and the loop that improves the sentinel (in an atris checkout):
|
|
869
|
+
|
|
870
|
+
atris sentinel bench [--summary] score the guard and agent check on a seen and a held-out half
|
|
871
|
+
atris sentinel improve [--engine claude] one engine proposes a change, the exam decides if it stays
|
|
872
|
+
atris sentinel improve --schedule run the improve loop nightly
|
|
873
|
+
|
|
874
|
+
turning a confirmed finding into a fix:
|
|
875
|
+
|
|
876
|
+
atris sentinel fix list confirmed findings you can fix
|
|
877
|
+
atris sentinel fix <id> [path] smallest fix plus a regression test on a new branch
|
|
878
|
+
atris sentinel fix help how a fix is proved before it is committed
|
|
879
|
+
|
|
732
880
|
reviews on a timer:
|
|
733
881
|
|
|
734
882
|
atris sentinel schedule --every day [path] quick review once a day, saved to your account
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// atris sentinel bench and atris sentinel improve.
|
|
4
|
+
//
|
|
5
|
+
// atris sentinel bench [--summary] [--json] [--review] [--sets guard,agents]
|
|
6
|
+
// atris sentinel bench --split <file> --set guard|agents|review
|
|
7
|
+
// atris sentinel improve [--engine claude] [--offline] [--json]
|
|
8
|
+
// atris sentinel improve --schedule [--every 24] [--engine claude] | --unschedule
|
|
9
|
+
|
|
10
|
+
const path = require('path');
|
|
11
|
+
|
|
12
|
+
function readFlag(argv, name) {
|
|
13
|
+
const i = argv.indexOf(name);
|
|
14
|
+
if (i !== -1 && argv[i + 1] !== undefined && !String(argv[i + 1]).startsWith('--')) return argv[i + 1];
|
|
15
|
+
const inline = argv.find((a) => String(a).startsWith(`${name}=`));
|
|
16
|
+
return inline ? inline.slice(name.length + 1) : null;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function checkFlags(argv, flags, values) {
|
|
20
|
+
for (let i = 0; i < argv.length; i++) {
|
|
21
|
+
const a = String(argv[i]);
|
|
22
|
+
const name = a.split('=')[0];
|
|
23
|
+
if (values.includes(name)) { if (!a.includes('=')) i += 1; continue; }
|
|
24
|
+
if (flags.includes(a)) continue;
|
|
25
|
+
throw new Error(`unknown option: ${a}`);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function benchHelp() {
|
|
30
|
+
console.log(`
|
|
31
|
+
atris sentinel bench, a fixed exam for the sentinel
|
|
32
|
+
|
|
33
|
+
atris sentinel bench score the guard and the agent check
|
|
34
|
+
atris sentinel bench --summary totals only, no item lines
|
|
35
|
+
atris sentinel bench --review also run the review set on engines (costs engine time)
|
|
36
|
+
atris sentinel bench --json raw numbers for scripts
|
|
37
|
+
atris sentinel bench --split <file> --set guard|agents|review
|
|
38
|
+
split new items into the seen half (in the repo)
|
|
39
|
+
and the sealed held-out half (on this machine).
|
|
40
|
+
the plaintext input is replaced with an encrypted copy
|
|
41
|
+
|
|
42
|
+
the seen half lives in atris/sentinel-bench. the held-out half lives in
|
|
43
|
+
~/.atris/sentinel-holdout, encrypted with a key in your keychain. its
|
|
44
|
+
items are never printed. only lib/sentinel-rules.json of the checkout is
|
|
45
|
+
judged, always with this atris's own code. every number
|
|
46
|
+
is raw: attacks blocked, everyday commands wrongly blocked, problems
|
|
47
|
+
found, false alarms. there is no blended score.
|
|
48
|
+
`);
|
|
49
|
+
return 0;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async function benchCommand(argv, deps = {}) {
|
|
53
|
+
if (argv.includes('--help') || argv.includes('-h') || argv[0] === 'help') return benchHelp();
|
|
54
|
+
try { checkFlags(argv, ['--summary', '--json', '--review'], ['--split', '--set', '--sets', '--target', '--seen', '--hunter', '--verifier', '--budget']); } catch (e) { console.error(` ${e.message}`); return 2; }
|
|
55
|
+
const bench = require('../lib/sentinel-bench');
|
|
56
|
+
const env = deps.env || process.env;
|
|
57
|
+
const root = path.resolve(readFlag(argv, '--target') || deps.root || process.cwd());
|
|
58
|
+
const seenDir = path.resolve(readFlag(argv, '--seen') || bench.seenDirOf(root));
|
|
59
|
+
const holdoutDir = bench.defaultHoldoutDir(env);
|
|
60
|
+
const split = readFlag(argv, '--split');
|
|
61
|
+
if (split) {
|
|
62
|
+
const set = readFlag(argv, '--set');
|
|
63
|
+
if (!set) { console.error(' --split needs --set guard, agents or review'); return 2; }
|
|
64
|
+
let res;
|
|
65
|
+
try { res = bench.splitCorpus(path.resolve(split), set, { seenDir, holdoutDir, env }); } catch (e) { console.error(` could not split: ${e.message}`); return 1; }
|
|
66
|
+
if (argv.includes('--json')) { console.log(JSON.stringify({ ok: true, ...res }, null, 2)); return 0; }
|
|
67
|
+
console.log(`\n ${res.seen} items went to the seen half: ${path.relative(process.cwd(), res.seen_file) || res.seen_file}`);
|
|
68
|
+
console.log(` ${res.holdout} items went to the held-out half on this machine`);
|
|
69
|
+
if (res.skipped) console.log(` ${res.skipped} items were already in a half and were skipped`);
|
|
70
|
+
if (res.invalid) console.log(` ${res.invalid} lines were not valid items and were skipped`);
|
|
71
|
+
console.log(` the plaintext input was replaced with an encrypted copy at ${res.sealed_source}. no readable held-out item is left behind\n`);
|
|
72
|
+
return 0;
|
|
73
|
+
}
|
|
74
|
+
const review = argv.includes('--review');
|
|
75
|
+
const setsRaw = readFlag(argv, '--sets');
|
|
76
|
+
const sets = setsRaw ? setsRaw.split(',').map((s) => s.trim()).filter(Boolean) : ['guard', 'agents', ...(review ? ['review'] : [])];
|
|
77
|
+
const bad = sets.filter((s) => !bench.SETS.includes(s));
|
|
78
|
+
if (bad.length) { console.error(` unknown set: ${bad.join(', ')}. use guard, agents or review`); return 2; }
|
|
79
|
+
const budget = readFlag(argv, '--budget');
|
|
80
|
+
const reviewOpts = review ? {
|
|
81
|
+
hunter: readFlag(argv, '--hunter') || 'codex',
|
|
82
|
+
verifier: readFlag(argv, '--verifier') || 'claude',
|
|
83
|
+
...(budget ? { budget: Number(budget) } : {}),
|
|
84
|
+
log: argv.includes('--json') ? () => {} : (m) => console.error(` ${m}`),
|
|
85
|
+
...(deps.runJobs ? { runJobs: deps.runJobs } : {}),
|
|
86
|
+
} : null;
|
|
87
|
+
let report;
|
|
88
|
+
try { report = await bench.runBench({ target: root, seenDir, sets, review: reviewOpts, env }); } catch (e) { console.error(` the exam did not run: ${e.message}`); return 1; }
|
|
89
|
+
const pub = bench.publicReport(report, { summary: argv.includes('--summary') });
|
|
90
|
+
if (argv.includes('--json')) console.log(JSON.stringify({ ok: true, ...pub }, null, 2));
|
|
91
|
+
else console.log(bench.renderReport(pub, { holdoutDir: holdoutDir.replace(require('os').homedir(), '~') }));
|
|
92
|
+
return 0;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function improveHelp() {
|
|
96
|
+
console.log(`
|
|
97
|
+
atris sentinel improve, one exam-checked change to the sentinel
|
|
98
|
+
|
|
99
|
+
atris sentinel improve [--engine claude] propose one change and judge it
|
|
100
|
+
atris sentinel improve --offline skip the published advisory lookup
|
|
101
|
+
atris sentinel improve --schedule [--every 24] [--engine claude]
|
|
102
|
+
run it on a timer, nightly by default
|
|
103
|
+
atris sentinel improve --unschedule turn the timer off
|
|
104
|
+
|
|
105
|
+
claude proposes one change to lib/sentinel-rules.json on a new branch,
|
|
106
|
+
sealed off from the held-out exam. the sentinel bench, fixed code with no
|
|
107
|
+
model, scores it before and after. it is kept only when
|
|
108
|
+
held-out attack catch holds or rises, wrong blocks and false alarms do
|
|
109
|
+
not rise and no seen item that passed now fails. a kept change waits on
|
|
110
|
+
its branch for you. nothing is merged, pushed or released.
|
|
111
|
+
`);
|
|
112
|
+
return 0;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
async function improveCommand(argv, deps = {}) {
|
|
116
|
+
if (argv.includes('--help') || argv.includes('-h') || argv[0] === 'help') return improveHelp();
|
|
117
|
+
try { checkFlags(argv, ['--json', '--offline', '--schedule', '--unschedule'], ['--engine', '--every']); } catch (e) { console.error(` ${e.message}`); return 2; }
|
|
118
|
+
const improve = require('../lib/sentinel-improve');
|
|
119
|
+
const env = deps.env || process.env;
|
|
120
|
+
const engine = readFlag(argv, '--engine') || 'claude';
|
|
121
|
+
if (!/^[a-z][a-z0-9:_-]{0,30}$/.test(engine)) { console.error(` unknown engine: ${engine}`); return 2; }
|
|
122
|
+
const refused = require('../lib/sentinel-improve').refusedEngine(engine);
|
|
123
|
+
if (refused && !argv.includes('--unschedule')) { console.error(` ${refused}`); return 2; }
|
|
124
|
+
const root = deps.root || process.cwd();
|
|
125
|
+
if (argv.includes('--schedule') || argv.includes('--unschedule')) {
|
|
126
|
+
const { installKeep, removeKeep } = require('../lib/campaign-keep');
|
|
127
|
+
const everyHours = Number(readFlag(argv, '--every') || 24);
|
|
128
|
+
if (!Number.isFinite(everyHours) || everyHours <= 0) { console.error(' --every needs a number of hours'); return 2; }
|
|
129
|
+
let top;
|
|
130
|
+
try { top = require('child_process').execFileSync('git', ['rev-parse', '--show-toplevel'], { cwd: root, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }).trim(); } catch { console.error(' run this inside the atris checkout the loop should improve'); return 2; }
|
|
131
|
+
const plan = improve.schedulePlan({ root: top, engine, everyHours, env, atrisBin: deps.atrisBin });
|
|
132
|
+
if (argv.includes('--unschedule')) {
|
|
133
|
+
const removed = (deps.removeKeep || removeKeep)(plan);
|
|
134
|
+
if (!removed.ok) { console.error(` could not turn the timer off: ${removed.error}`); return 1; }
|
|
135
|
+
console.log(` sentinel improve timer: ${removed.removed === false ? 'was not on' : 'off'} for ${top}`);
|
|
136
|
+
return 0;
|
|
137
|
+
}
|
|
138
|
+
const space = require('../lib/worktree-space');
|
|
139
|
+
const bin = deps.atrisBin || path.join(__dirname, '..', 'bin', 'atris.js');
|
|
140
|
+
if (!deps.atrisBin && (space.isAgentCopyPath(bin) || space.isScratchPath(bin) || space.isAgentCopyPath(top))) {
|
|
141
|
+
console.error(' refusing: this atris or this checkout lives in a folder that gets removed. install the timer from a lasting checkout');
|
|
142
|
+
return 2;
|
|
143
|
+
}
|
|
144
|
+
const installed = (deps.installKeep || installKeep)(plan);
|
|
145
|
+
if (!installed.ok) { console.error(` could not turn the timer on: ${installed.error}`); return 1; }
|
|
146
|
+
console.log(` sentinel improve timer: on for ${top}, every ${everyHours} ${everyHours === 1 ? 'hour' : 'hours'} with ${engine}`);
|
|
147
|
+
console.log(` log: ${plan.log_path}`);
|
|
148
|
+
console.log(' each run leaves a receipt in ~/.atris/sentinel-improve/receipts. kept changes wait on their branch for you');
|
|
149
|
+
return 0;
|
|
150
|
+
}
|
|
151
|
+
let res;
|
|
152
|
+
try {
|
|
153
|
+
res = await improve.improve({ root, engine, env, offline: argv.includes('--offline'), log: argv.includes('--json') ? () => {} : (m) => console.log(` ${m}`), ...(deps.improveOpts || {}) });
|
|
154
|
+
} catch (e) { console.error(` improve did not run: ${e.message}`); return 1; }
|
|
155
|
+
if (argv.includes('--json')) {
|
|
156
|
+
console.log(JSON.stringify({ ok: !res.error, kept: res.kept, branch: res.branch, error: res.error || null, decision: res.decision, receipt: res.receipt }, null, 2));
|
|
157
|
+
return res.error ? 1 : 0;
|
|
158
|
+
}
|
|
159
|
+
console.log('');
|
|
160
|
+
if (res.error) console.log(` stopped: ${res.error}`);
|
|
161
|
+
else if (res.kept) console.log(` kept. the change waits on branch ${res.branch} for your review`);
|
|
162
|
+
else console.log(' rejected. the branch was removed');
|
|
163
|
+
if (res.decision) {
|
|
164
|
+
for (const g of res.decision.gains) console.log(` gain: ${g}`);
|
|
165
|
+
for (const n of res.decision.notes || []) console.log(` note: ${n}`);
|
|
166
|
+
for (const r of res.decision.reasons) console.log(` stop: ${r}`);
|
|
167
|
+
}
|
|
168
|
+
console.log(` receipt: ${res.receipt.replace(require('os').homedir(), '~')}\n`);
|
|
169
|
+
return res.error ? 1 : 0;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
module.exports = { benchCommand, improveCommand };
|