atris 3.74.22 → 3.74.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/bin/atris.js +10 -1
- package/commands/improve.js +1 -1
- package/commands/mission.js +20 -4
- package/commands/rsi.js +25 -1
- package/commands/scorecard.js +297 -0
- package/commands/spend.js +78 -0
- package/lib/claude-spend-guard.js +288 -0
- package/lib/engine-ask.js +5 -2
- package/lib/engine-validate.js +2 -1
- package/lib/known-commands.js +2 -2
- package/lib/member-save-work.js +2 -1
- package/lib/repo-scorecard.js +1135 -0
- package/lib/roster-models.js +3 -1
- package/lib/rsi-loop.js +353 -0
- package/lib/runner-command.js +66 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -442,6 +442,23 @@ Under the hood, Atris can keep score on real repo work.
|
|
|
442
442
|
- `atris autopilot` can run that check after review and record the result in the journal.
|
|
443
443
|
- Future task picks can use recent results, so Atris learns from repo-local history without claiming model retraining.
|
|
444
444
|
|
|
445
|
+
## Which AI Can Do Your Work
|
|
446
|
+
|
|
447
|
+
`atris scorecard` measures which AI can do your team's work, on your own repo.
|
|
448
|
+
|
|
449
|
+
```bash
|
|
450
|
+
atris scorecard # find past changes whose own tests can grade an AI (no AI runs)
|
|
451
|
+
atris scorecard run # try each one with Haiku on your Claude plan
|
|
452
|
+
atris scorecard run --models haiku,opus # compare models; past Haiku needs ATRIS_CLAUDE_SPEND_OK=<model>
|
|
453
|
+
```
|
|
454
|
+
|
|
455
|
+
- A past change becomes a task when its tests fail before it and pass after it. The ask is what your team wrote for that change.
|
|
456
|
+
- The ask also names what the change's tests call that the change added or changed: a new file, a function with its arguments, an export, found with no AI. A good ticket names these; a commit message rarely does, and no model can guess a name nobody wrote down.
|
|
457
|
+
- Each try starts from the change's parent in a throwaway copy with no remote. The change's own tests stay hidden until they grade the try.
|
|
458
|
+
- Every try runs on your Claude plan login. Pay-per-use keys are removed, a try stops the moment Claude Code reports any other billing source, and nothing starts with extra usage on unless you pass `--allow-extra-usage`.
|
|
459
|
+
- The scorecard shows how many changes each model did, how long it took, what the same runs would cost on pay-per-use, and which changes no model could do from what was written.
|
|
460
|
+
- Tests that need their own command or settings go in `.atris/scorecard.json`.
|
|
461
|
+
|
|
445
462
|
## Benchmark Harness
|
|
446
463
|
|
|
447
464
|
Atris ships one public head-to-head benchmark harness for comparing a pinned
|
package/bin/atris.js
CHANGED
|
@@ -651,6 +651,7 @@ function showHelpAll() {
|
|
|
651
651
|
console.log(' meet - onboard a stranger in one sitting and print their /book link');
|
|
652
652
|
console.log(' avail - Booking availability (/book/{username} weekly windows)');
|
|
653
653
|
console.log(' scoreboard - Today\'s github contributions vs your best day, streak, repo commits (--watch, --json)');
|
|
654
|
+
console.log(' scorecard - Which AI can do your team\'s work: past changes graded by their own tests, run on your Claude plan (find, run, show)');
|
|
654
655
|
console.log(' brain - Compile MAP/TODO/wiki/state into a loadable agent brain');
|
|
655
656
|
console.log(' lesson - atris lesson add <slug> <pass|fail> "<text>" (mine/sweep/resolve/revert/ledger; --detector validates the falsifier at write time)');
|
|
656
657
|
console.log(' taste - Record the operator\'s keep, kill, and more creative verdicts');
|
|
@@ -669,7 +670,7 @@ function showHelpAll() {
|
|
|
669
670
|
console.log(' land - The landing: what is actually done vs still in the air; --reap backs up + clears overdue');
|
|
670
671
|
console.log(' caretaker - Classify open pull requests on origin (scan only; no fix, comment, or merge)');
|
|
671
672
|
console.log(' drive - One self-driving tick: mission doctor -> auto-fix -> count disengagements');
|
|
672
|
-
console.log(' rsi - Read the Dream-RSI attempt ledger
|
|
673
|
+
console.log(' rsi - Read the Dream-RSI attempt ledger; rsi loop ships notebook fixes with no mission');
|
|
673
674
|
console.log(` autoland - Approve the policy once; ${require('../lib/autoland').certifiedWorkLandsPhrase(process.cwd())}, you keep irreversible calls`);
|
|
674
675
|
console.log(' engine - engine registry, answer validation, dispatch flights, and live progress');
|
|
675
676
|
console.log(' link - What two things could become together: one line, why, a first thing to make, who makes it');
|
|
@@ -2234,6 +2235,10 @@ if (command === 'guide') {
|
|
|
2234
2235
|
Promise.resolve().then(() => require('../commands/scoreboard').scoreboardCommand(process.argv.slice(3)))
|
|
2235
2236
|
.then((code) => exitAfterStdoutDrain(typeof code === 'number' ? code : 0))
|
|
2236
2237
|
.catch((err) => { console.error(`\n✗ Error: ${err.message || err}`); process.exit(1); });
|
|
2238
|
+
} else if (command === 'scorecard') {
|
|
2239
|
+
Promise.resolve().then(() => require('../commands/scorecard').scorecardCommand(process.argv.slice(3)))
|
|
2240
|
+
.then((code) => exitAfterStdoutDrain(typeof code === 'number' ? code : 0))
|
|
2241
|
+
.catch((err) => { console.error(`\n✗ Error: ${err.message || err}`); process.exit(1); });
|
|
2237
2242
|
} else if (command === 'meet') {
|
|
2238
2243
|
Promise.resolve().then(() => require('../commands/meet').meetCommand(process.argv.slice(3)))
|
|
2239
2244
|
.then((code) => exitAfterStdoutDrain(typeof code === 'number' ? code : 0))
|
|
@@ -2495,6 +2500,10 @@ if (command === 'guide') {
|
|
|
2495
2500
|
Promise.resolve().then(() => require('../commands/balance').balanceCommand(process.argv.slice(3)))
|
|
2496
2501
|
.then((code) => exitAfterStdoutDrain(typeof code === 'number' ? code : 0))
|
|
2497
2502
|
.catch((err) => { console.error(String(err.message || err).replace(/\s+/g, ' ')); process.exit(1); });
|
|
2503
|
+
} else if (command === 'spend') {
|
|
2504
|
+
// Spend guard: unattended Claude runs on paid models need a yes (2026-10-28, $900 day).
|
|
2505
|
+
const code = require('../commands/spend').spendCommand(process.argv.slice(3));
|
|
2506
|
+
exitAfterStdoutDrain(typeof code === 'number' ? code : 0);
|
|
2498
2507
|
} else if (command === 'usage') {
|
|
2499
2508
|
Promise.resolve().then(() => require('../commands/usage').usageCommand(process.argv.slice(3)))
|
|
2500
2509
|
.then((code) => exitAfterStdoutDrain(typeof code === 'number' ? code : 0))
|
package/commands/improve.js
CHANGED
|
@@ -453,7 +453,7 @@ function collectImproveVitals(options = {}, deps = {}) {
|
|
|
453
453
|
return fs.existsSync(plistPath);
|
|
454
454
|
})();
|
|
455
455
|
const installed = launchdInstalled || legacyCronInstalled;
|
|
456
|
-
const installNudge = installed ? null : 'the scheduled improve loop is off. turn it on: atris pulse install
|
|
456
|
+
const installNudge = installed ? null : 'the scheduled improve loop is off. turn it on: atris pulse install';
|
|
457
457
|
|
|
458
458
|
const experimentsPath = path.join(root, '.atris', 'state', 'experiments-daily.json');
|
|
459
459
|
const experiments = readJsonFile(experimentsPath, {});
|
package/commands/mission.js
CHANGED
|
@@ -9360,7 +9360,7 @@ function buildTickPrompt(mission, tickIndex, maxTicks, frozen, pings = [], cwd =
|
|
|
9360
9360
|
`## Your task`,
|
|
9361
9361
|
`Do ONE increment of work toward the stop condition. ONE. No more.`,
|
|
9362
9362
|
`- ${memberIdentityLine(mission.owner)} Your member files are preloaded above; open atris/team/${mission.owner}/MEMBER.md only if that copy is marked truncated. Work in that identity, inside its scope and stop rules. Atris writes your member log from your receipt, so end the receipt with one \`learned: <one line>\` line instead of editing the log.`,
|
|
9363
|
-
`-
|
|
9363
|
+
`- Start with the job: inspect the files or state directly relevant to the objective, then do the smallest useful action. Skip workspace boot and start-of-session routines, including brain activation, general Atris or task listings, and reading MAP, persona, or skill files solely for startup. Inspect mission or task state only if the job requires it.`,
|
|
9364
9364
|
`- Pick the smallest concrete action that moves the mission forward.`,
|
|
9365
9365
|
`- Before acting, state your single next move in one sentence.`,
|
|
9366
9366
|
`- Edit / run / research as needed for the lane.`,
|
|
@@ -10082,6 +10082,16 @@ function engineHealthRoot(cwd) {
|
|
|
10082
10082
|
return dir && path.basename(dir) === '.git' ? path.dirname(dir) : cwd;
|
|
10083
10083
|
}
|
|
10084
10084
|
|
|
10085
|
+
function reviewerHealthRoot(engine) {
|
|
10086
|
+
return path.join(require('../lib/state-home').atrisHome(), 'reviewer-health', engine);
|
|
10087
|
+
}
|
|
10088
|
+
|
|
10089
|
+
function reviewerCooling(engine, registry) {
|
|
10090
|
+
const saved = registry.readEngineRegistry(reviewerHealthRoot(engine), { persist: false })
|
|
10091
|
+
.engines.find((entry) => entry.id === engine);
|
|
10092
|
+
return saved?.health?.status === 'cooling' && !registry.engineReadyAt(saved);
|
|
10093
|
+
}
|
|
10094
|
+
|
|
10085
10095
|
// Every spelling and version of one model family counts as the same model,
|
|
10086
10096
|
// with or without the long-context tag. Unknown counts as same.
|
|
10087
10097
|
function reviewerMayBeSameModel(a, b) {
|
|
@@ -10109,12 +10119,12 @@ function engineVerifierLineup(mission, cwd, preferredEngine = '', options = {})
|
|
|
10109
10119
|
const workerEngine = canonicalEngineName(mission?.runner) || 'claude';
|
|
10110
10120
|
const workerModel = resolveMissionTickRunnerModel(mission);
|
|
10111
10121
|
const team = registry.resolveEngineForRoleRanked('validator', root, { job: 'review', quiet: true }).team || [];
|
|
10112
|
-
const backup = team.find((engine) => engine && engine.id !== primary
|
|
10122
|
+
const backup = team.find((engine) => engine && engine.id !== primary && !reviewerCooling(engine.id, registry)
|
|
10113
10123
|
&& !(engine.id === workerEngine && reviewerMayBeSameModel(engine.roster_model, workerModel)));
|
|
10114
10124
|
if (!backup) return lineup;
|
|
10115
10125
|
lineup.push({ engine: backup.id, model: backup.roster_model || '' });
|
|
10116
10126
|
const saved = registry.readEngineRegistry(root, { persist: false }).engines.find((engine) => engine.id === primary);
|
|
10117
|
-
if (saved && !registry.engineReadyAt(saved)) lineup.shift();
|
|
10127
|
+
if ((saved && !registry.engineReadyAt(saved)) || reviewerCooling(primary, registry)) lineup.shift();
|
|
10118
10128
|
} catch {}
|
|
10119
10129
|
return lineup;
|
|
10120
10130
|
}
|
|
@@ -10134,7 +10144,7 @@ async function runEngineVerifier(mission, options = {}) {
|
|
|
10134
10144
|
sessionId: crypto.randomUUID(),
|
|
10135
10145
|
cwd,
|
|
10136
10146
|
signal: options.signal,
|
|
10137
|
-
timeoutMs: ENGINE_VERIFY_TIMEOUT_MS,
|
|
10147
|
+
timeoutMs: options.timeoutMs || ENGINE_VERIFY_TIMEOUT_MS,
|
|
10138
10148
|
prompt: buildEngineVerifyPrompt(mission, options.tickIndex || 1, missionDoneLooksLike(mission, cwd), {
|
|
10139
10149
|
timeoutMs: ENGINE_VERIFY_TIMEOUT_MS,
|
|
10140
10150
|
checkPassed: options.checkPassed || '',
|
|
@@ -10147,11 +10157,16 @@ async function runEngineVerifier(mission, options = {}) {
|
|
|
10147
10157
|
restoreRunnerProfile();
|
|
10148
10158
|
}
|
|
10149
10159
|
verdict = engineVerifierResultFromRun(engineResult, engine);
|
|
10160
|
+
if (engineResult?.ok && !engineResult?.timedOut && !engineResult?.aborted) {
|
|
10161
|
+
try { setEngineHealth(engine, 'ready', healthRoot); } catch {}
|
|
10162
|
+
try { setEngineHealth(engine, 'ready', reviewerHealthRoot(engine)); } catch {}
|
|
10163
|
+
}
|
|
10150
10164
|
if (verdict.mode !== 'engine-unavailable' || engineResult?.aborted) return verdict;
|
|
10151
10165
|
// Member lab 10/26: codex hung on every review, and each job waited out
|
|
10152
10166
|
// the full five minutes. Bench it so the next review starts on the backup.
|
|
10153
10167
|
if (engineResult?.timedOut) {
|
|
10154
10168
|
try { setEngineHealth(engine, 'cooling', healthRoot); } catch {}
|
|
10169
|
+
try { setEngineHealth(engine, 'cooling', reviewerHealthRoot(engine)); } catch {}
|
|
10155
10170
|
}
|
|
10156
10171
|
}
|
|
10157
10172
|
return verdict;
|
|
@@ -13007,6 +13022,7 @@ module.exports = {
|
|
|
13007
13022
|
receiptLearnedLine,
|
|
13008
13023
|
tickTouchedCheckDefinition,
|
|
13009
13024
|
engineVerifierLineup,
|
|
13025
|
+
runEngineVerifier,
|
|
13010
13026
|
claudeRunnerMcpArgs,
|
|
13011
13027
|
crystallizeMemberSkillLesson,
|
|
13012
13028
|
listSiblingWorkspaceMissionHints,
|
package/commands/rsi.js
CHANGED
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
// outcomes, the live policy id, and the last dream receipt. Read-only: it
|
|
7
7
|
// never touches record.py, never writes state, and exits 0 even when the
|
|
8
8
|
// workspace has no ledger yet.
|
|
9
|
+
//
|
|
10
|
+
// `atris rsi loop` ships notebook fixes with no mission queued (lib/rsi-loop.js).
|
|
9
11
|
|
|
10
12
|
const fs = require('fs');
|
|
11
13
|
const rsi = require('../lib/rsi-record');
|
|
@@ -114,8 +116,29 @@ Usage:
|
|
|
114
116
|
atris rsi same as status
|
|
115
117
|
atris rsi status trees + attempts by lane, last night's outcomes, policy, last dream
|
|
116
118
|
atris rsi status --json machine-readable status
|
|
119
|
+
atris rsi loop [--minutes M] [--json]
|
|
120
|
+
find notebook fixes with no mission queued, verify, commit and
|
|
121
|
+
push them on one reusable isolated branch (default budget 5 minutes)
|
|
122
|
+
|
|
123
|
+
status is read-only. loop checks atris/ pages for line refs that point at
|
|
124
|
+
moved code and for wording the slop rules can repair, commits each check
|
|
125
|
+
that verifies, and pushes the branch. It removes the local branch and checkout
|
|
126
|
+
only once the remote tip matches; the remote branch is reused on later runs.
|
|
127
|
+
The budget is checked between steps and limits network operations.
|
|
128
|
+
The main checkout is never edited.
|
|
129
|
+
|
|
130
|
+
State lives in .atris/state/rsi/ (or $ATRIS_RSI_STATE); loop reports go to loops.jsonl there.`);
|
|
131
|
+
}
|
|
117
132
|
|
|
118
|
-
|
|
133
|
+
async function runLoopCommand(args) {
|
|
134
|
+
const rsiLoop = require('../lib/rsi-loop');
|
|
135
|
+
const i = args.indexOf('--minutes');
|
|
136
|
+
const raw = i >= 0 ? String(args[i + 1] || '') : '';
|
|
137
|
+
const minutes = i < 0 ? undefined : (/^\d+(\.\d+)?$/.test(raw) ? Number(raw) : NaN);
|
|
138
|
+
const result = await rsiLoop.runLoop({ minutes, workspace: process.cwd() });
|
|
139
|
+
console.log(args.includes('--json') ? JSON.stringify(result) : rsiLoop.formatLoopReport(result));
|
|
140
|
+
if (result.refused) return 2;
|
|
141
|
+
return result.ok ? 0 : 1;
|
|
119
142
|
}
|
|
120
143
|
|
|
121
144
|
function run(argv = []) {
|
|
@@ -125,6 +148,7 @@ function run(argv = []) {
|
|
|
125
148
|
return 0;
|
|
126
149
|
}
|
|
127
150
|
const sub = args[0] && !args[0].startsWith('-') ? args[0] : 'status';
|
|
151
|
+
if (sub === 'loop') return runLoopCommand(args.slice(1));
|
|
128
152
|
if (sub !== 'status') {
|
|
129
153
|
console.log(`unknown rsi subcommand: ${sub}`);
|
|
130
154
|
showHelp();
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// atris scorecard: which AI can do your team's work, measured on your own
|
|
4
|
+
// past changes and graded by the tests that shipped with them. Finding tasks
|
|
5
|
+
// runs no AI. Tries run on your Claude plan login, never an API key.
|
|
6
|
+
|
|
7
|
+
const fs = require('fs');
|
|
8
|
+
const path = require('path');
|
|
9
|
+
const { hasFlag, readFlag, readIntFlag, readNumberFlag } = require('../lib/arg-parser');
|
|
10
|
+
const sc = require('../lib/repo-scorecard');
|
|
11
|
+
|
|
12
|
+
const USAGE = [
|
|
13
|
+
'usage: atris scorecard [find | run | show] [options]',
|
|
14
|
+
' which AI can do your team\'s work, measured on your own past changes.',
|
|
15
|
+
'',
|
|
16
|
+
' atris scorecard find tasks the first time, then show the scorecard',
|
|
17
|
+
' atris scorecard find turn past changes into tasks their own tests grade (no AI runs)',
|
|
18
|
+
' atris scorecard run try each task with each model on your Claude plan',
|
|
19
|
+
' atris scorecard show print the scorecard from saved results',
|
|
20
|
+
'',
|
|
21
|
+
'find options:',
|
|
22
|
+
' --max N keep at most N tasks (default 12)',
|
|
23
|
+
' --scan N look through the last N changes (default 300)',
|
|
24
|
+
' --config FILE test commands for this repo (default .atris/scorecard.json)',
|
|
25
|
+
' --with-env copy .env files into each copy, read-only, billing keys left out',
|
|
26
|
+
'',
|
|
27
|
+
'run options:',
|
|
28
|
+
' --models a,b models to try (default haiku). any model past haiku needs',
|
|
29
|
+
' ATRIS_CLAUDE_SPEND_OK=<model> because it uses much more of your plan',
|
|
30
|
+
' --tries N tries per task and model (default 1). a stopped run picks up where it left off',
|
|
31
|
+
' --jobs N tries at a time (default 2)',
|
|
32
|
+
' --minutes N time limit per try (default 10, 20 for opus and fable)',
|
|
33
|
+
' --task ID only this task (its id or the first characters of its change)',
|
|
34
|
+
' --allow-extra-usage run even when extra usage is on for your Claude account',
|
|
35
|
+
'',
|
|
36
|
+
' --json print the result as json',
|
|
37
|
+
].join('\n');
|
|
38
|
+
|
|
39
|
+
function storePaths(root) {
|
|
40
|
+
const dir = sc.storeDir(root);
|
|
41
|
+
return { dir, tasks: path.join(dir, 'tasks.json'), results: path.join(dir, 'results.jsonl') };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function readTasks(file) {
|
|
45
|
+
try {
|
|
46
|
+
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
47
|
+
} catch {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function readResults(file) {
|
|
53
|
+
try {
|
|
54
|
+
return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map((line) => {
|
|
55
|
+
try { return JSON.parse(line); } catch { return null; }
|
|
56
|
+
}).filter(Boolean);
|
|
57
|
+
} catch {
|
|
58
|
+
return [];
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function firstLine(text, width) {
|
|
63
|
+
const line = String(text || '').split('\n')[0];
|
|
64
|
+
return line.length > width ? `${line.slice(0, width - 3)}...` : line;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const SKIP_WORDS = {
|
|
68
|
+
no_tests: 'had no tests changed beside the code',
|
|
69
|
+
too_big: 'were too big for one ask',
|
|
70
|
+
short_ask: 'had too few words to work from',
|
|
71
|
+
unknown_runner: 'used a test setup this cannot run yet',
|
|
72
|
+
fails_after: 'had tests that fail even with the change',
|
|
73
|
+
passes_before: 'had tests that already passed before the change',
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
function skipSummary(skipped) {
|
|
77
|
+
return Object.entries(skipped || {}).filter(([, n]) => n > 0)
|
|
78
|
+
.map(([key, n]) => `${n} ${SKIP_WORDS[key] || key}`).join(', ');
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async function find(root, args, { quiet = false } = {}) {
|
|
82
|
+
const paths = storePaths(root);
|
|
83
|
+
const json = hasFlag(args, '--json');
|
|
84
|
+
const say = (line) => { if (!json && !quiet) console.log(line); };
|
|
85
|
+
const max = readIntFlag(args, '--max', sc.MAX_TASKS);
|
|
86
|
+
const scan = readIntFlag(args, '--scan', 300);
|
|
87
|
+
say(`atris scorecard ${sc.repoName(root)}`);
|
|
88
|
+
say(`looking through the last ${scan} changes for ones whose own tests can grade an AI. no AI runs in this step.`);
|
|
89
|
+
say('');
|
|
90
|
+
const scratch = sc.workDir(root);
|
|
91
|
+
let result;
|
|
92
|
+
try {
|
|
93
|
+
result = await sc.findTasks(root, {
|
|
94
|
+
max,
|
|
95
|
+
scan,
|
|
96
|
+
configPath: readFlag(args, '--config', '') || undefined,
|
|
97
|
+
withEnv: hasFlag(args, '--with-env'),
|
|
98
|
+
workDir: scratch,
|
|
99
|
+
onCheck: (task, verdict) => {
|
|
100
|
+
const word = verdict === 'found' ? 'task' : 'skip';
|
|
101
|
+
const why = verdict === 'found' ? sc.runnerLabel(task.runner) : SKIP_WORDS[verdict];
|
|
102
|
+
say(` ${word} ${task.fix.slice(0, 7)} ${firstLine(task.ask, 60).padEnd(60)} ${why}`);
|
|
103
|
+
},
|
|
104
|
+
});
|
|
105
|
+
} finally {
|
|
106
|
+
sc.stopAll();
|
|
107
|
+
fs.rmSync(scratch, { recursive: true, force: true });
|
|
108
|
+
}
|
|
109
|
+
const saved = {
|
|
110
|
+
repo: sc.repoName(root),
|
|
111
|
+
root,
|
|
112
|
+
found_at: new Date().toISOString(),
|
|
113
|
+
looked: result.looked,
|
|
114
|
+
candidates: result.candidates,
|
|
115
|
+
skipped: result.skipped,
|
|
116
|
+
tasks: result.tasks,
|
|
117
|
+
};
|
|
118
|
+
fs.mkdirSync(paths.dir, { recursive: true });
|
|
119
|
+
fs.writeFileSync(paths.tasks, `${JSON.stringify(saved, null, 2)}\n`);
|
|
120
|
+
if (json) {
|
|
121
|
+
console.log(JSON.stringify(saved, null, 2));
|
|
122
|
+
return 0;
|
|
123
|
+
}
|
|
124
|
+
if (!quiet) {
|
|
125
|
+
say('');
|
|
126
|
+
say(`found ${result.tasks.length} ${result.tasks.length === 1 ? 'task' : 'tasks'} in ${result.looked} past changes.`);
|
|
127
|
+
const skips = skipSummary(result.skipped);
|
|
128
|
+
if (skips) say(`the rest ${skips}.`);
|
|
129
|
+
if (!result.tasks.length) {
|
|
130
|
+
say('');
|
|
131
|
+
say('if your tests need a command or settings to run, put them in .atris/scorecard.json, for example:');
|
|
132
|
+
say(' {"tests": [{"match": "^backend/", "dir": "backend", "cmd": "python -m pytest -q {files}", "env": {"ENV": "test"}}]}');
|
|
133
|
+
return 1;
|
|
134
|
+
}
|
|
135
|
+
say('');
|
|
136
|
+
say('next: atris scorecard run');
|
|
137
|
+
say(` tries each task with ${sc.shortModel(sc.DEFAULT_MODELS[0])} on your Claude plan, up to 10 minutes each, 2 at a time.`);
|
|
138
|
+
}
|
|
139
|
+
return result.tasks.length ? 0 : 1;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function pickTasks(saved, args) {
|
|
143
|
+
const only = readFlag(args, '--task', '');
|
|
144
|
+
if (!only) return saved.tasks;
|
|
145
|
+
return saved.tasks.filter((task) => task.id === only || task.id.startsWith(only) || task.fix.startsWith(only));
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
async function run(root, args) {
|
|
149
|
+
const paths = storePaths(root);
|
|
150
|
+
const json = hasFlag(args, '--json');
|
|
151
|
+
const say = (line) => { if (!json) console.log(line); };
|
|
152
|
+
let saved = readTasks(paths.tasks);
|
|
153
|
+
if (!saved) {
|
|
154
|
+
say('no tasks yet, so finding them first.');
|
|
155
|
+
say('');
|
|
156
|
+
const code = await find(root, args.filter((a) => a !== '--json'), { quiet: json });
|
|
157
|
+
if (code !== 0) return code;
|
|
158
|
+
saved = readTasks(paths.tasks);
|
|
159
|
+
say('');
|
|
160
|
+
}
|
|
161
|
+
const tasks = pickTasks(saved, args);
|
|
162
|
+
if (!tasks.length) {
|
|
163
|
+
console.error(saved.tasks.length ? 'no task matches --task.' : 'there are no tasks to run. try atris scorecard find --scan 1000');
|
|
164
|
+
return 1;
|
|
165
|
+
}
|
|
166
|
+
const models = sc.normalizeModels(readFlag(args, '--models', ''));
|
|
167
|
+
let login;
|
|
168
|
+
try {
|
|
169
|
+
login = sc.preflight(models, { allowExtraUsage: hasFlag(args, '--allow-extra-usage') });
|
|
170
|
+
} catch (err) {
|
|
171
|
+
console.error(err.message || String(err));
|
|
172
|
+
return 1;
|
|
173
|
+
}
|
|
174
|
+
const tries = Math.max(1, readIntFlag(args, '--tries', 1) || 1);
|
|
175
|
+
const jobs = Math.max(1, readIntFlag(args, '--jobs', 2) || 2);
|
|
176
|
+
const minutes = readNumberFlag(args, '--minutes', null);
|
|
177
|
+
const before = readResults(paths.results);
|
|
178
|
+
const names = models.map(sc.shortModel).join(', ');
|
|
179
|
+
say(`atris scorecard run ${sc.repoName(root)}`);
|
|
180
|
+
say(`${tasks.length} ${tasks.length === 1 ? 'task' : 'tasks'} x ${names}, ${tries} ${tries === 1 ? 'try' : 'tries'} each, ${jobs} at a time.`);
|
|
181
|
+
say(`on your ${login.plan}. pay-per-use keys are removed from every run${login.extraUsage ? '' : ', so this bills $0'}.`);
|
|
182
|
+
say('');
|
|
183
|
+
|
|
184
|
+
const scratch = sc.workDir(root);
|
|
185
|
+
const cleanup = () => {
|
|
186
|
+
sc.stopAll();
|
|
187
|
+
fs.rmSync(scratch, { recursive: true, force: true });
|
|
188
|
+
};
|
|
189
|
+
const onSignal = () => {
|
|
190
|
+
cleanup();
|
|
191
|
+
console.error('\nstopped. finished tries are saved; run the same command to pick up where it left off.');
|
|
192
|
+
process.exit(130);
|
|
193
|
+
};
|
|
194
|
+
process.once('SIGINT', onSignal);
|
|
195
|
+
process.once('SIGTERM', onSignal);
|
|
196
|
+
let outcome;
|
|
197
|
+
try {
|
|
198
|
+
outcome = await sc.runAll(root, tasks, {
|
|
199
|
+
models,
|
|
200
|
+
tries,
|
|
201
|
+
jobs,
|
|
202
|
+
minutes,
|
|
203
|
+
bin: login.bin,
|
|
204
|
+
withEnv: hasFlag(args, '--with-env'),
|
|
205
|
+
done: before,
|
|
206
|
+
workDir: scratch,
|
|
207
|
+
outDir: paths.dir,
|
|
208
|
+
onResult: (row) => {
|
|
209
|
+
fs.appendFileSync(paths.results, `${JSON.stringify(row)}\n`);
|
|
210
|
+
const task = tasks.find((t) => t.id === row.id) || { fix: row.id, ask: '' };
|
|
211
|
+
let word = row.passed ? 'pass' : 'fail';
|
|
212
|
+
let note = row.timed_out ? `${row.limit_minutes} min (time limit)` : `${row.minutes} min`;
|
|
213
|
+
if (row.engine_error) { word = 'skip'; note = firstLine(row.engine_tail, 50) || 'claude did not finish'; }
|
|
214
|
+
if (row.harness_error) { word = 'skip'; note = firstLine(row.harness_error, 50); }
|
|
215
|
+
say(` ${word} ${sc.shortModel(row.model).padEnd(10)} ${task.fix.slice(0, 7)} ${firstLine(task.ask, 50).padEnd(50)} ${note}`);
|
|
216
|
+
},
|
|
217
|
+
});
|
|
218
|
+
} finally {
|
|
219
|
+
process.removeListener('SIGINT', onSignal);
|
|
220
|
+
process.removeListener('SIGTERM', onSignal);
|
|
221
|
+
cleanup();
|
|
222
|
+
}
|
|
223
|
+
const rows = readResults(paths.results);
|
|
224
|
+
const summary = sc.summarize(saved.tasks, rows, models);
|
|
225
|
+
if (json) {
|
|
226
|
+
console.log(JSON.stringify({ ...summary, stopped: outcome.stopped, ran: outcome.rows.length }, null, 2));
|
|
227
|
+
return outcome.stopped ? 1 : 0;
|
|
228
|
+
}
|
|
229
|
+
if (!outcome.planned) say('every try asked for is already done.');
|
|
230
|
+
say('');
|
|
231
|
+
console.log(sc.renderScorecard(summary, { repo: sc.repoName(root), plan: login.plan, extraUsage: login.extraUsage }));
|
|
232
|
+
if (outcome.stopped) {
|
|
233
|
+
const billed = outcome.rows.find((row) => row.not_plan);
|
|
234
|
+
say('');
|
|
235
|
+
say(billed
|
|
236
|
+
? `stopped: ${billed.engine_tail} log in to your Claude plan with \`claude\` then /login, and run it again.`
|
|
237
|
+
: 'stopped early: your Claude plan hit its usage limit. run the same command after it resets to finish.');
|
|
238
|
+
return 1;
|
|
239
|
+
}
|
|
240
|
+
return 0;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
function show(root, args) {
|
|
244
|
+
const paths = storePaths(root);
|
|
245
|
+
const saved = readTasks(paths.tasks);
|
|
246
|
+
if (!saved) {
|
|
247
|
+
console.error('no scorecard yet. run: atris scorecard find');
|
|
248
|
+
return 1;
|
|
249
|
+
}
|
|
250
|
+
const rows = readResults(paths.results);
|
|
251
|
+
const summary = sc.summarize(saved.tasks, rows);
|
|
252
|
+
if (hasFlag(args, '--json')) {
|
|
253
|
+
console.log(JSON.stringify(summary, null, 2));
|
|
254
|
+
return 0;
|
|
255
|
+
}
|
|
256
|
+
if (!rows.some((row) => saved.tasks.some((task) => task.id === row.id))) {
|
|
257
|
+
console.log(`atris scorecard ${saved.repo}`);
|
|
258
|
+
console.log(`${saved.tasks.length} tasks found, none tried yet:`);
|
|
259
|
+
for (const task of saved.tasks) console.log(` ${task.fix.slice(0, 7)} ${firstLine(task.ask, 80)}`);
|
|
260
|
+
console.log('');
|
|
261
|
+
console.log('next: atris scorecard run');
|
|
262
|
+
return 0;
|
|
263
|
+
}
|
|
264
|
+
const login = sc.planLogin();
|
|
265
|
+
console.log(sc.renderScorecard(summary, { repo: saved.repo, plan: login.ok ? login.plan : '', extraUsage: login.extraUsage }));
|
|
266
|
+
return 0;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
async function scorecardCommand(args = []) {
|
|
270
|
+
if (hasFlag(args, '--help') || hasFlag(args, '-h') || args[0] === 'help') {
|
|
271
|
+
console.log(USAGE);
|
|
272
|
+
return 0;
|
|
273
|
+
}
|
|
274
|
+
let root;
|
|
275
|
+
try {
|
|
276
|
+
root = sc.repoRoot();
|
|
277
|
+
} catch {
|
|
278
|
+
console.error('atris scorecard works inside a git repo. cd into one and try again.');
|
|
279
|
+
return 1;
|
|
280
|
+
}
|
|
281
|
+
const sub = args[0] && !args[0].startsWith('-') ? args[0] : '';
|
|
282
|
+
const rest = sub ? args.slice(1) : args;
|
|
283
|
+
if (sub === 'find') return find(root, rest);
|
|
284
|
+
if (sub === 'run') return run(root, rest);
|
|
285
|
+
if (sub === 'show') return show(root, rest);
|
|
286
|
+
if (sub) {
|
|
287
|
+
console.error(`unknown step "${sub}".\n\n${USAGE}`);
|
|
288
|
+
return 1;
|
|
289
|
+
}
|
|
290
|
+
if (!readTasks(storePaths(root).tasks)) {
|
|
291
|
+
const code = await find(root, rest);
|
|
292
|
+
return code;
|
|
293
|
+
}
|
|
294
|
+
return show(root, rest);
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
module.exports = { scorecardCommand, USAGE };
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const guard = require('../lib/claude-spend-guard');
|
|
5
|
+
|
|
6
|
+
function showSpendHelp() {
|
|
7
|
+
console.log('usage: atris spend guard [install|uninstall|status]');
|
|
8
|
+
console.log(' atris spend log [--limit N]');
|
|
9
|
+
console.log('');
|
|
10
|
+
console.log('The guard stops unattended Claude runs on paid models (anything but Haiku)');
|
|
11
|
+
console.log(`unless ${guard.APPROVAL_ENV}=<model> names the model for that run.`);
|
|
12
|
+
console.log('It also stops unattended runs that carry a pay-per-use Anthropic key.');
|
|
13
|
+
console.log('Interactive Claude sessions are never blocked.');
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function printStatus(status) {
|
|
17
|
+
if (!status.readable) {
|
|
18
|
+
console.log(`guard: unknown, could not read ${status.settings}`);
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
console.log(status.installed ? 'guard: on' : 'guard: off');
|
|
22
|
+
console.log(`settings: ${status.settings}`);
|
|
23
|
+
console.log(`hook: ${status.script}`);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function guardCommand(action = 'status') {
|
|
27
|
+
if (action === 'install') {
|
|
28
|
+
const status = guard.installGuard();
|
|
29
|
+
printStatus(status);
|
|
30
|
+
if (status.installed) {
|
|
31
|
+
console.log('Unattended Claude runs on paid models now need a yes on this computer.');
|
|
32
|
+
}
|
|
33
|
+
return status.installed ? 0 : 1;
|
|
34
|
+
}
|
|
35
|
+
if (action === 'uninstall') {
|
|
36
|
+
printStatus(guard.uninstallGuard());
|
|
37
|
+
return 0;
|
|
38
|
+
}
|
|
39
|
+
if (action === 'status') {
|
|
40
|
+
const status = guard.guardStatus();
|
|
41
|
+
printStatus(status);
|
|
42
|
+
return status.installed ? 0 : 1;
|
|
43
|
+
}
|
|
44
|
+
showSpendHelp();
|
|
45
|
+
return 1;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function logCommand(args) {
|
|
49
|
+
const i = args.indexOf('--limit');
|
|
50
|
+
const limit = i >= 0 ? Math.max(1, Number(args[i + 1]) || 20) : 20;
|
|
51
|
+
const file = guard.ledgerPath();
|
|
52
|
+
if (!fs.existsSync(file)) {
|
|
53
|
+
console.log('no guard decisions recorded on this computer.');
|
|
54
|
+
return 0;
|
|
55
|
+
}
|
|
56
|
+
const rows = fs.readFileSync(file, 'utf8').trim().split('\n').slice(-limit);
|
|
57
|
+
for (const line of rows) {
|
|
58
|
+
try {
|
|
59
|
+
const r = JSON.parse(line);
|
|
60
|
+
console.log(`${r.ts} ${r.decision} ${r.model} ${r.job || ''} ${r.cwd || ''}`.trimEnd());
|
|
61
|
+
} catch {}
|
|
62
|
+
}
|
|
63
|
+
return 0;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function spendCommand(args = []) {
|
|
67
|
+
const [sub, ...rest] = args;
|
|
68
|
+
if (!sub || sub === '--help' || sub === '-h' || sub === 'help') {
|
|
69
|
+
showSpendHelp();
|
|
70
|
+
return sub ? 0 : 1;
|
|
71
|
+
}
|
|
72
|
+
if (sub === 'guard') return guardCommand(rest[0] || 'status');
|
|
73
|
+
if (sub === 'log') return logCommand(rest);
|
|
74
|
+
showSpendHelp();
|
|
75
|
+
return 1;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
module.exports = { spendCommand };
|