ai-agentmanager 1.8.0 → 1.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/README.md +8 -4
- package/package.json +1 -1
- package/public/app.js +81 -17
- package/public/export.js +110 -0
- package/public/index.html +2 -0
- package/public/land.js +2 -2
- package/public/review.js +67 -5
- package/public/scorecard.js +68 -0
- package/public/share.js +1 -1
- package/public/side.js +47 -9
- package/public/speak.js +171 -0
- package/public/style.css +40 -0
- package/public/walkthrough.js +16 -0
- package/src/brief.js +79 -0
- package/src/cli.js +2 -2
- package/src/manager.js +77 -1
- package/src/review.js +42 -2
- package/src/scorecard.js +189 -0
- package/src/server.js +2 -0
package/src/manager.js
CHANGED
|
@@ -29,6 +29,12 @@ import {
|
|
|
29
29
|
parseJsonReply,
|
|
30
30
|
loadPlanFile,
|
|
31
31
|
planApprovedPrompt,
|
|
32
|
+
intentMarkdown,
|
|
33
|
+
intentPath,
|
|
34
|
+
readSpecDoc,
|
|
35
|
+
specDocsInfo,
|
|
36
|
+
specPath,
|
|
37
|
+
specPrompt,
|
|
32
38
|
planPath,
|
|
33
39
|
planReviewPrompt,
|
|
34
40
|
} from './brief.js';
|
|
@@ -41,6 +47,7 @@ const SUMMARIES = {
|
|
|
41
47
|
humanai: { doc: 'humanai', run: 'runHumanAi' },
|
|
42
48
|
diagrams: { doc: 'diagrams', run: 'runDiagrams' },
|
|
43
49
|
walkthrough: { doc: 'walkthrough', run: 'runWalkthrough' },
|
|
50
|
+
scorecard: { doc: 'scorecard', run: 'runScorecard' },
|
|
44
51
|
};
|
|
45
52
|
/** Read / write where a summary of `kind` is kept on the agent record. */
|
|
46
53
|
const getSummary = (agent, kind) => (SUMMARIES[kind].doc ? agent.docs?.[kind] : agent[SUMMARIES[kind].field]);
|
|
@@ -94,6 +101,7 @@ import { spawn } from 'node:child_process';
|
|
|
94
101
|
import * as git from './git.js';
|
|
95
102
|
import { parseDiff } from '../public/review.js';
|
|
96
103
|
import { issueSummary, leadingIssue, wordsAfterLink } from './jira.js';
|
|
104
|
+
import { combine, facts as scoreFacts, scorecardMarkdown, scorecardPrompt } from './scorecard.js';
|
|
97
105
|
import { conflictMessage, findConflicts, related, repoKey, summarize as summarizeConflicts } from './conflicts.js';
|
|
98
106
|
import { answerMessage, loadQuestions, questionsPath, questionsPrompt } from './inbox.js';
|
|
99
107
|
import { STUCK_DEFAULTS, detectStuck } from './stuck.js';
|
|
@@ -336,6 +344,8 @@ export class AgentManager extends EventEmitter {
|
|
|
336
344
|
const prs = Object.values(agent.prs ?? {});
|
|
337
345
|
return {
|
|
338
346
|
...agent,
|
|
347
|
+
// Intent, spec and plan files that exist, with when each changed (task agents).
|
|
348
|
+
...(NO_REPO.has(agent.kind) ? {} : { specDocs: specDocsInfo(agent.id) }),
|
|
339
349
|
results: this.results(agent, rt, prs),
|
|
340
350
|
priority: agent.priority ?? 2,
|
|
341
351
|
activeAt: this.activeAt(agent, rt),
|
|
@@ -675,6 +685,7 @@ export class AgentManager extends EventEmitter {
|
|
|
675
685
|
this.store.put(agent);
|
|
676
686
|
this.registerProject(agent);
|
|
677
687
|
this.changed(agent);
|
|
688
|
+
this.writeIntent(agent.id);
|
|
678
689
|
if (issue) this.fetchSynopsis(agent.id).catch(() => {});
|
|
679
690
|
|
|
680
691
|
this.plugins.emit('agent.created', { agent: this.auditView(agent) });
|
|
@@ -701,9 +712,27 @@ export class AgentManager extends EventEmitter {
|
|
|
701
712
|
synopsis: { text: summary, source: 'jira', key: issue.key },
|
|
702
713
|
...(bug && !bug.title ? { bugs: { ...cur.bugs, [issue.key]: { ...bug, title: summary } } } : {}),
|
|
703
714
|
});
|
|
715
|
+
this.writeIntent(cur.id);
|
|
704
716
|
return summary;
|
|
705
717
|
}
|
|
706
718
|
|
|
719
|
+
/** INTENT.md for a task agent: the request as given and where it came from (see brief.js). */
|
|
720
|
+
writeIntent(id) {
|
|
721
|
+
const agent = this.store.get(id);
|
|
722
|
+
if (!agent || NO_REPO.has(agent.kind)) return;
|
|
723
|
+
try {
|
|
724
|
+
fs.mkdirSync(briefDir(id), { recursive: true });
|
|
725
|
+
fs.writeFileSync(intentPath(id), intentMarkdown(agent));
|
|
726
|
+
} catch {}
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
/** One of an agent's intent, spec and plan files. */
|
|
730
|
+
specDoc(ref, name) {
|
|
731
|
+
const agent = this.mustGet(ref);
|
|
732
|
+
if (!['intent', 'spec', 'plan'].includes(name)) throw Object.assign(new Error('Unknown file (intent, spec or plan)'), { status: 400 });
|
|
733
|
+
return readSpecDoc(agent.id, name) ?? { name, markdown: null };
|
|
734
|
+
}
|
|
735
|
+
|
|
707
736
|
/** Fill in summaries still missing (after Jira settings change); returns how many it found. */
|
|
708
737
|
async refreshSynopses() {
|
|
709
738
|
let found = 0;
|
|
@@ -983,6 +1012,13 @@ export class AgentManager extends EventEmitter {
|
|
|
983
1012
|
prompts.push(briefPrompt(path.join(dir, path.basename(briefPath(agent.id)))));
|
|
984
1013
|
if (this.config.inbox !== false) prompts.push(questionsPrompt(path.join(dir, path.basename(questionsPath(agent.id)))));
|
|
985
1014
|
}
|
|
1015
|
+
// SPEC.md before changing code, and PLAN.md kept as it goes (unless the plan review asks for the plan).
|
|
1016
|
+
if (task && this.config.specs !== false) {
|
|
1017
|
+
fs.mkdirSync(briefDir(agent.id), { recursive: true });
|
|
1018
|
+
const dir = fs.realpathSync(briefDir(agent.id));
|
|
1019
|
+
writableDir ??= dir;
|
|
1020
|
+
prompts.push(specPrompt(path.join(dir, path.basename(specPath(agent.id))), path.join(dir, path.basename(planPath(agent.id))), { withPlan: !agent.planReview }));
|
|
1021
|
+
}
|
|
986
1022
|
if (agent.planReview && task) {
|
|
987
1023
|
fs.mkdirSync(briefDir(agent.id), { recursive: true });
|
|
988
1024
|
const file = path.join(fs.realpathSync(briefDir(agent.id)), path.basename(planPath(agent.id)));
|
|
@@ -1853,6 +1889,8 @@ export class AgentManager extends EventEmitter {
|
|
|
1853
1889
|
this.patch(id, { brief });
|
|
1854
1890
|
// The agent says it is done: have the judge check the work before the person looks.
|
|
1855
1891
|
if (brief.status === 'done' && agent.brief?.status !== 'done' && this.judge.enabled) this.requestReview(id, { trigger: 'brief' }).catch(() => {});
|
|
1892
|
+
// ...and score it for the reviewer ("scorecard": { "auto": false } in config.json turns this off).
|
|
1893
|
+
if (brief.status === 'done' && agent.brief?.status !== 'done' && this.config.scorecard?.auto !== false && !NO_REPO.has(agent.kind) && this.changesSource) this.summarize(id, 'scorecard').catch(() => {});
|
|
1856
1894
|
}
|
|
1857
1895
|
if (agent.planReview) {
|
|
1858
1896
|
const file = loadPlanFile(id, agent.planReview);
|
|
@@ -2008,7 +2046,9 @@ export class AgentManager extends EventEmitter {
|
|
|
2008
2046
|
.join('\n\n');
|
|
2009
2047
|
}
|
|
2010
2048
|
}
|
|
2011
|
-
|
|
2049
|
+
// The spec and plan go along with the brief: what done means, and how it gets there.
|
|
2050
|
+
const extra = ['spec', 'plan'].map((n) => readSpecDoc(agent.id, n)?.markdown.trim()).filter(Boolean);
|
|
2051
|
+
return { intent: agent.intent, status, digest, repo, briefMarkdown: [briefMarkdown, ...extra].filter(Boolean).join('\n\n') };
|
|
2012
2052
|
}
|
|
2013
2053
|
|
|
2014
2054
|
/** The engine (and model) that writes an agent's summaries: its own when it can make a one-shot call, else Claude Code. */
|
|
@@ -2100,6 +2140,42 @@ export class AgentManager extends EventEmitter {
|
|
|
2100
2140
|
);
|
|
2101
2141
|
}
|
|
2102
2142
|
|
|
2143
|
+
/**
|
|
2144
|
+
* The review scorecard (see scorecard.js): the same diff the review screen shows, scored against
|
|
2145
|
+
* the intent, spec and plan, with the facts the app measures capping what the model says.
|
|
2146
|
+
*/
|
|
2147
|
+
async runScorecard(agent, ctx) {
|
|
2148
|
+
if (!this.changesSource) throw new Error('Changes are not available here');
|
|
2149
|
+
const data = await this.changesSource(agent.id);
|
|
2150
|
+
const { hunks, cut } = walkthroughHunks(parseDiff(data.diff), { maxChars: 50_000 });
|
|
2151
|
+
if (!hunks.length) throw Object.assign(new Error('No changes to score yet'), { status: 400 });
|
|
2152
|
+
const cur0 = this.store.get(agent.id) ?? agent;
|
|
2153
|
+
const doc = (n) => readSpecDoc(agent.id, n)?.markdown ?? '';
|
|
2154
|
+
const f = scoreFacts({ files: data.files, prs: Object.values(cur0.prs ?? {}), brief: cur0.brief, plan: doc('plan'), spec: doc('spec') });
|
|
2155
|
+
const writer = this.writerFor(agent);
|
|
2156
|
+
const prompt = scorecardPrompt({ ...ctx, briefMarkdown: this.briefFile(agent.id).markdown ?? '', intentDoc: doc('intent'), specDoc: doc('spec'), planDoc: doc('plan'), hunks, facts: f });
|
|
2157
|
+
const sc = combine(parseJsonReply(await this.oneShot(agent, prompt, writer)), f, hunks);
|
|
2158
|
+
const cur = this.store.get(agent.id);
|
|
2159
|
+
if (!cur) return null;
|
|
2160
|
+
return this.view(
|
|
2161
|
+
this.patch(
|
|
2162
|
+
agent.id,
|
|
2163
|
+
summaryPatch(cur, 'scorecard', {
|
|
2164
|
+
data: sc,
|
|
2165
|
+
cut,
|
|
2166
|
+
diffHash: hash(data.diff),
|
|
2167
|
+
markdown: scorecardMarkdown(sc, cur.name),
|
|
2168
|
+
writer: `${writer.engine.label}${writer.model ? ` (${writer.model})` : ''}`,
|
|
2169
|
+
postedTo: cur.docs?.scorecard?.postedTo ?? null,
|
|
2170
|
+
at: now(),
|
|
2171
|
+
briefUpdatedAt: cur.brief?.updatedAt ?? null,
|
|
2172
|
+
pending: false,
|
|
2173
|
+
error: null,
|
|
2174
|
+
}),
|
|
2175
|
+
),
|
|
2176
|
+
);
|
|
2177
|
+
}
|
|
2178
|
+
|
|
2103
2179
|
/** Diagrams only: the problem and its solution drawn from the documents already written and the live context. */
|
|
2104
2180
|
async runDiagrams(agent, ctx) {
|
|
2105
2181
|
const cur0 = this.store.get(agent.id) ?? agent;
|
package/src/review.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// back to the agent as one message, and commit / push / open a pull request from the UI.
|
|
3
3
|
import * as git from './git.js';
|
|
4
4
|
import { hash } from './judge.js';
|
|
5
|
+
import { prSections } from './brief.js';
|
|
5
6
|
import { NO_REPO } from './manager.js';
|
|
6
7
|
import { ghBin } from './tracker.js';
|
|
7
8
|
|
|
@@ -142,6 +143,32 @@ export class Review {
|
|
|
142
143
|
};
|
|
143
144
|
}
|
|
144
145
|
|
|
146
|
+
/**
|
|
147
|
+
* Post the review scorecard on the agent's open pull request as a comment, so approvers see it in
|
|
148
|
+
* GitHub. Posting again edits the last comment instead of adding another.
|
|
149
|
+
*/
|
|
150
|
+
async postScorecard(ref) {
|
|
151
|
+
const agent = this.agentWithRepo(ref);
|
|
152
|
+
const sc = agent.docs?.scorecard;
|
|
153
|
+
if (!sc?.markdown) throw err(409, 'Score the changes first');
|
|
154
|
+
const pr = Object.values(agent.prs ?? {}).find((p) => p.state === 'OPEN');
|
|
155
|
+
if (!pr) throw err(409, 'No open pull request for this agent');
|
|
156
|
+
this.manager.plugins.check('agent.pr', { agent: this.manager.auditView(agent), title: 'scorecard comment', draft: false });
|
|
157
|
+
const opts = { cwd: agent.workdir, timeout: 60_000 };
|
|
158
|
+
const args = ['pr', 'comment', pr.url, '--body', sc.markdown];
|
|
159
|
+
const again = sc.postedTo === pr.url;
|
|
160
|
+
try {
|
|
161
|
+
await git.run(ghBin(), again ? [...args, '--edit-last'] : args, opts);
|
|
162
|
+
} catch (e) {
|
|
163
|
+
// The comment we edited may be gone; add a new one.
|
|
164
|
+
if (!again) throw err(502, `Posting the scorecard failed: ${e.message.replace(/^gh .*? failed: /, '')}`);
|
|
165
|
+
await git.run(ghBin(), args, opts);
|
|
166
|
+
}
|
|
167
|
+
const cur = this.manager.store.get(agent.id);
|
|
168
|
+
this.manager.patch(agent.id, { docs: { ...cur.docs, scorecard: { ...cur.docs.scorecard, postedTo: pr.url, postedAt: new Date().toISOString() } } });
|
|
169
|
+
return { url: pr.url, edited: again };
|
|
170
|
+
}
|
|
171
|
+
|
|
145
172
|
/** Send the comments to the agent as one message (typed in when it is idle). */
|
|
146
173
|
comment(ref, { comments = [], note = '' } = {}) {
|
|
147
174
|
const agent = this.agentWithRepo(ref);
|
|
@@ -193,8 +220,21 @@ export class Review {
|
|
|
193
220
|
}
|
|
194
221
|
const args = ['pr', 'create', '--head', branch];
|
|
195
222
|
if (target) args.push('--base', target);
|
|
196
|
-
|
|
197
|
-
|
|
223
|
+
// The description: your text (else the commits), the review scorecard, then the intent, spec and plan.
|
|
224
|
+
const sc = agent.docs?.scorecard;
|
|
225
|
+
let card = '';
|
|
226
|
+
if (sc?.markdown) {
|
|
227
|
+
const now = base ? hash(await git.fullDiff(dir, base.sha)) : null;
|
|
228
|
+
card = sc.diffHash && now && sc.diffHash !== now ? `${sc.markdown}\n\n<sub>Scored before the latest changes.</sub>` : sc.markdown;
|
|
229
|
+
}
|
|
230
|
+
const sections = [card, prSections(agent.id)].filter(Boolean).join('\n\n');
|
|
231
|
+
if (String(title).trim() || sections) {
|
|
232
|
+
const log = base ? await git.run('git', ['log', '--reverse', '--format=%s', `${base.sha}..HEAD`], opts).catch(() => '') : '';
|
|
233
|
+
const subjects = log.split('\n').filter(Boolean);
|
|
234
|
+
const t = String(title).trim() || subjects[0] || agent.name;
|
|
235
|
+
const lead = String(body ?? '').trim() || (subjects.length > 1 ? subjects.map((x) => `- ${x}`).join('\n') : '');
|
|
236
|
+
args.push('--title', t, '--body', [lead, sections].filter(Boolean).join('\n\n'));
|
|
237
|
+
} else args.push('--fill');
|
|
198
238
|
if (draft) args.push('--draft');
|
|
199
239
|
let out;
|
|
200
240
|
try {
|
package/src/scorecard.js
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
// The review scorecard: an agent's changes scored against what was asked (intent), what done means
|
|
2
|
+
// (spec), staying on task (scope), its plan, the tests and safety, with an overall readiness and the
|
|
3
|
+
// few places a person should actually read. A model judges; facts the app measures (CI, test files,
|
|
4
|
+
// plan ticks, the spec's criteria) cap and correct it, and the recommendation follows fixed rules,
|
|
5
|
+
// so a confident-sounding model can't talk a change into "ready".
|
|
6
|
+
|
|
7
|
+
export const DIMENSIONS = [
|
|
8
|
+
['intent', 'Intent', 'Does it do what was asked?'],
|
|
9
|
+
['spec', 'Spec', 'Are the acceptance criteria met?'],
|
|
10
|
+
['tests', 'Tests', 'Is it verified: CI, tests for the changed code, checks run?'],
|
|
11
|
+
['safety', 'Safety', 'Risky areas handled: security, data, migrations, dependencies, public APIs (higher is safer)'],
|
|
12
|
+
['scope', 'Scope', 'Only what the task needs, in proportion?'],
|
|
13
|
+
['plan', 'Plan', 'Done as planned, departures explained?'],
|
|
14
|
+
];
|
|
15
|
+
const WEIGHTS = { intent: 25, spec: 20, tests: 20, safety: 15, scope: 10, plan: 10 };
|
|
16
|
+
const TEST_FILE = /(^|\/)(tests?|__tests__|spec|specs)\/|[._-](test|spec)\.[a-z]+$|_test\.go$|Test\.java$/i;
|
|
17
|
+
|
|
18
|
+
/** Facts the app knows without a model: CI, test files touched, the agent's checks, plan and spec progress. */
|
|
19
|
+
export function facts({ files = [], prs = [], brief = null, plan = '', spec = '' }) {
|
|
20
|
+
const paths = files.map((f) => f.path);
|
|
21
|
+
const tests = paths.filter((p) => TEST_FILE.test(p));
|
|
22
|
+
const code = paths.filter((p) => !TEST_FILE.test(p) && !/\.(md|txt|rst|adoc)$/i.test(p) && !/(^|\/)docs?\//i.test(p));
|
|
23
|
+
const open = prs.find((p) => p.state === 'OPEN') ?? prs[0];
|
|
24
|
+
const checks = brief?.checks ?? [];
|
|
25
|
+
const count = (st) => checks.filter((c) => c.state === st).length;
|
|
26
|
+
const steps = [...String(plan).matchAll(/^\s*[-*]\s+\[([ xX~!])\]/gm)].map((m) => m[1].toLowerCase());
|
|
27
|
+
const criteriaBlock = /^##\s*Acceptance criteria\s*$([\s\S]*?)(?=^##\s|(?![\s\S]))/im.exec(String(spec))?.[1] ?? '';
|
|
28
|
+
return {
|
|
29
|
+
files: paths.length,
|
|
30
|
+
added: files.reduce((s, f) => s + (f.added ?? 0), 0),
|
|
31
|
+
removed: files.reduce((s, f) => s + (f.removed ?? 0), 0),
|
|
32
|
+
testFiles: tests.length,
|
|
33
|
+
codeFiles: code.length,
|
|
34
|
+
ci: open?.checks?.state ?? null, // passed | failed | running | null (no PR or no checks)
|
|
35
|
+
ciFailed: open?.checks?.failed ?? [],
|
|
36
|
+
checks: { passed: count('passed'), failed: count('failed'), pending: count('pending') + count('running') },
|
|
37
|
+
plan: steps.length ? { steps: steps.length, done: steps.filter((s) => s === 'x').length } : null,
|
|
38
|
+
criteria: criteriaBlock.split('\n').filter((l) => /^\s*[-*\d]/.test(l) && /\w/.test(l)).length,
|
|
39
|
+
hasSpec: Boolean(String(spec).trim()),
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const factLines = (f) =>
|
|
44
|
+
[
|
|
45
|
+
`files changed: ${f.files} (+${f.added} -${f.removed}); code files: ${f.codeFiles}; test files: ${f.testFiles}`,
|
|
46
|
+
`CI on the pull request: ${f.ci ?? 'no pull request or no checks yet'}${f.ciFailed.length ? ` (failing: ${f.ciFailed.join(', ')})` : ''}`,
|
|
47
|
+
`the agent's own checks (from its brief): ${f.checks.passed} passed, ${f.checks.failed} failed, ${f.checks.pending} pending`,
|
|
48
|
+
`plan: ${f.plan ? `${f.plan.done} of ${f.plan.steps} steps ticked` : 'no plan written'}`,
|
|
49
|
+
`spec: ${f.hasSpec ? `${f.criteria} acceptance criteria` : 'no spec written'}`,
|
|
50
|
+
].join('\n');
|
|
51
|
+
|
|
52
|
+
export function scorecardPrompt({ intent, status, digest, briefMarkdown, intentDoc, specDoc, planDoc, hunks, facts: f }) {
|
|
53
|
+
return [
|
|
54
|
+
"You score an AI coding agent's changes for a reviewer who wants to approve with confidence and read as little as possible.",
|
|
55
|
+
'Be skeptical: a claim without evidence in the diff, the tests or the checks counts as unverified. Base everything on the material below.',
|
|
56
|
+
'',
|
|
57
|
+
'Reply with one JSON object only:',
|
|
58
|
+
'{"summary": "<two sentences: what the change does and how sure you are>",',
|
|
59
|
+
' "scores": {"intent": {"score": 0-100, "why": "<one sentence>"}, "scope": {...}, "plan": {...}, "tests": {...}, "safety": {...}},',
|
|
60
|
+
' "criteria": [{"criterion": "<an acceptance criterion from the spec>", "status": "met|not_met|unverified", "evidence": "<where: file, test, check>"}],',
|
|
61
|
+
' "look": [{"id": "H3", "reason": "<why a person should read this hunk>"}]}',
|
|
62
|
+
'',
|
|
63
|
+
'Scoring (100 is best; be calibrated, not generous):',
|
|
64
|
+
'- intent: how fully the change does what INTENT asks. Missing parts of the ask pull it down hard.',
|
|
65
|
+
'- scope: 100 when every change serves the task; lower for unrelated edits, drive-by refactors, or a change far larger than the task needs.',
|
|
66
|
+
'- plan: how closely the change follows PLAN (steps done, departures explained). Without a plan, judge whether the work is coherent.',
|
|
67
|
+
'- tests: whether the changed behavior is verified: tests added or updated for it, CI, checks the agent ran. Code changes with no tests and no passing checks score low.',
|
|
68
|
+
'- safety: 100 when nothing risky is touched or risks are handled; lower for security, auth, data loss, migrations, concurrency, money, new dependencies or public API changes without care.',
|
|
69
|
+
'- criteria: one entry per acceptance criterion in SPEC, in its order; leave the list empty if there is no spec.',
|
|
70
|
+
'- look: at most 5 hunks a person must read (the risky, subtle or unverified ones), most important first. Not the obvious ones.',
|
|
71
|
+
'',
|
|
72
|
+
'<facts_measured_by_the_app>',
|
|
73
|
+
factLines(f),
|
|
74
|
+
'</facts_measured_by_the_app>',
|
|
75
|
+
'<intent>',
|
|
76
|
+
intentDoc || intent || '(none)',
|
|
77
|
+
'</intent>',
|
|
78
|
+
'<spec>',
|
|
79
|
+
specDoc || '(no spec written)',
|
|
80
|
+
'</spec>',
|
|
81
|
+
'<plan>',
|
|
82
|
+
planDoc || '(no plan written)',
|
|
83
|
+
'</plan>',
|
|
84
|
+
'<agent_status>',
|
|
85
|
+
status,
|
|
86
|
+
'</agent_status>',
|
|
87
|
+
'<conversation>',
|
|
88
|
+
(digest || '(none)').slice(-12_000),
|
|
89
|
+
'</conversation>',
|
|
90
|
+
'<brief>',
|
|
91
|
+
briefMarkdown || '(none)',
|
|
92
|
+
'</brief>',
|
|
93
|
+
'<hunks>',
|
|
94
|
+
...hunks.map((h) => `<hunk id="${h.id}" file="${h.file}">\n${h.head}\n${h.text}\n</hunk>`),
|
|
95
|
+
'</hunks>',
|
|
96
|
+
].join('\n');
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const clamp = (n) => Math.max(0, Math.min(100, Math.round(Number(n) || 0)));
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* The scorecard from the model's reply and the facts: scores capped by what was measured, the spec
|
|
103
|
+
* score computed from the criteria, the overall weighted, and a recommendation by rule.
|
|
104
|
+
*/
|
|
105
|
+
export function combine(reply, f, hunks) {
|
|
106
|
+
const s = reply?.scores ?? {};
|
|
107
|
+
const score = (k) => (s[k] && Number.isFinite(Number(s[k].score)) ? { score: clamp(s[k].score), why: String(s[k].why ?? '').slice(0, 300), notes: [] } : null);
|
|
108
|
+
const out = { intent: score('intent'), scope: score('scope'), plan: score('plan'), tests: score('tests'), safety: score('safety'), spec: null };
|
|
109
|
+
const cap = (k, max, note) => {
|
|
110
|
+
if (out[k] && out[k].score > max) {
|
|
111
|
+
out[k].score = max;
|
|
112
|
+
out[k].notes.push(note);
|
|
113
|
+
}
|
|
114
|
+
};
|
|
115
|
+
// Tests: failing CI or failing checks, or code changed with nothing verifying it.
|
|
116
|
+
if (f.ci === 'failed') cap('tests', 30, `CI is failing${f.ciFailed.length ? ` (${f.ciFailed.join(', ')})` : ''}`);
|
|
117
|
+
if (f.checks.failed) cap('tests', 40, `${f.checks.failed} of the agent's checks failed`);
|
|
118
|
+
if (f.codeFiles && !f.testFiles && !f.checks.passed && f.ci !== 'passed') cap('tests', 45, 'code changed with no tests and no passing checks');
|
|
119
|
+
// Plan: bounded by the steps ticked.
|
|
120
|
+
if (f.plan && out.plan) cap('plan', Math.round(40 + (60 * f.plan.done) / f.plan.steps), `${f.plan.done} of ${f.plan.steps} plan steps ticked`);
|
|
121
|
+
if (!f.plan && out.plan) out.plan.notes.push('no plan written');
|
|
122
|
+
// Spec: computed from the criteria (met 1, unverified ½, not met 0); none without a spec.
|
|
123
|
+
const criteria = (Array.isArray(reply?.criteria) ? reply.criteria : [])
|
|
124
|
+
.filter((c) => c?.criterion)
|
|
125
|
+
.slice(0, 20)
|
|
126
|
+
.map((c) => ({ criterion: String(c.criterion).slice(0, 300), status: ['met', 'not_met', 'unverified'].includes(c.status) ? c.status : 'unverified', evidence: String(c.evidence ?? '').slice(0, 300) }));
|
|
127
|
+
// A spec with criteria the model didn't assess: they count as unverified, not as passed.
|
|
128
|
+
if (f.hasSpec && f.criteria && !criteria.length) out.spec = { score: 50, why: `${f.criteria} acceptance criteri${f.criteria === 1 ? 'on' : 'a'} not assessed`, notes: ['treated as unverified'] };
|
|
129
|
+
if (f.hasSpec && criteria.length) {
|
|
130
|
+
const pts = criteria.reduce((n, c) => n + (c.status === 'met' ? 1 : c.status === 'unverified' ? 0.5 : 0), 0);
|
|
131
|
+
const met = criteria.filter((c) => c.status === 'met').length;
|
|
132
|
+
out.spec = { score: clamp((100 * pts) / criteria.length), why: `${met} of ${criteria.length} acceptance criteri${criteria.length === 1 ? 'on' : 'a'} met${criteria.some((c) => c.status === 'not_met') ? ', some not met' : ''}`, notes: [] };
|
|
133
|
+
}
|
|
134
|
+
// Overall: weighted over the scores there are.
|
|
135
|
+
const have = Object.entries(WEIGHTS).filter(([k]) => out[k]);
|
|
136
|
+
const total = have.reduce((n, [, w]) => n + w, 0);
|
|
137
|
+
const overall = total ? Math.round(have.reduce((n, [k, w]) => n + out[k].score * w, 0) / total) : null;
|
|
138
|
+
const notMet = criteria.filter((c) => c.status === 'not_met');
|
|
139
|
+
const reasons = [];
|
|
140
|
+
if (f.ci === 'failed') reasons.push('CI is failing');
|
|
141
|
+
if (notMet.length) reasons.push(`${notMet.length} acceptance criteri${notMet.length === 1 ? 'on' : 'a'} not met`);
|
|
142
|
+
if (out.intent && out.intent.score < 50) reasons.push('it does not do enough of what was asked');
|
|
143
|
+
if (out.tests && out.tests.score < 40 && f.ci !== 'failed') reasons.push('it is barely verified');
|
|
144
|
+
let recommendation;
|
|
145
|
+
if (reasons.length) recommendation = 'not_ready';
|
|
146
|
+
else if (overall != null && overall >= 80 && (out.tests?.score ?? 0) >= 70 && (out.safety?.score ?? 100) >= 70 && (out.spec?.score ?? 100) >= 70 && f.ci !== 'running') recommendation = 'ready';
|
|
147
|
+
else recommendation = 'look';
|
|
148
|
+
// Where to look, mapped back to hunks.
|
|
149
|
+
const byId = new Map(hunks.map((h) => [h.id, h]));
|
|
150
|
+
const look = (Array.isArray(reply?.look) ? reply.look : [])
|
|
151
|
+
.map((l) => ({ h: byId.get(String(l?.id ?? '').trim()), reason: String(l?.reason ?? '').slice(0, 300) }))
|
|
152
|
+
.filter((l) => l.h)
|
|
153
|
+
.slice(0, 5)
|
|
154
|
+
.map(({ h, reason }) => ({ file: h.file, head: h.head, reason }));
|
|
155
|
+
return { summary: String(reply?.summary ?? '').slice(0, 600), overall, recommendation, reasons, scores: out, criteria, look, facts: f };
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
export const RECOMMENDATION = { ready: 'Ready to approve', look: 'Look at these first', not_ready: 'Not ready' };
|
|
159
|
+
|
|
160
|
+
/** Markdown: for Copy / Download, the pull request's description and a PR comment. */
|
|
161
|
+
export function scorecardMarkdown(sc, title = '') {
|
|
162
|
+
const bar = (n) => (n == null ? '—' : `${n}`);
|
|
163
|
+
const rows = DIMENSIONS.map(([k, label]) => {
|
|
164
|
+
const d = sc.scores[k];
|
|
165
|
+
return `| ${label} | ${bar(d?.score)} | ${d ? [d.why, ...d.notes].filter(Boolean).join('; ').replace(/\|/g, '/') : k === 'spec' ? 'no spec written' : '—'} |`;
|
|
166
|
+
});
|
|
167
|
+
const out = [
|
|
168
|
+
`### Review scorecard${title ? `: ${title}` : ''}`,
|
|
169
|
+
'',
|
|
170
|
+
`**${sc.overall ?? '—'}/100 · ${RECOMMENDATION[sc.recommendation]}**${sc.reasons.length ? ` (${sc.reasons.join('; ')})` : ''}`,
|
|
171
|
+
'',
|
|
172
|
+
sc.summary,
|
|
173
|
+
'',
|
|
174
|
+
'| Score | /100 | Why |',
|
|
175
|
+
'| --- | --- | --- |',
|
|
176
|
+
...rows,
|
|
177
|
+
];
|
|
178
|
+
if (sc.criteria.length) {
|
|
179
|
+
out.push('', '**Acceptance criteria**', '');
|
|
180
|
+
for (const c of sc.criteria) out.push(`- ${c.status === 'met' ? '✅' : c.status === 'not_met' ? '❌' : '❔'} ${c.criterion}${c.evidence ? ` (${c.evidence})` : ''}`);
|
|
181
|
+
}
|
|
182
|
+
if (sc.look.length) {
|
|
183
|
+
out.push('', '**Where to look**', '');
|
|
184
|
+
for (const l of sc.look) out.push(`- \`${l.file}\` \`${l.head.replace(/^(@@[^@]*@@).*/, '$1')}\`: ${l.reason}`);
|
|
185
|
+
}
|
|
186
|
+
const f = sc.facts;
|
|
187
|
+
out.push('', `<sub>Measured: ${f.files} files (+${f.added} −${f.removed}), ${f.testFiles} test files, CI ${f.ci ?? 'n/a'}, checks ${f.checks.passed}✓ ${f.checks.failed}✗${f.plan ? `, plan ${f.plan.done}/${f.plan.steps}` : ''}. Scored by AI Agent Manager.</sub>`);
|
|
188
|
+
return out.join('\n');
|
|
189
|
+
}
|
package/src/server.js
CHANGED
|
@@ -240,6 +240,8 @@ export async function startServer({ port = PORT, host = HOST } = {}) {
|
|
|
240
240
|
return manager.document(req.params.id, req.params.kind);
|
|
241
241
|
}),
|
|
242
242
|
);
|
|
243
|
+
api.get('/agents/:id/specs/:name', wrap((req) => manager.specDoc(req.params.id, req.params.name)));
|
|
244
|
+
api.post('/agents/:id/scorecard/post', wrap((req) => review.postScorecard(req.params.id)));
|
|
243
245
|
api.post('/agents/:id/plan', wrap((req) => manager.startPlanning(req.params.id)));
|
|
244
246
|
api.post('/agents/:id/plan/approve', wrap((req) => manager.approvePlan(req.params.id, req.body?.note)));
|
|
245
247
|
api.post('/agents/:id/plan/changes', wrap((req) => manager.requestPlanChanges(req.params.id, req.body?.note)));
|