@rigour-labs/core 6.6.5 → 6.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -92
- package/dist/environment.test.js +0 -4
- package/dist/gates/agent-team.d.ts +0 -2
- package/dist/gates/agent-team.js +0 -8
- package/dist/gates/checkpoint.d.ts +0 -1
- package/dist/gates/checkpoint.js +0 -1
- package/dist/gates/context.d.ts +0 -4
- package/dist/gates/context.js +7 -17
- package/dist/gates/deep-analysis.d.ts +0 -1
- package/dist/gates/deep-runner.js +0 -1
- package/dist/gates/frontend-secret-exposure.d.ts +1 -1
- package/dist/gates/frontend-secret-exposure.js +9 -3
- package/dist/gates/frontend-secret-exposure.test.js +6 -3
- package/dist/gates/hallucinated-imports/index.d.ts +0 -2
- package/dist/gates/hallucinated-imports/index.js +2 -2
- package/dist/gates/hallucinated-imports/js-resolver.js +1 -0
- package/dist/gates/hallucinated-imports.test.js +10 -0
- package/dist/gates/runner.js +3 -4
- package/dist/gates/test-quality-matchers.d.ts +0 -4
- package/dist/gates/test-quality-matchers.js +0 -7
- package/dist/gates/test-quality.d.ts +0 -2
- package/dist/gates/test-quality.js +1 -10
- package/dist/hooks/checker.js +1 -1
- package/dist/hooks/index.d.ts +2 -3
- package/dist/hooks/index.js +1 -2
- package/dist/hooks/stop-review.d.ts +6 -9
- package/dist/hooks/stop-review.js +5 -10
- package/dist/hooks/types.d.ts +0 -14
- package/dist/hooks/types.js +1 -15
- package/dist/index.d.ts +11 -3
- package/dist/index.js +11 -3
- package/dist/inference/cloud-provider.d.ts +2 -0
- package/dist/inference/cloud-provider.js +16 -6
- package/dist/inference/cloud-provider.test.js +21 -0
- package/dist/pattern-index/embeddings.d.ts +0 -5
- package/dist/pattern-index/embeddings.js +11 -4
- package/dist/pattern-index/semantic-runtime.d.ts +10 -0
- package/dist/pattern-index/semantic-runtime.js +46 -0
- package/dist/pattern-index/semantic-runtime.test.d.ts +1 -0
- package/dist/pattern-index/semantic-runtime.test.js +31 -0
- package/dist/review/backtest-init.d.ts +8 -0
- package/dist/review/backtest-init.js +87 -0
- package/dist/review/backtest-init.test.d.ts +1 -0
- package/dist/review/backtest-init.test.js +74 -0
- package/dist/review/backtest-judges.d.ts +38 -0
- package/dist/review/backtest-judges.js +63 -0
- package/dist/review/backtest-judges.test.d.ts +1 -0
- package/dist/review/backtest-judges.test.js +32 -0
- package/dist/review/backtest.d.ts +320 -0
- package/dist/review/backtest.js +207 -0
- package/dist/review/backtest.test.d.ts +1 -0
- package/dist/review/backtest.test.js +129 -0
- package/dist/review/baseline.js +2 -1
- package/dist/review/dead-code.test.js +15 -0
- package/dist/review/fix-scope.d.ts +21 -0
- package/dist/review/fix-scope.js +94 -0
- package/dist/review/fix-scope.test.d.ts +1 -0
- package/dist/review/fix-scope.test.js +104 -0
- package/dist/review/lint-overlay.d.ts +21 -0
- package/dist/review/lint-overlay.js +89 -0
- package/dist/review/orphan-files.js +3 -2
- package/dist/review/partial-wiring.js +5 -3
- package/dist/review/query-patterns.js +3 -2
- package/dist/review/quiet.d.ts +6 -0
- package/dist/review/quiet.js +12 -2
- package/dist/review/quiet.test.js +11 -0
- package/dist/review/review.d.ts +6 -0
- package/dist/review/review.js +10 -2
- package/dist/review/reviewer/adapters.d.ts +51 -0
- package/dist/review/reviewer/adapters.js +181 -0
- package/dist/review/reviewer/adapters.test.d.ts +1 -0
- package/dist/review/reviewer/adapters.test.js +23 -0
- package/dist/review/reviewer/background.d.ts +41 -0
- package/dist/review/reviewer/background.js +153 -0
- package/dist/review/reviewer/background.test.d.ts +1 -0
- package/dist/review/reviewer/background.test.js +118 -0
- package/dist/review/reviewer/consensus.d.ts +36 -0
- package/dist/review/reviewer/consensus.js +144 -0
- package/dist/review/reviewer/context.d.ts +75 -0
- package/dist/review/reviewer/context.js +140 -0
- package/dist/review/reviewer/context.test.d.ts +1 -0
- package/dist/review/reviewer/context.test.js +18 -0
- package/dist/review/reviewer/exec.d.ts +17 -0
- package/dist/review/reviewer/exec.js +27 -0
- package/dist/review/reviewer/exec.test.d.ts +1 -0
- package/dist/review/reviewer/exec.test.js +23 -0
- package/dist/review/reviewer/inputs.d.ts +39 -0
- package/dist/review/reviewer/inputs.js +107 -0
- package/dist/review/reviewer/merge-impact.d.ts +2 -0
- package/dist/review/reviewer/merge-impact.js +64 -0
- package/dist/review/reviewer/panel.d.ts +67 -0
- package/dist/review/reviewer/panel.js +101 -0
- package/dist/review/reviewer/panel.test.d.ts +1 -0
- package/dist/review/reviewer/panel.test.js +110 -0
- package/dist/review/reviewer/prompt.d.ts +32 -0
- package/dist/review/reviewer/prompt.js +145 -0
- package/dist/review/reviewer/settings.d.ts +58 -0
- package/dist/review/reviewer/settings.js +152 -0
- package/dist/review/reviewer/settings.test.d.ts +1 -0
- package/dist/review/reviewer/settings.test.js +57 -0
- package/dist/review/reviewer/store.d.ts +49 -0
- package/dist/review/reviewer/store.js +99 -0
- package/dist/review/reviewer/usage.d.ts +2 -0
- package/dist/review/reviewer/usage.js +28 -0
- package/dist/review/reviewer/usage.test.d.ts +1 -0
- package/dist/review/reviewer/usage.test.js +14 -0
- package/dist/review/reviewer/verdict.d.ts +141 -0
- package/dist/review/reviewer/verdict.js +166 -0
- package/dist/review/reviewer.d.ts +84 -49
- package/dist/review/reviewer.js +364 -195
- package/dist/review/reviewer.test.js +316 -50
- package/dist/review/test-files.d.ts +5 -1
- package/dist/review/test-files.js +6 -2
- package/dist/review/toolchain.d.ts +5 -2
- package/dist/review/toolchain.js +89 -7
- package/dist/review/toolchain.test.js +45 -4
- package/dist/review/typed/program.d.ts +26 -0
- package/dist/review/typed/program.js +90 -0
- package/dist/review/typed/redundancy.d.ts +10 -0
- package/dist/review/typed/redundancy.js +378 -0
- package/dist/review/typed/redundancy.test.d.ts +1 -0
- package/dist/review/typed/redundancy.test.js +179 -0
- package/dist/review/typed/schema-nullability.d.ts +6 -0
- package/dist/review/typed/schema-nullability.js +218 -0
- package/dist/review/typed/schema-nullability.test.d.ts +1 -0
- package/dist/review/typed/schema-nullability.test.js +63 -0
- package/dist/review/unused-exports.js +10 -7
- package/dist/semantic/learn/types.d.ts +2 -2
- package/dist/services/adaptive-thresholds.d.ts +2 -52
- package/dist/services/adaptive-thresholds.js +2 -147
- package/dist/services/adaptive-thresholds.test.js +1 -81
- package/dist/settings.d.ts +14 -6
- package/dist/settings.js +0 -12
- package/dist/storage/team-diagnostics.js +1 -1
- package/dist/storage/team-schema.d.ts +1 -1
- package/dist/storage/team-schema.js +2 -0
- package/dist/storage/team-store.js +26 -3
- package/dist/storage/team-sync-scope.test.js +35 -2
- package/dist/telemetry/telemetry.d.ts +5 -0
- package/dist/telemetry/telemetry.js +11 -1
- package/dist/templates/paradigms.js +0 -8
- package/dist/templates/presets.d.ts +0 -1
- package/dist/templates/presets.js +0 -23
- package/dist/templates/universal-config.js +16 -46
- package/dist/types/index.d.ts +202 -574
- package/dist/types/index.js +64 -67
- package/dist/types/index.test.d.ts +1 -0
- package/dist/types/index.test.js +33 -0
- package/dist/utils/command-line.d.ts +8 -0
- package/dist/utils/command-line.js +8 -0
- package/dist/utils/command-line.test.d.ts +1 -0
- package/dist/utils/command-line.test.js +8 -0
- package/dist/utils/profile.js +1 -1
- package/package.json +6 -7
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What the reviewer works from, gathered before any model runs: the pull request (found by
|
|
3
|
+
* branch first, since at push time the commit is not on the forge yet; by commit for a detached
|
|
4
|
+
* checkout such as a backtest; only a number counts), every human review and inline comment
|
|
5
|
+
* (bots and the author excluded; everything from `reviewsBefore` on hidden for a backtest), the
|
|
6
|
+
* description, the diff written to a file (a reviewer whose shell is sandboxed still reads it,
|
|
7
|
+
* and no reviewer spends turns paging it), and whether HEAD merges the base in.
|
|
8
|
+
*/
|
|
9
|
+
import { createHash } from 'crypto';
|
|
10
|
+
import fs from 'fs';
|
|
11
|
+
import path from 'path';
|
|
12
|
+
import { GH_TIMEOUT_MS, parseJsonArrays } from './exec.js';
|
|
13
|
+
export function ghFor(cwd, exec, env) {
|
|
14
|
+
return args => exec('gh', args, { cwd, timeoutMs: GH_TIMEOUT_MS, env });
|
|
15
|
+
}
|
|
16
|
+
const PR_FIELDS = 'number,state,isDraft,author,body';
|
|
17
|
+
/** The pull request for the branch (or the one named, or the one the commit is on), or none, or why it could not be read. */
|
|
18
|
+
export async function findPullRequest(gh, branch, head, named) {
|
|
19
|
+
if (named !== undefined)
|
|
20
|
+
return viewPullRequest(gh, String(named));
|
|
21
|
+
if (branch !== 'HEAD') {
|
|
22
|
+
const byBranch = await viewPullRequest(gh, branch);
|
|
23
|
+
if (byBranch.pr || !byBranch.error)
|
|
24
|
+
return byBranch;
|
|
25
|
+
if (!/no pull requests? found|could not resolve/i.test(byBranch.error))
|
|
26
|
+
return byBranch;
|
|
27
|
+
}
|
|
28
|
+
const byCommit = await gh(['api', `repos/{owner}/{repo}/commits/${head}/pulls`, '-q', '.[0].number']);
|
|
29
|
+
if (byCommit.exitCode !== 0)
|
|
30
|
+
return branch === 'HEAD' ? { error: `could not look up the pull request for ${head.slice(0, 9)}: ${byCommit.stderr.trim()}` } : {};
|
|
31
|
+
const number = byCommit.stdout.trim();
|
|
32
|
+
return /^\d+$/.test(number) ? viewPullRequest(gh, number) : {};
|
|
33
|
+
}
|
|
34
|
+
async function viewPullRequest(gh, selector) {
|
|
35
|
+
const result = await gh(['pr', 'view', selector, '--json', PR_FIELDS]);
|
|
36
|
+
if (result.exitCode !== 0) {
|
|
37
|
+
const message = result.stderr.trim() || result.stdout.trim();
|
|
38
|
+
return /no pull requests? found/i.test(message) ? {} : { error: `could not read the pull request for ${selector}: ${message.slice(0, 200)}` };
|
|
39
|
+
}
|
|
40
|
+
try {
|
|
41
|
+
const parsed = JSON.parse(result.stdout);
|
|
42
|
+
if (!Number.isInteger(parsed.number))
|
|
43
|
+
return { error: `gh returned no pull request number for ${selector}` };
|
|
44
|
+
return { pr: { number: parsed.number, state: String(parsed.state ?? '').toLowerCase(), draft: !!parsed.isDraft, author: String(parsed.author?.login ?? ''), body: String(parsed.body ?? '') } };
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
return { error: `gh returned something other than a pull request for ${selector}: ${result.stdout.slice(0, 120)}` };
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/** Written by a person: not a bot, and not the pull request's author answering their own review. */
|
|
51
|
+
export function isHuman(item, author) {
|
|
52
|
+
return !!item?.user && item.user.type !== 'Bot' && !/\[bot\]$/.test(item.user.login) && item.user.login !== author;
|
|
53
|
+
}
|
|
54
|
+
/** Every review by a person and every inline comment by a person, hidden from `reviewsBefore` on. */
|
|
55
|
+
export async function humanReviews(gh, pr, reviewsBefore) {
|
|
56
|
+
const reviews = await gh(['api', `repos/{owner}/{repo}/pulls/${pr.number}/reviews`, '--paginate']);
|
|
57
|
+
if (reviews.exitCode !== 0)
|
|
58
|
+
return { error: `could not read the reviews of pull request ${pr.number}: ${reviews.stderr.trim().slice(0, 200)}` };
|
|
59
|
+
const comments = await gh(['api', `repos/{owner}/{repo}/pulls/${pr.number}/comments`, '--paginate']);
|
|
60
|
+
if (comments.exitCode !== 0)
|
|
61
|
+
return { error: `could not read the comments of pull request ${pr.number}: ${comments.stderr.trim().slice(0, 200)}` };
|
|
62
|
+
const human = (x) => isHuman(x, pr.author);
|
|
63
|
+
const before = (at) => !reviewsBefore || (typeof at === 'string' && at < reviewsBefore);
|
|
64
|
+
const rounds = parseJsonArrays(reviews.stdout).filter((r) => human(r) && String(r.body ?? '').trim() && before(r.submitted_at));
|
|
65
|
+
const inline = parseJsonArrays(comments.stdout).filter((c) => human(c) && before(c.created_at));
|
|
66
|
+
let markdown = rounds.map((r) => `## Review by ${r.user.login}, ${r.submitted_at}, ${r.state} (on ${String(r.commit_id ?? '').slice(0, 9)})\n\n${String(r.body).trim()}\n`).join('\n');
|
|
67
|
+
if (inline.length)
|
|
68
|
+
markdown += `\n## Inline comments\n${inline.map((c) => `- ${c.created_at} ${c.path}:${c.line ?? c.original_line ?? '?'}: ${String(c.body ?? '').trim()}`).join('\n')}\n`;
|
|
69
|
+
const latest = rounds.at(-1);
|
|
70
|
+
return {
|
|
71
|
+
reviews: {
|
|
72
|
+
markdown: markdown || 'none\n',
|
|
73
|
+
key: [...rounds.map((r) => `${r.id},${r.submitted_at},${r.commit_id},${r.state}`), ...inline.map((c) => `${c.id},${c.updated_at}`)].join('|'),
|
|
74
|
+
count: rounds.length,
|
|
75
|
+
...(latest ? { label: `${latest.user.login}, ${latest.submitted_at} (${rounds.length} review${rounds.length === 1 ? '' : 's'})` } : {}),
|
|
76
|
+
},
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
/** The repository's rules as the reviewer must read them. */
|
|
80
|
+
export function rulesText(cwd) {
|
|
81
|
+
return ['AGENTS.md', 'CLAUDE.md'].map(name => {
|
|
82
|
+
try {
|
|
83
|
+
return fs.readFileSync(path.join(cwd, name), 'utf8');
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
return '';
|
|
87
|
+
}
|
|
88
|
+
}).join('\n');
|
|
89
|
+
}
|
|
90
|
+
export function sha(parts) {
|
|
91
|
+
return createHash('sha256').update(parts.join('\0')).digest('hex').slice(0, 16);
|
|
92
|
+
}
|
|
93
|
+
/** True when HEAD is a merge whose second parent is on the base: the branch merged main in. */
|
|
94
|
+
export async function mergesBaseIn(cwd, baseSha, exec) {
|
|
95
|
+
const parents = (await exec('git', ['rev-list', '--parents', '-n1', 'HEAD'], { cwd, timeoutMs: GH_TIMEOUT_MS })).stdout.trim().split(/\s+/);
|
|
96
|
+
if (parents.length <= 2)
|
|
97
|
+
return false;
|
|
98
|
+
return (await exec('git', ['merge-base', '--is-ancestor', 'HEAD^2', baseSha], { cwd, timeoutMs: GH_TIMEOUT_MS })).exitCode === 0;
|
|
99
|
+
}
|
|
100
|
+
/** Added and removed lines between two commits. */
|
|
101
|
+
export async function linesChanged(cwd, from, to, exec) {
|
|
102
|
+
const numstat = (await exec('git', ['diff', '--numstat', `${from}..${to}`], { cwd, timeoutMs: GH_TIMEOUT_MS })).stdout;
|
|
103
|
+
return numstat.split('\n').reduce((sum, line) => {
|
|
104
|
+
const [added, removed] = line.split('\t');
|
|
105
|
+
return sum + (Number(added) || 0) + (Number(removed) || 0);
|
|
106
|
+
}, 0);
|
|
107
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* After a merge from main: which main-side files the branch's changed files import, and how main
|
|
3
|
+
* changed them. A branch call site written against the old definition is the regression a plain
|
|
4
|
+
* diff review never shows (main changes how an id is parsed; the branch's link builder keeps
|
|
5
|
+
* building the old shape). Rendered as Markdown for the reviewer; empty when nothing overlaps.
|
|
6
|
+
*/
|
|
7
|
+
import path from 'path';
|
|
8
|
+
import { GH_TIMEOUT_MS } from './exec.js';
|
|
9
|
+
const CODE = /\.(ts|tsx|js|jsx|mjs|cjs|svelte)$/;
|
|
10
|
+
const TEST = /\.(test|spec|e2e)\.[cm]?[jt]sx?$/;
|
|
11
|
+
const IMPORT = /import\s*(?:type\s*)?(?:\{([^}]*)\}|(\w+))?\s*(?:,\s*\{([^}]*)\})?\s*from\s*['"]([^'"]+)['"]|import\(\s*['"]([^'"]+)['"]\s*\)/g;
|
|
12
|
+
const MAX_DIFF_CHARS = 12_000;
|
|
13
|
+
const withoutExtension = (file) => file.replace(CODE, '').replace(/\/index$/, '');
|
|
14
|
+
export async function mergeImpact(cwd, oldBase, newBase, head, exec) {
|
|
15
|
+
const git = async (args) => (await exec('git', args, { cwd, timeoutMs: GH_TIMEOUT_MS })).stdout;
|
|
16
|
+
const files = (text) => text.split('\n').filter(f => f && CODE.test(f));
|
|
17
|
+
const mainSide = new Set(files(await git(['diff', '--name-only', oldBase, newBase])));
|
|
18
|
+
const branchFiles = files(await git(['diff', '--name-only', `${newBase}...${head}`]));
|
|
19
|
+
// A file both sides changed was reconciled by the merge itself; the silent risk is a
|
|
20
|
+
// main-only change under a branch file that still calls the old shape.
|
|
21
|
+
for (const file of branchFiles)
|
|
22
|
+
mainSide.delete(file);
|
|
23
|
+
const mainByStem = new Map([...mainSide].map(file => [withoutExtension(file), file]));
|
|
24
|
+
const hits = [];
|
|
25
|
+
for (const file of branchFiles) {
|
|
26
|
+
if (TEST.test(file))
|
|
27
|
+
continue;
|
|
28
|
+
const shown = await exec('git', ['show', `${head}:${file}`], { cwd, timeoutMs: GH_TIMEOUT_MS });
|
|
29
|
+
if (shown.exitCode !== 0)
|
|
30
|
+
continue;
|
|
31
|
+
for (const match of shown.stdout.matchAll(IMPORT)) {
|
|
32
|
+
const target = resolve(match[4] ?? match[5], file);
|
|
33
|
+
const mainFile = target ? mainByStem.get(target) : undefined;
|
|
34
|
+
if (!mainFile)
|
|
35
|
+
continue;
|
|
36
|
+
const names = [match[1], match[3]].filter(Boolean).join(',').split(',').map(s => s.trim().replace(/^type\s+/, '').split(/\s+as\s+/)[0]).filter(Boolean);
|
|
37
|
+
if (match[2])
|
|
38
|
+
names.push(match[2]);
|
|
39
|
+
hits.push({ file, mainFile, names });
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
if (hits.length === 0)
|
|
43
|
+
return '';
|
|
44
|
+
const out = ['## Merge impact: main-side files the branch imports that main changed', ''];
|
|
45
|
+
const seen = new Set();
|
|
46
|
+
for (const hit of hits) {
|
|
47
|
+
if (seen.has(`${hit.file}|${hit.mainFile}`))
|
|
48
|
+
continue;
|
|
49
|
+
seen.add(`${hit.file}|${hit.mainFile}`);
|
|
50
|
+
const named = hit.names.join(', ') || '(default)';
|
|
51
|
+
out.push(`### ${hit.file} imports ${named} from ${hit.mainFile}`);
|
|
52
|
+
const tests = [...mainSide].filter(f => f !== hit.mainFile && f.replace(/\.(test|spec)\./, '.') === hit.mainFile);
|
|
53
|
+
const diff = await git(['diff', '--unified=3', oldBase, newBase, '--', hit.mainFile, ...tests]);
|
|
54
|
+
out.push('```diff', diff.trim().slice(0, MAX_DIFF_CHARS), '```', '', `Check every call site of ${named} in ${hit.file} against the new definition and the new tests.`, '');
|
|
55
|
+
}
|
|
56
|
+
return out.join('\n');
|
|
57
|
+
}
|
|
58
|
+
function resolve(specifier, from) {
|
|
59
|
+
if (specifier.startsWith('.'))
|
|
60
|
+
return withoutExtension(path.posix.normalize(path.posix.join(path.posix.dirname(from), specifier)));
|
|
61
|
+
if (specifier.startsWith('$lib/'))
|
|
62
|
+
return withoutExtension(`src/lib/${specifier.slice(5)}`);
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Two or three judges as a panel, run around the consensus rules (consensus.ts). The judges' own
|
|
3
|
+
* items are grouped into findings; a finding a strict majority raised is confirmed. Every other
|
|
4
|
+
* judge then cross-examines, in one batched call each, only the findings it did not raise: confirm
|
|
5
|
+
* or refute with file:line evidence; no evidence or no answer is unsure. The calls decide by
|
|
6
|
+
* majority; a disputed finding is shown and never blocks. A finding disputed before, on a file no
|
|
7
|
+
* commit has touched since, stays disputed without another call; past the item cap, disputed too.
|
|
8
|
+
*
|
|
9
|
+
* Every finding keeps what a later re-scoring needs (who raised it, every judge's call, the
|
|
10
|
+
* cross-examination answers, each judge's own item and the class), so a probabilistic truth model
|
|
11
|
+
* can be fitted to the ledger later without running anyone again.
|
|
12
|
+
*/
|
|
13
|
+
import { type Call, type PanelStatus } from './consensus.js';
|
|
14
|
+
import type { OpenItem } from './verdict.js';
|
|
15
|
+
export interface PanelItem {
|
|
16
|
+
/** The item as its first raiser reported it. */
|
|
17
|
+
item: OpenItem;
|
|
18
|
+
/** The judges who raised it unprompted. */
|
|
19
|
+
judges: string[];
|
|
20
|
+
/** Every judge's call on it, raised or answered in cross-examination. */
|
|
21
|
+
calls: Record<string, Call>;
|
|
22
|
+
/** Every cross-examination answer about it, with the code each judge quoted. */
|
|
23
|
+
cross?: Array<{
|
|
24
|
+
by: string;
|
|
25
|
+
call: Call;
|
|
26
|
+
evidence?: string;
|
|
27
|
+
}>;
|
|
28
|
+
/** Each judge's own item when more than one wrote it: what a later re-scoring of the panel needs. */
|
|
29
|
+
members?: Array<{
|
|
30
|
+
judge: string;
|
|
31
|
+
item: OpenItem;
|
|
32
|
+
}>;
|
|
33
|
+
status: PanelStatus;
|
|
34
|
+
/** Why it is disputed without a cross-examination (over the cap, disputed before and untouched since). */
|
|
35
|
+
note?: string;
|
|
36
|
+
}
|
|
37
|
+
/** A judge's answer about another judge's items: confirm or refute, each with the code that shows it. */
|
|
38
|
+
export interface Answer {
|
|
39
|
+
id: string;
|
|
40
|
+
call: 'confirm' | 'refute' | 'unsure';
|
|
41
|
+
evidence?: string;
|
|
42
|
+
}
|
|
43
|
+
export interface PanelInput {
|
|
44
|
+
judges: string[];
|
|
45
|
+
items: OpenItem[][];
|
|
46
|
+
/** Items disputed in the previous verdict, and the files changed since it. */
|
|
47
|
+
previousDisputed: OpenItem[];
|
|
48
|
+
touched: Set<string>;
|
|
49
|
+
maxItems: number;
|
|
50
|
+
/** Asks `judge` about items other judges raised; resolves to its answers (a missing one counts as unsure). */
|
|
51
|
+
ask: (judge: string, items: OpenItem[]) => Promise<Answer[]>;
|
|
52
|
+
/** Why no more agent runs may start today (a daily cap), checked before each judge's cross-examination. */
|
|
53
|
+
capped?: () => string | undefined;
|
|
54
|
+
/** Whether an answer's evidence quotes real code: a `file:line` the checkout has. */
|
|
55
|
+
evidenced: (evidence: string) => boolean;
|
|
56
|
+
}
|
|
57
|
+
/** The panel's decision on every finding any judge raised. */
|
|
58
|
+
export declare function runPanel(input: PanelInput): Promise<PanelItem[]>;
|
|
59
|
+
/** The answers in a judge's cross-examination reply; nothing usable is no answers (every item then disputed). */
|
|
60
|
+
export declare function parseAnswers(text: string): Answer[];
|
|
61
|
+
/** The accounting with the panel's decision applied: the judges' own items leave, and only what it confirmed blocks. */
|
|
62
|
+
export declare function applyPanel<T extends {
|
|
63
|
+
open: OpenItem[];
|
|
64
|
+
}>(accounted: T, judgeItemIds: Set<string>, decided: PanelItem[]): T & {
|
|
65
|
+
disputed: OpenItem[];
|
|
66
|
+
dropped: OpenItem[];
|
|
67
|
+
};
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Two or three judges as a panel, run around the consensus rules (consensus.ts). The judges' own
|
|
3
|
+
* items are grouped into findings; a finding a strict majority raised is confirmed. Every other
|
|
4
|
+
* judge then cross-examines, in one batched call each, only the findings it did not raise: confirm
|
|
5
|
+
* or refute with file:line evidence; no evidence or no answer is unsure. The calls decide by
|
|
6
|
+
* majority; a disputed finding is shown and never blocks. A finding disputed before, on a file no
|
|
7
|
+
* commit has touched since, stays disputed without another call; past the item cap, disputed too.
|
|
8
|
+
*
|
|
9
|
+
* Every finding keeps what a later re-scoring needs (who raised it, every judge's call, the
|
|
10
|
+
* cross-examination answers, each judge's own item and the class), so a probabilistic truth model
|
|
11
|
+
* can be fitted to the ledger later without running anyone again.
|
|
12
|
+
*/
|
|
13
|
+
import { clusterItems, MATCH_THRESHOLD, order, similarity, statusOf } from './consensus.js';
|
|
14
|
+
/** The panel's decision on every finding any judge raised. */
|
|
15
|
+
export async function runPanel(input) {
|
|
16
|
+
const n = input.judges.length;
|
|
17
|
+
const clusters = clusterItems(input.judges, input.items);
|
|
18
|
+
const decided = clusters.map(c => {
|
|
19
|
+
const raisers = [...new Set(c.members.map(m => m.judge))];
|
|
20
|
+
const calls = Object.fromEntries(raisers.map(j => [j, 'raised']));
|
|
21
|
+
const item = { ...c.members[0].item, evidence: c.members.map(m => m.item.evidence).find(Boolean) };
|
|
22
|
+
return { item, judges: raisers, calls, status: statusOf(calls, n), ...(c.members.length > 1 ? { members: c.members } : {}) };
|
|
23
|
+
});
|
|
24
|
+
const undecided = decided.filter(d => d.status !== 'confirmed');
|
|
25
|
+
const disputedBefore = (item) => !!item.file && !input.touched.has(item.file) && input.previousDisputed.some(p => similarity(item, p) >= MATCH_THRESHOLD);
|
|
26
|
+
for (const d of undecided.filter(d => disputedBefore(d.item))) {
|
|
27
|
+
for (const judge of input.judges)
|
|
28
|
+
d.calls[judge] ??= 'not asked';
|
|
29
|
+
d.status = 'disputed';
|
|
30
|
+
d.note = 'disputed before; its file is unchanged since';
|
|
31
|
+
}
|
|
32
|
+
const toAsk = undecided.filter(d => !d.note).sort((a, b) => order(a.item, b.item));
|
|
33
|
+
for (const d of toAsk.slice(input.maxItems)) {
|
|
34
|
+
for (const judge of input.judges)
|
|
35
|
+
d.calls[judge] ??= 'not asked';
|
|
36
|
+
d.status = 'disputed';
|
|
37
|
+
d.note = `over review.reviewer.panel_max_items (${input.maxItems})`;
|
|
38
|
+
}
|
|
39
|
+
const asked = toAsk.slice(0, input.maxItems);
|
|
40
|
+
// One batched call per judge, about every asked finding it did not raise; the calls run together.
|
|
41
|
+
await Promise.all(input.judges.map(async (judge) => {
|
|
42
|
+
const mine = asked.filter(d => !d.judges.includes(judge));
|
|
43
|
+
if (!mine.length)
|
|
44
|
+
return;
|
|
45
|
+
const cap = input.capped?.();
|
|
46
|
+
if (cap) {
|
|
47
|
+
for (const d of mine) {
|
|
48
|
+
d.calls[judge] = 'not asked';
|
|
49
|
+
d.note = cap;
|
|
50
|
+
}
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
let answers = [];
|
|
54
|
+
try {
|
|
55
|
+
answers = await input.ask(judge, mine.map(d => d.item));
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
answers = [];
|
|
59
|
+
}
|
|
60
|
+
for (const d of mine) {
|
|
61
|
+
const answer = answers.find(x => x.id === d.item.id);
|
|
62
|
+
const evidenced = !!answer?.evidence && input.evidenced(answer.evidence);
|
|
63
|
+
d.calls[judge] = answer && evidenced ? answer.call : 'unsure';
|
|
64
|
+
d.cross = [...(d.cross ?? []), { by: judge, call: d.calls[judge], ...(answer?.evidence ? { evidence: answer.evidence } : {}) }];
|
|
65
|
+
}
|
|
66
|
+
}));
|
|
67
|
+
for (const d of asked)
|
|
68
|
+
d.status = statusOf(d.calls, n);
|
|
69
|
+
return decided.sort((x, y) => order(x.item, y.item));
|
|
70
|
+
}
|
|
71
|
+
/** The answers in a judge's cross-examination reply; nothing usable is no answers (every item then disputed). */
|
|
72
|
+
export function parseAnswers(text) {
|
|
73
|
+
const candidates = [text.trim(), ...[...text.matchAll(/```(?:json)?\s*([\s\S]*?)```/g)].map(m => m[1].trim()), ...[...text.matchAll(/\{\s*"answers"/g)].map(m => text.slice(m.index))];
|
|
74
|
+
for (const candidate of candidates) {
|
|
75
|
+
for (let end = candidate.lastIndexOf('}'); end > 0; end = candidate.lastIndexOf('}', end - 1)) {
|
|
76
|
+
try {
|
|
77
|
+
const parsed = JSON.parse(candidate.slice(0, end + 1));
|
|
78
|
+
if (!Array.isArray(parsed?.answers))
|
|
79
|
+
continue;
|
|
80
|
+
return parsed.answers.filter((a) => typeof a?.id === 'string' && ['confirm', 'refute', 'unsure'].includes(a.call))
|
|
81
|
+
.map((a) => ({ id: a.id, call: a.call, ...(typeof a.evidence === 'string' ? { evidence: a.evidence } : {}) }));
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
// not a whole object yet: try a shorter one
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return [];
|
|
89
|
+
}
|
|
90
|
+
/** The accounting with the panel's decision applied: the judges' own items leave, and only what it confirmed blocks. */
|
|
91
|
+
export function applyPanel(accounted, judgeItemIds, decided) {
|
|
92
|
+
const withJudges = (d) => ({ ...d.item, reviewer: d.judges.join('+') });
|
|
93
|
+
const confirmed = decided.filter(d => d.status === 'confirmed').map(withJudges);
|
|
94
|
+
const open = [...accounted.open.filter(item => !judgeItemIds.has(item.id)), ...confirmed.filter(c => !accounted.open.some(o => o.id === c.id && !judgeItemIds.has(o.id)))];
|
|
95
|
+
return {
|
|
96
|
+
...accounted,
|
|
97
|
+
open,
|
|
98
|
+
disputed: decided.filter(d => d.status === 'disputed').map(withJudges),
|
|
99
|
+
dropped: decided.filter(d => d.status === 'dropped').map(withJudges),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { clusterItems, similarity, statusOf } from './consensus.js';
|
|
3
|
+
import { applyPanel, parseAnswers, runPanel } from './panel.js';
|
|
4
|
+
let n = 0;
|
|
5
|
+
const item = (file, line, cls, issue, consequence = 'wrong rows') => ({ id: `i${++n}`, kind: 'finding', class: cls, file, line, issue, consequence });
|
|
6
|
+
/** Evidence counts only as a real file:line: here, any src/q.ts line. */
|
|
7
|
+
const evidenced = (text) => /src\/q\.ts:\d+/.test(text);
|
|
8
|
+
const members = (judges, items) => clusterItems(judges, items).map(c => c.members.map(m => `${m.judge}:${m.item.issue}`));
|
|
9
|
+
describe('matching findings across judges', () => {
|
|
10
|
+
it('needs the same file, and shared wording unless it is the same line and class', () => {
|
|
11
|
+
const a = item('src/a.ts', 10, 'correctness', 'window not bounded at both ends', 'scans every past event');
|
|
12
|
+
expect(similarity(a, { ...a, file: 'src/b.ts' })).toBe(0);
|
|
13
|
+
expect(similarity(a, item('src/a.ts', 11, 'correctness', 'dedupe key changes when the user edits', 'duplicate emails go out'))).toBe(0);
|
|
14
|
+
expect(similarity(a, item('src/a.ts', 10, 'correctness', 'something else entirely', 'duplicate emails go out'))).toBeGreaterThanOrEqual(0.5);
|
|
15
|
+
});
|
|
16
|
+
it('keeps two different findings on adjacent lines apart, even listed in a misleading order (first-fit by line would cross them)', () => {
|
|
17
|
+
const judgeA = [item('src/q.ts', 10, 'production-cost', 'time window not bounded at both ends'), item('src/q.ts', 11, 'correctness', 'dedupe key changes when the user edits')];
|
|
18
|
+
const judgeB = [item('src/q.ts', 10, 'correctness', 'dedupe key built from updated_at changes when the user edits'), item('src/q.ts', 11, 'production-cost', 'read window has no upper bound at either end')];
|
|
19
|
+
expect(members(['a', 'b'], [judgeA, judgeB])).toEqual([
|
|
20
|
+
['a:time window not bounded at both ends', 'b:read window has no upper bound at either end'],
|
|
21
|
+
['a:dedupe key changes when the user edits', 'b:dedupe key built from updated_at changes when the user edits'],
|
|
22
|
+
]);
|
|
23
|
+
});
|
|
24
|
+
it('matches one bug two judges cite at different lines', () => {
|
|
25
|
+
const judgeA = [item('src/q.ts', 10, 'correctness', 'lock is checked after the first read of rows')];
|
|
26
|
+
const judgeB = [item('src/q.ts', 40, 'correctness', 'the first read of rows happens before the lock is checked')];
|
|
27
|
+
expect(members(['a', 'b'], [judgeA, judgeB])).toHaveLength(1);
|
|
28
|
+
});
|
|
29
|
+
it('puts what one judge split under the one item another judge wrote', () => {
|
|
30
|
+
const judgeA = [item('src/q.ts', 10, 'production-cost', 'both reads, rows and cards, are unbounded')];
|
|
31
|
+
const judgeB = [item('src/q.ts', 10, 'production-cost', 'rows read is unbounded'), item('src/q.ts', 14, 'production-cost', 'cards read is unbounded')];
|
|
32
|
+
expect(members(['a', 'b'], [judgeA, judgeB])).toEqual([['a:both reads, rows and cards, are unbounded', 'b:rows read is unbounded', 'b:cards read is unbounded']]);
|
|
33
|
+
});
|
|
34
|
+
it('never merges one judge\'s own two findings, however close', () => {
|
|
35
|
+
const judgeA = [item('src/x.ts', 10, 'correctness', 'session.user may be null when the token expired'), item('src/x.ts', 12, 'correctness', 'session.expires compared as a string, not a date')];
|
|
36
|
+
expect(members(['a', 'b'], [judgeA, []])).toEqual([['a:session.user may be null when the token expired'], ['a:session.expires compared as a string, not a date']]);
|
|
37
|
+
});
|
|
38
|
+
it('groups three judges, and is the same every run', () => {
|
|
39
|
+
const items = [
|
|
40
|
+
[item('src/q.ts', 10, 'correctness', 'lock checked after the read')],
|
|
41
|
+
[item('src/q.ts', 11, 'correctness', 'the read happens before the lock is checked'), item('src/r.ts', 3, 'dead-code', 'export unused')],
|
|
42
|
+
[item('src/q.ts', 10, 'correctness', 'lock checked only after the read')],
|
|
43
|
+
];
|
|
44
|
+
const once = members(['a', 'b', 'c'], items);
|
|
45
|
+
expect(once).toEqual(members(['a', 'b', 'c'], items));
|
|
46
|
+
expect(once.map(c => c.length)).toEqual([3, 1]);
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
describe('deciding a finding', () => {
|
|
50
|
+
it('confirms on a strict majority, drops on evidence against with nobody unsure, else disputes', () => {
|
|
51
|
+
expect(statusOf({ a: 'raised', b: 'raised' }, 2)).toBe('confirmed');
|
|
52
|
+
expect(statusOf({ a: 'raised', b: 'confirm' }, 2)).toBe('confirmed');
|
|
53
|
+
expect(statusOf({ a: 'raised', b: 'refute' }, 2)).toBe('dropped');
|
|
54
|
+
expect(statusOf({ a: 'raised', b: 'unsure' }, 2)).toBe('disputed');
|
|
55
|
+
expect(statusOf({ a: 'raised', b: 'raised' }, 3)).toBe('confirmed');
|
|
56
|
+
expect(statusOf({ a: 'raised', b: 'confirm', c: 'refute' }, 3)).toBe('confirmed');
|
|
57
|
+
expect(statusOf({ a: 'raised', b: 'refute', c: 'refute' }, 3)).toBe('dropped');
|
|
58
|
+
expect(statusOf({ a: 'raised', b: 'refute', c: 'unsure' }, 3)).toBe('disputed');
|
|
59
|
+
});
|
|
60
|
+
it('asks each judge once about what it did not raise, needs file:line evidence, and caps what it asks', async () => {
|
|
61
|
+
const asked = {};
|
|
62
|
+
const lone = item('src/q.ts', 10, 'correctness', 'lock checked after the read');
|
|
63
|
+
const refuted = item('src/q.ts', 30, 'correctness', 'the cursor skips the last page');
|
|
64
|
+
const bare = item('src/q.ts', 50, 'correctness', 'an index is missing on created_at');
|
|
65
|
+
const overCap = item('src/z.ts', 1, 'correctness', 'retries forever on 429');
|
|
66
|
+
const answers = { b: [{ id: lone.id, call: 'confirm', evidence: 'src/q.ts:12 reads before lock()' }, { id: refuted.id, call: 'refute', evidence: 'src/q.ts:33 the loop runs once more' }, { id: bare.id, call: 'confirm' }] };
|
|
67
|
+
const decided = await runPanel({
|
|
68
|
+
judges: ['a', 'b'], items: [[lone, refuted, bare, overCap], []], previousDisputed: [], touched: new Set(), maxItems: 3, evidenced,
|
|
69
|
+
ask: async (judge, items) => { asked[judge] = items.map(i => i.issue); return answers[judge] ?? []; },
|
|
70
|
+
});
|
|
71
|
+
expect(asked).toEqual({ b: ['lock checked after the read', 'the cursor skips the last page', 'an index is missing on created_at'] });
|
|
72
|
+
const status = Object.fromEntries(decided.map(d => [d.item.issue, d.status]));
|
|
73
|
+
expect(status).toEqual({ 'lock checked after the read': 'confirmed', 'the cursor skips the last page': 'dropped', 'an index is missing on created_at': 'disputed', 'retries forever on 429': 'disputed' });
|
|
74
|
+
expect(decided.find(d => d.item.id === overCap.id)?.note).toContain('panel_max_items');
|
|
75
|
+
expect(decided.find(d => d.item.id === lone.id)).toMatchObject({ judges: ['a'], calls: { a: 'raised', b: 'confirm' }, cross: [{ by: 'b', call: 'confirm', evidence: 'src/q.ts:12 reads before lock()' }] });
|
|
76
|
+
});
|
|
77
|
+
it('does not ask again about a finding disputed before on a file nothing has touched since', async () => {
|
|
78
|
+
const again = item('src/q.ts', 10, 'correctness', 'lock checked after the read');
|
|
79
|
+
const asked = [];
|
|
80
|
+
const ask = async (_judge, items) => { asked.push(...items.map(i => i.id)); return []; };
|
|
81
|
+
const untouched = await runPanel({ judges: ['a', 'b'], items: [[again], []], previousDisputed: [{ ...again, id: 'old' }], touched: new Set(), maxItems: 20, evidenced, ask });
|
|
82
|
+
expect(untouched[0]).toMatchObject({ status: 'disputed', note: 'disputed before; its file is unchanged since' });
|
|
83
|
+
expect(asked).toEqual([]);
|
|
84
|
+
await runPanel({ judges: ['a', 'b'], items: [[again], []], previousDisputed: [{ ...again, id: 'old' }], touched: new Set(['src/q.ts']), maxItems: 20, evidenced, ask });
|
|
85
|
+
expect(asked).toEqual([again.id]);
|
|
86
|
+
});
|
|
87
|
+
it('a judge that fails to answer leaves the findings disputed, never confirmed', async () => {
|
|
88
|
+
const lone = item('src/q.ts', 10, 'correctness', 'lock checked after the read');
|
|
89
|
+
const decided = await runPanel({ judges: ['a', 'b'], items: [[lone], []], previousDisputed: [], touched: new Set(), maxItems: 20, evidenced, ask: async () => { throw new Error('timed out'); } });
|
|
90
|
+
expect(decided[0]).toMatchObject({ status: 'disputed', calls: { a: 'raised', b: 'unsure' } });
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
describe('the panel in the accounting', () => {
|
|
94
|
+
it('blocks only on what it confirmed, and lists the rest', () => {
|
|
95
|
+
const confirmed = item('src/q.ts', 10, 'correctness', 'lock checked after the read');
|
|
96
|
+
const disputed = item('src/q.ts', 30, 'correctness', 'cursor skips a page');
|
|
97
|
+
const human = { ...item('', 0, 'prior point', 'a human point'), kind: 'prior' };
|
|
98
|
+
const result = applyPanel({ open: [confirmed, disputed, human] }, new Set([confirmed.id, disputed.id]), [
|
|
99
|
+
{ item: confirmed, judges: ['a', 'b'], calls: { a: 'raised', b: 'raised' }, status: 'confirmed' },
|
|
100
|
+
{ item: disputed, judges: ['a'], calls: { a: 'raised', b: 'unsure' }, status: 'disputed' },
|
|
101
|
+
]);
|
|
102
|
+
expect(result.open.map(i => i.issue)).toEqual(['a human point', 'lock checked after the read']);
|
|
103
|
+
expect(result.open[1].reviewer).toBe('a+b');
|
|
104
|
+
expect(result.disputed.map(i => i.issue)).toEqual(['cursor skips a page']);
|
|
105
|
+
});
|
|
106
|
+
it('reads the answers out of a reply with prose around them, and nothing from a reply without them', () => {
|
|
107
|
+
expect(parseAnswers('Checked.\n```json\n{"answers":[{"id":"x","call":"refute","evidence":"a.ts:3"},{"id":"y","call":"maybe"}]}\n```')).toEqual([{ id: 'x', call: 'refute', evidence: 'a.ts:3' }]);
|
|
108
|
+
expect(parseAnswers('I could not decide.')).toEqual([]);
|
|
109
|
+
});
|
|
110
|
+
});
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export interface PromptInputs {
|
|
2
|
+
repoRoot: string;
|
|
3
|
+
branch: string;
|
|
4
|
+
head: string;
|
|
5
|
+
base: string;
|
|
6
|
+
baseSha: string;
|
|
7
|
+
mode: 'full' | 'delta';
|
|
8
|
+
reviewsFile: string;
|
|
9
|
+
humanCount: number;
|
|
10
|
+
prBodyFile: string;
|
|
11
|
+
diffstatFile: string;
|
|
12
|
+
diffFile: string;
|
|
13
|
+
hintsFile: string;
|
|
14
|
+
/** What the team already knows (reviewer/context.ts). */
|
|
15
|
+
contextFile: string;
|
|
16
|
+
deltaBlock: string;
|
|
17
|
+
mergeBlock: string;
|
|
18
|
+
}
|
|
19
|
+
export declare function renderPrompt(v: PromptInputs): string;
|
|
20
|
+
export declare function deltaBlock(previousHead: string, previousVerdict: string, previousOpenFile: string, commitsFile: string, deltaDiffFile: string, settledFile: string): string;
|
|
21
|
+
export declare function mergeBlock(base: string, impactFile: string | undefined): string;
|
|
22
|
+
/** Changes when the instructions change, so a cached verdict from older instructions is not reused. */
|
|
23
|
+
export declare const PROMPT_VERSION: string;
|
|
24
|
+
/** One judge's single call on the items the other judge raised alone: confirm or refute each, with the code that shows it. */
|
|
25
|
+
export declare function crossExamPrompt(repoRoot: string, head: string, diffFile: string, items: Array<{
|
|
26
|
+
id: string;
|
|
27
|
+
class: string;
|
|
28
|
+
file?: string;
|
|
29
|
+
line?: number;
|
|
30
|
+
issue: string;
|
|
31
|
+
consequence?: string;
|
|
32
|
+
}>): string;
|