@olegkoval/agent-skills 1.64.0 → 1.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapters/claude/olko-github-pr/skills/lekker-review/SKILL.md +8 -0
- package/adapters/claude/olko-github-pr/skills/lekker-review/references/github-post.md +6 -0
- package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/human-noise.mjs +194 -0
- package/package.json +1 -1
- package/plugins/olko-apple-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-creative/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-garmin-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-git-tools/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/skills/lekker-review/SKILL.md +8 -0
- package/plugins/olko-github-pr/skills/lekker-review/references/github-post.md +6 -0
- package/plugins/olko-github-pr/skills/lekker-review/scripts/human-noise.mjs +194 -0
- package/plugins/olko-github-pr/skills/lekker-review/workflow.js +19 -1
- package/plugins/olko-obsidian/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-product/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-reflection/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-release/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-skill-meta/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-web-ops/.claude-plugin/plugin.json +1 -1
|
@@ -1325,6 +1325,14 @@ is not failure, the review may find things the bots never would. `0 bot
|
|
|
1325
1325
|
comment(s)` is ambiguous (nothing found, or the bots have not run yet); never
|
|
1326
1326
|
turn it into a 100% claim. Store: `~/.cache/lekker-review/bot-precision.jsonl`.
|
|
1327
1327
|
|
|
1328
|
+
Bots are not the only signal. To measure noise from what humans did, run
|
|
1329
|
+
`node ${SKILL_ROOT}/scripts/human-noise.mjs --repo <REPO_SLUG> --pr <N> [--pr <N>...]`
|
|
1330
|
+
on merged or settled PRs: it classifies each posted lekker inline comment as
|
|
1331
|
+
fixed (thread outdated), dismissed (affirmative author reply), resolved (code
|
|
1332
|
+
outcome unknown) or ignored, and prints the dismissal rate per agent. Resolved
|
|
1333
|
+
threads remain in the total but do not count as dismissed. Only comments ending in the
|
|
1334
|
+
`<!-- lekker:<dimension> -->` marker (see `references/github-post.md`) are
|
|
1335
|
+
counted, so PRs posted before the marker existed score zero comments.
|
|
1328
1336
|
**Eval harness.** `evals/manifest.json` lists PRs with a known outcome (`expect:
|
|
1329
1337
|
"buggy"` with `knownBug: "file:line description"`, or `"clean"`). Run lekker on
|
|
1330
1338
|
each, save the `findings` arrays as `{"<repo>#<pr>": [...]}`, then
|
|
@@ -33,6 +33,12 @@ Rules:
|
|
|
33
33
|
determine commentability). A finding with `line: null` (set by
|
|
34
34
|
`scripts/validate-lines.mjs` when its quoted code is not in the file) is
|
|
35
35
|
never an inline comment: list it in the review body instead.
|
|
36
|
+
- **End every inline comment body with the marker** `<!-- lekker:<dimension> -->`
|
|
37
|
+
on its own last line, using the finding's `dimension` (`quality`,
|
|
38
|
+
`implementation`, `simplification`, `conventions`, `test-quality`, `critic`;
|
|
39
|
+
`unknown` when absent). It is invisible on GitHub, and is the only way
|
|
40
|
+
`scripts/human-noise.mjs` can tell lekker's comments from the author's own,
|
|
41
|
+
because the pending review is posted under the requesting user's account.
|
|
36
42
|
- **Keep every comment body to 2-4 sentences and under ~700 characters**
|
|
37
43
|
(excluding any suggestion block). Inline comments are read one at a time in a
|
|
38
44
|
narrow column; a long one does not get read, so length actively costs you the
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Measure lekker's noise from what humans did with its comments.
|
|
3
|
+
//
|
|
4
|
+
// bot-precision.mjs only compares predictions with other bots. The honest signal
|
|
5
|
+
// is whether the author acted on a comment, so this reads lekker's own posted
|
|
6
|
+
// inline comments and classifies each one:
|
|
7
|
+
// fixed the thread is outdated: the code at that location changed after the comment
|
|
8
|
+
// dismissed the PR author replied with a dismissal
|
|
9
|
+
// resolved resolved, but no code outcome is known
|
|
10
|
+
// ignored none of the above (no code change, no dismissal, still open)
|
|
11
|
+
// and reports the dismissal rate per agent.
|
|
12
|
+
//
|
|
13
|
+
// Identification: the pending review is posted under the requesting user's own
|
|
14
|
+
// account, so the author cannot tell lekker from a human. github-post.md makes
|
|
15
|
+
// every inline comment end with `<!-- lekker:<dimension> -->`; only threads whose
|
|
16
|
+
// first comment carries that marker are counted.
|
|
17
|
+
//
|
|
18
|
+
// Usage:
|
|
19
|
+
// node human-noise.mjs --repo <owner/repo> --pr <n> [--pr <n>...] [--json]
|
|
20
|
+
// node human-noise.mjs --from <threads.json> [--json]
|
|
21
|
+
// threads.json: [{ prAuthor, threads: [{ path, line, isResolved, isOutdated,
|
|
22
|
+
// comments: [{ author, body }] }] }] (offline fixture / saved fetch)
|
|
23
|
+
|
|
24
|
+
import { execFileSync } from 'node:child_process';
|
|
25
|
+
import { readFileSync } from 'node:fs';
|
|
26
|
+
import { fileURLToPath } from 'node:url';
|
|
27
|
+
|
|
28
|
+
const MARKER = /<!--\s*lekker:([a-z0-9-]+)\s*-->/i;
|
|
29
|
+
export const AGENTS = ['quality', 'implementation', 'simplification', 'conventions', 'test-quality'];
|
|
30
|
+
|
|
31
|
+
const INTENT = /\b(intentional(ly)?|by design|on purpose|deliberate(ly)?|as designed|expected behaviou?r)\b/i;
|
|
32
|
+
const DISMISS = /\b(won'?t fix|wontfix|not a (bug|problem|concern)|false positive|disagree|n\/a|not applicable|out of scope|ignor(e|ing)|nit)\b/i;
|
|
33
|
+
const NEGATOR = /\b(not|no|never|nor|isn'?t|aren'?t|wasn'?t|don'?t|doesn'?t|won'?t|cannot)\b/i;
|
|
34
|
+
|
|
35
|
+
// Check only the three words before each keyword in its clause for negation.
|
|
36
|
+
function mentions(pattern, text) {
|
|
37
|
+
return [...text.matchAll(new RegExp(pattern.source, 'gi'))]
|
|
38
|
+
.some((match) => {
|
|
39
|
+
const before = text.slice(0, match.index).split(/[.!?;:,\n-]/).pop();
|
|
40
|
+
return !NEGATOR.test(before.trim().split(/\s+/).slice(-3).join(' '));
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Self-contained on purpose: this script must not depend on other lekker scripts.
|
|
45
|
+
export function classifyThread(thread, prAuthor) {
|
|
46
|
+
const text = (thread.replies || [])
|
|
47
|
+
.filter((r) => r.author && r.author === prAuthor)
|
|
48
|
+
.map((r) => r.body)
|
|
49
|
+
.join('\n');
|
|
50
|
+
if (thread.isOutdated) return 'fixed';
|
|
51
|
+
if (mentions(INTENT, text) || mentions(DISMISS, text)) return 'dismissed';
|
|
52
|
+
if (thread.isResolved) return 'resolved';
|
|
53
|
+
return 'ignored';
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function agentOf(body) {
|
|
57
|
+
const m = MARKER.exec(String(body || ''));
|
|
58
|
+
return m ? m[1].toLowerCase() : null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function classifyLekkerThread(thread, prAuthor) {
|
|
62
|
+
const [first, ...rest] = thread.comments || [];
|
|
63
|
+
const agent = first ? agentOf(first.body) : null;
|
|
64
|
+
if (!agent) return null;
|
|
65
|
+
const outcome = classifyThread({
|
|
66
|
+
isOutdated: thread.isOutdated,
|
|
67
|
+
isResolved: thread.isResolved,
|
|
68
|
+
replies: rest,
|
|
69
|
+
}, prAuthor);
|
|
70
|
+
return { agent, path: thread.path, line: thread.line, outcome };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function summarize(prs) {
|
|
74
|
+
const rows = prs.flatMap((pr) => (pr.threads || [])
|
|
75
|
+
.map((t) => classifyLekkerThread(t, pr.prAuthor))
|
|
76
|
+
.filter(Boolean));
|
|
77
|
+
const byAgent = rows.reduce((acc, r) => {
|
|
78
|
+
const cur = acc[r.agent] || { fixed: 0, dismissed: 0, ignored: 0, resolved: 0 };
|
|
79
|
+
acc[r.agent] = { ...cur, [r.outcome]: cur[r.outcome] + 1 };
|
|
80
|
+
return acc;
|
|
81
|
+
}, Object.create(null));
|
|
82
|
+
const agents = Object.entries(byAgent).map(([agent, c]) => {
|
|
83
|
+
const total = c.fixed + c.dismissed + c.ignored;
|
|
84
|
+
const allComments = total + c.resolved;
|
|
85
|
+
return {
|
|
86
|
+
agent,
|
|
87
|
+
total: allComments,
|
|
88
|
+
...c,
|
|
89
|
+
dismissalRate: total === 0 ? null : c.dismissed / total,
|
|
90
|
+
};
|
|
91
|
+
}).sort((a, b) => AGENTS.indexOf(a.agent) - AGENTS.indexOf(b.agent) || a.agent.localeCompare(b.agent));
|
|
92
|
+
return { comments: rows.length, agents };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const FETCH_TIMEOUT_MS = 30_000;
|
|
96
|
+
const THREADS_QUERY = `query($owner:String!,$name:String!,$pr:Int!,$threadCursor:String){repository(owner:$owner,name:$name){
|
|
97
|
+
pullRequest(number:$pr){author{login} reviewThreads(first:100,after:$threadCursor){nodes{id isResolved isOutdated path line
|
|
98
|
+
comments(first:50){nodes{author{login} body} pageInfo{hasNextPage endCursor}}} pageInfo{hasNextPage endCursor}}}}}`;
|
|
99
|
+
const COMMENTS_QUERY = `query($threadId:ID!,$commentCursor:String){node(id:$threadId){... on PullRequestReviewThread{
|
|
100
|
+
comments(first:50,after:$commentCursor){nodes{author{login} body} pageInfo{hasNextPage endCursor}}}}}`;
|
|
101
|
+
|
|
102
|
+
function graphql(repo, pr, query, cursors, run) {
|
|
103
|
+
const [owner, name] = repo.split('/');
|
|
104
|
+
const args = ['api', 'graphql', '-f', `query=${query}`, '-F', `owner=${owner}`,
|
|
105
|
+
'-F', `name=${name}`, '-F', `pr=${pr}`];
|
|
106
|
+
for (const [key, value] of Object.entries(cursors)) {
|
|
107
|
+
if (value !== undefined) args.push('-f', `${key}=${value}`);
|
|
108
|
+
}
|
|
109
|
+
let output;
|
|
110
|
+
try {
|
|
111
|
+
output = run('gh', args, { encoding: 'utf8', timeout: FETCH_TIMEOUT_MS });
|
|
112
|
+
} catch (error) {
|
|
113
|
+
if (error && typeof error === 'object' && error.code === 'ETIMEDOUT') {
|
|
114
|
+
throw new Error(`GitHub GraphQL fetch timed out after ${FETCH_TIMEOUT_MS / 1000}s for ${repo}#${pr}`, { cause: error });
|
|
115
|
+
}
|
|
116
|
+
throw error;
|
|
117
|
+
}
|
|
118
|
+
return JSON.parse(output).data;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function nextCursor(pageInfo, connection) {
|
|
122
|
+
if (!pageInfo.hasNextPage) return null;
|
|
123
|
+
if (!pageInfo.endCursor) throw new Error(`GitHub GraphQL ${connection} page has no end cursor`);
|
|
124
|
+
return pageInfo.endCursor;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function allComments(repo, pr, thread, run) {
|
|
128
|
+
const comments = thread.comments.nodes.map((c) => ({
|
|
129
|
+
author: c.author ? c.author.login : null,
|
|
130
|
+
body: c.body,
|
|
131
|
+
}));
|
|
132
|
+
let cursor = nextCursor(thread.comments.pageInfo, 'comments');
|
|
133
|
+
while (cursor) {
|
|
134
|
+
const data = graphql(repo, pr, COMMENTS_QUERY, { threadId: thread.id, commentCursor: cursor }, run);
|
|
135
|
+
const page = data.node.comments;
|
|
136
|
+
comments.push(...page.nodes.map((c) => ({
|
|
137
|
+
author: c.author ? c.author.login : null,
|
|
138
|
+
body: c.body,
|
|
139
|
+
})));
|
|
140
|
+
cursor = nextCursor(page.pageInfo, 'comments');
|
|
141
|
+
}
|
|
142
|
+
return comments;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function fetchPr(repo, pr, run = execFileSync) {
|
|
146
|
+
const threads = [];
|
|
147
|
+
let cursor;
|
|
148
|
+
let prAuthor = null;
|
|
149
|
+
do {
|
|
150
|
+
const data = graphql(repo, pr, THREADS_QUERY, { threadCursor: cursor }, run);
|
|
151
|
+
const pullRequest = data.repository.pullRequest;
|
|
152
|
+
if (pullRequest.author) prAuthor = pullRequest.author.login;
|
|
153
|
+
const page = pullRequest.reviewThreads;
|
|
154
|
+
threads.push(...page.nodes.map((thread) => ({
|
|
155
|
+
path: thread.path,
|
|
156
|
+
line: thread.line,
|
|
157
|
+
isResolved: thread.isResolved,
|
|
158
|
+
isOutdated: thread.isOutdated,
|
|
159
|
+
comments: allComments(repo, pr, thread, run),
|
|
160
|
+
})));
|
|
161
|
+
cursor = nextCursor(page.pageInfo, 'reviewThreads');
|
|
162
|
+
} while (cursor);
|
|
163
|
+
return {
|
|
164
|
+
prAuthor,
|
|
165
|
+
threads,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export function fetchPrs(repo, prNumbers, fetch = fetchPr) {
|
|
170
|
+
return Array.from(new Set(prNumbers), (pr) => fetch(repo, pr));
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function render(s) {
|
|
174
|
+
const pct = (r) => (r === null ? 'n/a' : `${Math.round(r * 100)}%`);
|
|
175
|
+
const lines = s.agents.map((a) =>
|
|
176
|
+
`${a.agent.padEnd(15)} ${String(a.total).padStart(3)} comments fixed ${a.fixed} dismissed ${a.dismissed} ignored ${a.ignored} resolved ${a.resolved} dismissal rate ${pct(a.dismissalRate)}`);
|
|
177
|
+
return [`lekker inline comments scored: ${s.comments}`, ...lines].join('\n');
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function main(argv) {
|
|
181
|
+
const values = (name) => argv.flatMap((a, i) => (a === '--' + name && argv[i + 1] ? [argv[i + 1]] : []));
|
|
182
|
+
const from = values('from')[0];
|
|
183
|
+
const repo = values('repo')[0];
|
|
184
|
+
const prNumbers = values('pr');
|
|
185
|
+
if (!from && !(repo && prNumbers.length)) {
|
|
186
|
+
console.error('usage: human-noise.mjs --repo <owner/repo> --pr <n> [--pr <n>...] | --from <threads.json> [--json]');
|
|
187
|
+
process.exit(2);
|
|
188
|
+
}
|
|
189
|
+
const prs = from ? JSON.parse(readFileSync(from, 'utf8')) : fetchPrs(repo, prNumbers);
|
|
190
|
+
const summary = summarize(prs);
|
|
191
|
+
console.log(argv.includes('--json') ? JSON.stringify(summary) : render(summary));
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
|
package/package.json
CHANGED
|
@@ -1324,6 +1324,14 @@ is not failure, the review may find things the bots never would. `0 bot
|
|
|
1324
1324
|
comment(s)` is ambiguous (nothing found, or the bots have not run yet); never
|
|
1325
1325
|
turn it into a 100% claim. Store: `~/.cache/lekker-review/bot-precision.jsonl`.
|
|
1326
1326
|
|
|
1327
|
+
Bots are not the only signal. To measure noise from what humans did, run
|
|
1328
|
+
`node ${SKILL_ROOT}/scripts/human-noise.mjs --repo <REPO_SLUG> --pr <N> [--pr <N>...]`
|
|
1329
|
+
on merged or settled PRs: it classifies each posted lekker inline comment as
|
|
1330
|
+
fixed (thread outdated), dismissed (affirmative author reply), resolved (code
|
|
1331
|
+
outcome unknown) or ignored, and prints the dismissal rate per agent. Resolved
|
|
1332
|
+
threads remain in the total but do not count as dismissed. Only comments ending in the
|
|
1333
|
+
`<!-- lekker:<dimension> -->` marker (see `references/github-post.md`) are
|
|
1334
|
+
counted, so PRs posted before the marker existed score zero comments.
|
|
1327
1335
|
**Eval harness.** `evals/manifest.json` lists PRs with a known outcome (`expect:
|
|
1328
1336
|
"buggy"` with `knownBug: "file:line description"`, or `"clean"`). Run lekker on
|
|
1329
1337
|
each, save the `findings` arrays as `{"<repo>#<pr>": [...]}`, then
|
|
@@ -33,6 +33,12 @@ Rules:
|
|
|
33
33
|
determine commentability). A finding with `line: null` (set by
|
|
34
34
|
`scripts/validate-lines.mjs` when its quoted code is not in the file) is
|
|
35
35
|
never an inline comment: list it in the review body instead.
|
|
36
|
+
- **End every inline comment body with the marker** `<!-- lekker:<dimension> -->`
|
|
37
|
+
on its own last line, using the finding's `dimension` (`quality`,
|
|
38
|
+
`implementation`, `simplification`, `conventions`, `test-quality`, `critic`;
|
|
39
|
+
`unknown` when absent). It is invisible on GitHub, and is the only way
|
|
40
|
+
`scripts/human-noise.mjs` can tell lekker's comments from the author's own,
|
|
41
|
+
because the pending review is posted under the requesting user's account.
|
|
36
42
|
- **Keep every comment body to 2-4 sentences and under ~700 characters**
|
|
37
43
|
(excluding any suggestion block). Inline comments are read one at a time in a
|
|
38
44
|
narrow column; a long one does not get read, so length actively costs you the
|
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Measure lekker's noise from what humans did with its comments.
|
|
3
|
+
//
|
|
4
|
+
// bot-precision.mjs only compares predictions with other bots. The honest signal
|
|
5
|
+
// is whether the author acted on a comment, so this reads lekker's own posted
|
|
6
|
+
// inline comments and classifies each one:
|
|
7
|
+
// fixed the thread is outdated: the code at that location changed after the comment
|
|
8
|
+
// dismissed the PR author replied with a dismissal
|
|
9
|
+
// resolved resolved, but no code outcome is known
|
|
10
|
+
// ignored none of the above (no code change, no dismissal, still open)
|
|
11
|
+
// and reports the dismissal rate per agent.
|
|
12
|
+
//
|
|
13
|
+
// Identification: the pending review is posted under the requesting user's own
|
|
14
|
+
// account, so the author cannot tell lekker from a human. github-post.md makes
|
|
15
|
+
// every inline comment end with `<!-- lekker:<dimension> -->`; only threads whose
|
|
16
|
+
// first comment carries that marker are counted.
|
|
17
|
+
//
|
|
18
|
+
// Usage:
|
|
19
|
+
// node human-noise.mjs --repo <owner/repo> --pr <n> [--pr <n>...] [--json]
|
|
20
|
+
// node human-noise.mjs --from <threads.json> [--json]
|
|
21
|
+
// threads.json: [{ prAuthor, threads: [{ path, line, isResolved, isOutdated,
|
|
22
|
+
// comments: [{ author, body }] }] }] (offline fixture / saved fetch)
|
|
23
|
+
|
|
24
|
+
import { execFileSync } from 'node:child_process';
|
|
25
|
+
import { readFileSync } from 'node:fs';
|
|
26
|
+
import { fileURLToPath } from 'node:url';
|
|
27
|
+
|
|
28
|
+
const MARKER = /<!--\s*lekker:([a-z0-9-]+)\s*-->/i;
|
|
29
|
+
export const AGENTS = ['quality', 'implementation', 'simplification', 'conventions', 'test-quality'];
|
|
30
|
+
|
|
31
|
+
const INTENT = /\b(intentional(ly)?|by design|on purpose|deliberate(ly)?|as designed|expected behaviou?r)\b/i;
|
|
32
|
+
const DISMISS = /\b(won'?t fix|wontfix|not a (bug|problem|concern)|false positive|disagree|n\/a|not applicable|out of scope|ignor(e|ing)|nit)\b/i;
|
|
33
|
+
const NEGATOR = /\b(not|no|never|nor|isn'?t|aren'?t|wasn'?t|don'?t|doesn'?t|won'?t|cannot)\b/i;
|
|
34
|
+
|
|
35
|
+
// Check only the three words before each keyword in its clause for negation.
|
|
36
|
+
function mentions(pattern, text) {
|
|
37
|
+
return [...text.matchAll(new RegExp(pattern.source, 'gi'))]
|
|
38
|
+
.some((match) => {
|
|
39
|
+
const before = text.slice(0, match.index).split(/[.!?;:,\n-]/).pop();
|
|
40
|
+
return !NEGATOR.test(before.trim().split(/\s+/).slice(-3).join(' '));
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Self-contained on purpose: this script must not depend on other lekker scripts.
|
|
45
|
+
export function classifyThread(thread, prAuthor) {
|
|
46
|
+
const text = (thread.replies || [])
|
|
47
|
+
.filter((r) => r.author && r.author === prAuthor)
|
|
48
|
+
.map((r) => r.body)
|
|
49
|
+
.join('\n');
|
|
50
|
+
if (thread.isOutdated) return 'fixed';
|
|
51
|
+
if (mentions(INTENT, text) || mentions(DISMISS, text)) return 'dismissed';
|
|
52
|
+
if (thread.isResolved) return 'resolved';
|
|
53
|
+
return 'ignored';
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function agentOf(body) {
|
|
57
|
+
const m = MARKER.exec(String(body || ''));
|
|
58
|
+
return m ? m[1].toLowerCase() : null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function classifyLekkerThread(thread, prAuthor) {
|
|
62
|
+
const [first, ...rest] = thread.comments || [];
|
|
63
|
+
const agent = first ? agentOf(first.body) : null;
|
|
64
|
+
if (!agent) return null;
|
|
65
|
+
const outcome = classifyThread({
|
|
66
|
+
isOutdated: thread.isOutdated,
|
|
67
|
+
isResolved: thread.isResolved,
|
|
68
|
+
replies: rest,
|
|
69
|
+
}, prAuthor);
|
|
70
|
+
return { agent, path: thread.path, line: thread.line, outcome };
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function summarize(prs) {
|
|
74
|
+
const rows = prs.flatMap((pr) => (pr.threads || [])
|
|
75
|
+
.map((t) => classifyLekkerThread(t, pr.prAuthor))
|
|
76
|
+
.filter(Boolean));
|
|
77
|
+
const byAgent = rows.reduce((acc, r) => {
|
|
78
|
+
const cur = acc[r.agent] || { fixed: 0, dismissed: 0, ignored: 0, resolved: 0 };
|
|
79
|
+
acc[r.agent] = { ...cur, [r.outcome]: cur[r.outcome] + 1 };
|
|
80
|
+
return acc;
|
|
81
|
+
}, Object.create(null));
|
|
82
|
+
const agents = Object.entries(byAgent).map(([agent, c]) => {
|
|
83
|
+
const total = c.fixed + c.dismissed + c.ignored;
|
|
84
|
+
const allComments = total + c.resolved;
|
|
85
|
+
return {
|
|
86
|
+
agent,
|
|
87
|
+
total: allComments,
|
|
88
|
+
...c,
|
|
89
|
+
dismissalRate: total === 0 ? null : c.dismissed / total,
|
|
90
|
+
};
|
|
91
|
+
}).sort((a, b) => AGENTS.indexOf(a.agent) - AGENTS.indexOf(b.agent) || a.agent.localeCompare(b.agent));
|
|
92
|
+
return { comments: rows.length, agents };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
const FETCH_TIMEOUT_MS = 30_000;
|
|
96
|
+
const THREADS_QUERY = `query($owner:String!,$name:String!,$pr:Int!,$threadCursor:String){repository(owner:$owner,name:$name){
|
|
97
|
+
pullRequest(number:$pr){author{login} reviewThreads(first:100,after:$threadCursor){nodes{id isResolved isOutdated path line
|
|
98
|
+
comments(first:50){nodes{author{login} body} pageInfo{hasNextPage endCursor}}} pageInfo{hasNextPage endCursor}}}}}`;
|
|
99
|
+
const COMMENTS_QUERY = `query($threadId:ID!,$commentCursor:String){node(id:$threadId){... on PullRequestReviewThread{
|
|
100
|
+
comments(first:50,after:$commentCursor){nodes{author{login} body} pageInfo{hasNextPage endCursor}}}}}`;
|
|
101
|
+
|
|
102
|
+
function graphql(repo, pr, query, cursors, run) {
|
|
103
|
+
const [owner, name] = repo.split('/');
|
|
104
|
+
const args = ['api', 'graphql', '-f', `query=${query}`, '-F', `owner=${owner}`,
|
|
105
|
+
'-F', `name=${name}`, '-F', `pr=${pr}`];
|
|
106
|
+
for (const [key, value] of Object.entries(cursors)) {
|
|
107
|
+
if (value !== undefined) args.push('-f', `${key}=${value}`);
|
|
108
|
+
}
|
|
109
|
+
let output;
|
|
110
|
+
try {
|
|
111
|
+
output = run('gh', args, { encoding: 'utf8', timeout: FETCH_TIMEOUT_MS });
|
|
112
|
+
} catch (error) {
|
|
113
|
+
if (error && typeof error === 'object' && error.code === 'ETIMEDOUT') {
|
|
114
|
+
throw new Error(`GitHub GraphQL fetch timed out after ${FETCH_TIMEOUT_MS / 1000}s for ${repo}#${pr}`, { cause: error });
|
|
115
|
+
}
|
|
116
|
+
throw error;
|
|
117
|
+
}
|
|
118
|
+
return JSON.parse(output).data;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function nextCursor(pageInfo, connection) {
|
|
122
|
+
if (!pageInfo.hasNextPage) return null;
|
|
123
|
+
if (!pageInfo.endCursor) throw new Error(`GitHub GraphQL ${connection} page has no end cursor`);
|
|
124
|
+
return pageInfo.endCursor;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function allComments(repo, pr, thread, run) {
|
|
128
|
+
const comments = thread.comments.nodes.map((c) => ({
|
|
129
|
+
author: c.author ? c.author.login : null,
|
|
130
|
+
body: c.body,
|
|
131
|
+
}));
|
|
132
|
+
let cursor = nextCursor(thread.comments.pageInfo, 'comments');
|
|
133
|
+
while (cursor) {
|
|
134
|
+
const data = graphql(repo, pr, COMMENTS_QUERY, { threadId: thread.id, commentCursor: cursor }, run);
|
|
135
|
+
const page = data.node.comments;
|
|
136
|
+
comments.push(...page.nodes.map((c) => ({
|
|
137
|
+
author: c.author ? c.author.login : null,
|
|
138
|
+
body: c.body,
|
|
139
|
+
})));
|
|
140
|
+
cursor = nextCursor(page.pageInfo, 'comments');
|
|
141
|
+
}
|
|
142
|
+
return comments;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function fetchPr(repo, pr, run = execFileSync) {
|
|
146
|
+
const threads = [];
|
|
147
|
+
let cursor;
|
|
148
|
+
let prAuthor = null;
|
|
149
|
+
do {
|
|
150
|
+
const data = graphql(repo, pr, THREADS_QUERY, { threadCursor: cursor }, run);
|
|
151
|
+
const pullRequest = data.repository.pullRequest;
|
|
152
|
+
if (pullRequest.author) prAuthor = pullRequest.author.login;
|
|
153
|
+
const page = pullRequest.reviewThreads;
|
|
154
|
+
threads.push(...page.nodes.map((thread) => ({
|
|
155
|
+
path: thread.path,
|
|
156
|
+
line: thread.line,
|
|
157
|
+
isResolved: thread.isResolved,
|
|
158
|
+
isOutdated: thread.isOutdated,
|
|
159
|
+
comments: allComments(repo, pr, thread, run),
|
|
160
|
+
})));
|
|
161
|
+
cursor = nextCursor(page.pageInfo, 'reviewThreads');
|
|
162
|
+
} while (cursor);
|
|
163
|
+
return {
|
|
164
|
+
prAuthor,
|
|
165
|
+
threads,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export function fetchPrs(repo, prNumbers, fetch = fetchPr) {
|
|
170
|
+
return Array.from(new Set(prNumbers), (pr) => fetch(repo, pr));
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function render(s) {
|
|
174
|
+
const pct = (r) => (r === null ? 'n/a' : `${Math.round(r * 100)}%`);
|
|
175
|
+
const lines = s.agents.map((a) =>
|
|
176
|
+
`${a.agent.padEnd(15)} ${String(a.total).padStart(3)} comments fixed ${a.fixed} dismissed ${a.dismissed} ignored ${a.ignored} resolved ${a.resolved} dismissal rate ${pct(a.dismissalRate)}`);
|
|
177
|
+
return [`lekker inline comments scored: ${s.comments}`, ...lines].join('\n');
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function main(argv) {
|
|
181
|
+
const values = (name) => argv.flatMap((a, i) => (a === '--' + name && argv[i + 1] ? [argv[i + 1]] : []));
|
|
182
|
+
const from = values('from')[0];
|
|
183
|
+
const repo = values('repo')[0];
|
|
184
|
+
const prNumbers = values('pr');
|
|
185
|
+
if (!from && !(repo && prNumbers.length)) {
|
|
186
|
+
console.error('usage: human-noise.mjs --repo <owner/repo> --pr <n> [--pr <n>...] | --from <threads.json> [--json]');
|
|
187
|
+
process.exit(2);
|
|
188
|
+
}
|
|
189
|
+
const prs = from ? JSON.parse(readFileSync(from, 'utf8')) : fetchPrs(repo, prNumbers);
|
|
190
|
+
const summary = summarize(prs);
|
|
191
|
+
console.log(argv.includes('--json') ? JSON.stringify(summary) : render(summary));
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
|
|
@@ -1037,9 +1037,27 @@ const budgetAtStart = budget.spent()
|
|
|
1037
1037
|
}
|
|
1038
1038
|
}
|
|
1039
1039
|
|
|
1040
|
-
//
|
|
1040
|
+
// Expose the originating agent as `dimension` (critic angles collapse to
|
|
1041
|
+
// "critic"). The posted inline comment carries it as a marker so
|
|
1042
|
+
// scripts/human-noise.mjs can score each agent by what humans did with it.
|
|
1043
|
+
const revmuxDimensions = {
|
|
1044
|
+
'lekker-quality': 'quality',
|
|
1045
|
+
'lekker-implementation': 'implementation',
|
|
1046
|
+
'lekker-simplification': 'simplification',
|
|
1047
|
+
'lekker-conventions': 'conventions',
|
|
1048
|
+
'lekker-test-quality': 'test-quality',
|
|
1049
|
+
tests: 'test-quality',
|
|
1050
|
+
}
|
|
1041
1051
|
const output = finalFindings.map(function(f) {
|
|
1042
1052
|
const clean = Object.assign({}, f)
|
|
1053
|
+
const dimension = clean._dimension || clean.dimension
|
|
1054
|
+
if (Array.isArray(dimension)) {
|
|
1055
|
+
clean.dimension = dimension.map(function(lens) {
|
|
1056
|
+
return revmuxDimensions[lens]
|
|
1057
|
+
}).find(Boolean) || 'unknown'
|
|
1058
|
+
} else if (clean._dimension) {
|
|
1059
|
+
clean.dimension = String(clean._dimension).split(':')[0]
|
|
1060
|
+
}
|
|
1043
1061
|
delete clean._dimension
|
|
1044
1062
|
return clean
|
|
1045
1063
|
})
|