@olegkoval/agent-skills 1.73.0 → 1.74.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapters/claude/olko-github-pr/skills/lekker-review/SKILL.md +3 -1
- package/adapters/claude/olko-github-pr/skills/lekker-review/evals/manifest.json +1 -1
- package/adapters/claude/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs +54 -16
- package/adapters/claude/olko-product/skills/frontend-direction/references/sources.md +7 -1
- package/adapters/cursor/olko-product/skills/frontend-direction/references/sources.md +7 -1
- package/adapters/grok/olko-product/skills/frontend-direction/references/sources.md +7 -1
- package/package.json +1 -1
- package/plugins/olko-apple-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-creative/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-garmin-kit/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-git-tools/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-github-pr/skills/lekker-review/SKILL.md +3 -1
- package/plugins/olko-github-pr/skills/lekker-review/evals/manifest.json +1 -1
- package/plugins/olko-github-pr/skills/lekker-review/scripts/eval-runner.mjs +54 -16
- package/plugins/olko-obsidian/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-product/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-product/skills/frontend-direction/references/sources.md +7 -1
- package/plugins/olko-reflection/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-release/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-skill-meta/.claude-plugin/plugin.json +1 -1
- package/plugins/olko-web-ops/.claude-plugin/plugin.json +1 -1
|
@@ -1430,7 +1430,9 @@ threads remain in the total but do not count as dismissed. Only comments ending
|
|
|
1430
1430
|
counted, so PRs posted before the marker existed score zero comments.
|
|
1431
1431
|
**Eval harness.** `evals/manifest.json` lists PRs with a known outcome (`expect:
|
|
1432
1432
|
"buggy"` with `knownBug: "file:line description"`, or `"clean"`). Run lekker on
|
|
1433
|
-
each, save the `findings` arrays
|
|
1433
|
+
each, save the `findings` arrays under each case's key (`id`, else
|
|
1434
|
+
`<repo>#<pr>@<sha>` when the entry has `sha`, else `<repo>#<pr>`), for example
|
|
1435
|
+
`{"<repo>#<pr>": [...]}`; a value may be an array of runs to get a hit rate, then
|
|
1434
1436
|
`node "${SKILL_ROOT}/scripts/eval-runner.mjs" --manifest "${SKILL_ROOT}/evals/manifest.json" --results <results.json>`
|
|
1435
1437
|
prints recall on buggy PRs and the false-positive count (critical or important
|
|
1436
1438
|
findings) on clean ones. The shipped manifest holds two placeholders; choosing
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"note": "PLACEHOLDERS ONLY. Replace with real PRs chosen by a human: known-buggy PRs with the bug located, and clean PRs that merged without follow-up fixes.",
|
|
2
|
+
"note": "PLACEHOLDERS ONLY. Replace with real PRs chosen by a human: known-buggy PRs with the bug located, and clean PRs that merged without follow-up fixes. Add id or sha to an entry to list the same PR at two commits (buggy head, merged head); a results value may be an array of runs.",
|
|
3
3
|
"prs": [
|
|
4
4
|
{
|
|
5
5
|
"repo": "example-org/example-repo",
|
|
@@ -4,15 +4,20 @@
|
|
|
4
4
|
// Usage:
|
|
5
5
|
// node eval-runner.mjs --manifest evals/manifest.json --results <results.json> [--json]
|
|
6
6
|
//
|
|
7
|
-
// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description" }] }
|
|
8
|
-
//
|
|
9
|
-
//
|
|
7
|
+
// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description", id?, sha? }] }
|
|
8
|
+
// The case key is `id` when present, else "<repo>#<pr>@<sha>" when `sha` is
|
|
9
|
+
// present, else "<repo>#<pr>". Use id or sha to list one PR at two commits.
|
|
10
|
+
// results.json: { "<case key>": [ <finding>, ... ] } the findings of one lekker run per case
|
|
11
|
+
// (the `findings` array the workflow returns), or an array of such arrays for
|
|
12
|
+
// repeated runs, which adds per-case hitRate and per-run counts to separate
|
|
13
|
+
// model noise from a real difference. A case missing from results is
|
|
10
14
|
// reported as unscored, never as zero findings.
|
|
11
15
|
//
|
|
12
16
|
// A buggy PR is HIT when some finding is on the knownBug file within LINE_WINDOW lines
|
|
13
17
|
// of the knownBug line. Known limitation: the match is by location only, so a
|
|
14
18
|
// same-file finding near the line counts even if it describes a different problem;
|
|
15
|
-
// judging "same bug" needs an LLM or a description match and is not attempted. Recall = hits /
|
|
19
|
+
// judging "same bug" needs an LLM or a description match and is not attempted. Recall = hits /
|
|
20
|
+
// scored buggy runs (one run per case unless a case has repeated runs). On a clean PR every
|
|
16
21
|
// verdict-affecting finding (critical or important) is a false positive.
|
|
17
22
|
|
|
18
23
|
import { readFileSync } from 'node:fs';
|
|
@@ -38,6 +43,15 @@ export function isHit(findings, knownBug) {
|
|
|
38
43
|
&& Math.abs(f.line - spot.line) <= LINE_WINDOW);
|
|
39
44
|
}
|
|
40
45
|
|
|
46
|
+
export function caseKey(e) {
|
|
47
|
+
if (typeof e.id === 'string' && e.id) return e.id;
|
|
48
|
+
return typeof e.sha === 'string' && e.sha ? `${e.repo}#${e.pr}@${e.sha}` : `${e.repo}#${e.pr}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const SHA = /^[0-9a-f]{7,40}$/i;
|
|
52
|
+
|
|
53
|
+
const isMultiRun = (v) => Array.isArray(v) && v.length > 0 && v.every(Array.isArray);
|
|
54
|
+
|
|
41
55
|
export function validateManifest(manifest) {
|
|
42
56
|
const prs = manifest && Array.isArray(manifest.prs) ? manifest.prs : null;
|
|
43
57
|
if (!prs) throw new Error('manifest needs a "prs" array');
|
|
@@ -45,7 +59,9 @@ export function validateManifest(manifest) {
|
|
|
45
59
|
prs.forEach((e, i) => {
|
|
46
60
|
if (!e.repo || !Number.isInteger(e.pr)) throw new Error(`prs[${i}]: repo and integer pr are required`);
|
|
47
61
|
if (e.expect !== 'buggy' && e.expect !== 'clean') throw new Error(`prs[${i}]: expect must be "buggy" or "clean"`);
|
|
48
|
-
|
|
62
|
+
if ('id' in e && (typeof e.id !== 'string' || !e.id)) throw new Error(`prs[${i}]: id must be a non-empty string`);
|
|
63
|
+
if ('sha' in e && (typeof e.sha !== 'string' || !SHA.test(e.sha))) throw new Error(`prs[${i}]: sha must be 7 to 40 hex characters`);
|
|
64
|
+
const key = caseKey(e);
|
|
49
65
|
if (seen.has(key)) throw new Error(`prs[${i}]: duplicate entry ${key}`);
|
|
50
66
|
seen.add(key);
|
|
51
67
|
if (e.expect === 'buggy' && !parseKnownBug(e.knownBug)) {
|
|
@@ -57,25 +73,42 @@ export function validateManifest(manifest) {
|
|
|
57
73
|
|
|
58
74
|
export function score(manifest, results) {
|
|
59
75
|
const rows = validateManifest(manifest).map((e) => {
|
|
60
|
-
const key =
|
|
61
|
-
const
|
|
62
|
-
if (!Array.isArray(
|
|
63
|
-
|
|
76
|
+
const key = caseKey(e);
|
|
77
|
+
const value = results[key];
|
|
78
|
+
if (!Array.isArray(value)) return { key, expect: e.expect, scored: false };
|
|
79
|
+
const runs = isMultiRun(value) ? value : null;
|
|
80
|
+
if (e.expect === 'buggy') {
|
|
81
|
+
if (!runs) return { key, expect: 'buggy', scored: true, hit: isHit(value, e.knownBug) };
|
|
82
|
+
const hits = runs.filter((r) => isHit(r, e.knownBug)).length;
|
|
83
|
+
return { key, expect: 'buggy', scored: true, hit: hits > 0, runs: runs.length, hits, hitRate: hits / runs.length };
|
|
84
|
+
}
|
|
85
|
+
const fp = (r) => r.filter((f) => BLOCKING.has(f.severity)).length;
|
|
86
|
+
if (!runs) return { key, expect: 'clean', scored: true, falsePositives: fp(value) };
|
|
87
|
+
const perRun = runs.map(fp);
|
|
64
88
|
return {
|
|
65
|
-
key, expect: 'clean', scored: true,
|
|
66
|
-
|
|
89
|
+
key, expect: 'clean', scored: true, falsePositives: perRun.reduce((n, x) => n + x, 0),
|
|
90
|
+
runs: runs.length, falsePositivesPerRun: perRun,
|
|
67
91
|
};
|
|
68
92
|
});
|
|
69
93
|
const buggy = rows.filter((r) => r.expect === 'buggy' && r.scored);
|
|
70
94
|
const clean = rows.filter((r) => r.expect === 'clean' && r.scored);
|
|
71
|
-
|
|
72
|
-
|
|
95
|
+
const buggyRuns = buggy.reduce((n, r) => n + (r.runs || 1), 0);
|
|
96
|
+
const cleanRuns = clean.reduce((n, r) => n + (r.runs || 1), 0);
|
|
97
|
+
const falsePositives = clean.reduce((n, r) => n + r.falsePositives, 0);
|
|
98
|
+
const out = {
|
|
99
|
+
recall: buggyRuns === 0 ? null : buggy.reduce((n, r) => n + (r.runs ? r.hits : Number(r.hit)), 0) / buggyRuns,
|
|
73
100
|
buggyScored: buggy.length,
|
|
74
|
-
falsePositives
|
|
101
|
+
falsePositives,
|
|
75
102
|
cleanScored: clean.length,
|
|
76
103
|
unscored: rows.filter((r) => !r.scored).map((r) => r.key),
|
|
77
104
|
rows,
|
|
78
105
|
};
|
|
106
|
+
if (rows.some((r) => r.runs)) {
|
|
107
|
+
Object.assign(out, {
|
|
108
|
+
buggyRuns, cleanRuns, falsePositivesPerCleanRun: cleanRuns === 0 ? null : falsePositives / cleanRuns,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
return out;
|
|
79
112
|
}
|
|
80
113
|
|
|
81
114
|
function main(argv) {
|
|
@@ -86,8 +119,13 @@ function main(argv) {
|
|
|
86
119
|
}
|
|
87
120
|
const out = score(JSON.parse(readFileSync(flag('manifest'), 'utf8')), JSON.parse(readFileSync(flag('results'), 'utf8')));
|
|
88
121
|
if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
|
|
89
|
-
const pct =
|
|
90
|
-
|
|
122
|
+
const pct = (x) => (x === null ? 'n/a' : `${Math.round(x * 100)}%`);
|
|
123
|
+
if (out.buggyRuns === undefined) {
|
|
124
|
+
console.log(`recall ${pct(out.recall)} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
const perRun = out.falsePositivesPerCleanRun === null ? 'n/a' : out.falsePositivesPerCleanRun.toFixed(2);
|
|
128
|
+
console.log(`recall ${pct(out.recall)} per run (${out.buggyRuns} runs over ${out.buggyScored} buggy cases), false positives ${out.falsePositives} (${out.cleanRuns} runs over ${out.cleanScored} clean cases, ${perRun} per run), unscored ${out.unscored.length}`);
|
|
91
129
|
}
|
|
92
130
|
|
|
93
131
|
if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
|
|
@@ -33,4 +33,10 @@ Seed: [Aman Singh's resource post](https://x.com/amananimates/status/21118486758
|
|
|
33
33
|
|
|
34
34
|
For each selected candidate, inspect its exact documentation and source, not only its homepage. Verify framework/version requirements, dependencies, responsive states, keyboard behavior, reduced-motion handling where relevant, and reuse terms. Paid sections and free sections may have different conditions.
|
|
35
35
|
|
|
36
|
-
Use the source that answers the task. A native HTML app can implement an original CSS interaction; a chart library is unnecessary when a comparison table is clearer. If no candidate fits, retain local components and derive distinctiveness from composition, content and typography.
|
|
36
|
+
Use the source that answers the task. A native HTML app can implement an original CSS interaction; a chart library is unnecessary when a comparison table is clearer. If no candidate fits, retain local components and derive distinctiveness from composition, content and typography.
|
|
37
|
+
|
|
38
|
+
Seed: [Benji Taylor's Stageshow post](https://x.com/benjitaylor/status/2116593247854453244), retrieved on 2026-11-05 through [FxTwitter](https://api.fxtwitter.com/status/2116593247854453244) after X returned 403. This entry is for marketing illustration, not for building a product screen. Its mock content is invented by design, so never use it in place of real product content. No license was detected on the repository; check the `stageshow/` folder's terms before installing or copying anything.
|
|
39
|
+
|
|
40
|
+
| Source | Start here when exploring |
|
|
41
|
+
| --- | --- |
|
|
42
|
+
| [Stageshow](https://github.com/benjitaylor/skills/tree/main/stageshow) | A separate skill for faux product UI stills and animated cursor demos on landing-page heroes and feature cards |
|
|
@@ -33,4 +33,10 @@ Seed: [Aman Singh's resource post](https://x.com/amananimates/status/21118486758
|
|
|
33
33
|
|
|
34
34
|
For each selected candidate, inspect its exact documentation and source, not only its homepage. Verify framework/version requirements, dependencies, responsive states, keyboard behavior, reduced-motion handling where relevant, and reuse terms. Paid sections and free sections may have different conditions.
|
|
35
35
|
|
|
36
|
-
Use the source that answers the task. A native HTML app can implement an original CSS interaction; a chart library is unnecessary when a comparison table is clearer. If no candidate fits, retain local components and derive distinctiveness from composition, content and typography.
|
|
36
|
+
Use the source that answers the task. A native HTML app can implement an original CSS interaction; a chart library is unnecessary when a comparison table is clearer. If no candidate fits, retain local components and derive distinctiveness from composition, content and typography.
|
|
37
|
+
|
|
38
|
+
Seed: [Benji Taylor's Stageshow post](https://x.com/benjitaylor/status/2116593247854453244), retrieved on 2026-11-05 through [FxTwitter](https://api.fxtwitter.com/status/2116593247854453244) after X returned 403. This entry is for marketing illustration, not for building a product screen. Its mock content is invented by design, so never use it in place of real product content. No license was detected on the repository; check the `stageshow/` folder's terms before installing or copying anything.
|
|
39
|
+
|
|
40
|
+
| Source | Start here when exploring |
|
|
41
|
+
| --- | --- |
|
|
42
|
+
| [Stageshow](https://github.com/benjitaylor/skills/tree/main/stageshow) | A separate skill for faux product UI stills and animated cursor demos on landing-page heroes and feature cards |
|
|
@@ -33,4 +33,10 @@ Seed: [Aman Singh's resource post](https://x.com/amananimates/status/21118486758
|
|
|
33
33
|
|
|
34
34
|
For each selected candidate, inspect its exact documentation and source, not only its homepage. Verify framework/version requirements, dependencies, responsive states, keyboard behavior, reduced-motion handling where relevant, and reuse terms. Paid sections and free sections may have different conditions.
|
|
35
35
|
|
|
36
|
-
Use the source that answers the task. A native HTML app can implement an original CSS interaction; a chart library is unnecessary when a comparison table is clearer. If no candidate fits, retain local components and derive distinctiveness from composition, content and typography.
|
|
36
|
+
Use the source that answers the task. A native HTML app can implement an original CSS interaction; a chart library is unnecessary when a comparison table is clearer. If no candidate fits, retain local components and derive distinctiveness from composition, content and typography.
|
|
37
|
+
|
|
38
|
+
Seed: [Benji Taylor's Stageshow post](https://x.com/benjitaylor/status/2116593247854453244), retrieved on 2026-11-05 through [FxTwitter](https://api.fxtwitter.com/status/2116593247854453244) after X returned 403. This entry is for marketing illustration, not for building a product screen. Its mock content is invented by design, so never use it in place of real product content. No license was detected on the repository; check the `stageshow/` folder's terms before installing or copying anything.
|
|
39
|
+
|
|
40
|
+
| Source | Start here when exploring |
|
|
41
|
+
| --- | --- |
|
|
42
|
+
| [Stageshow](https://github.com/benjitaylor/skills/tree/main/stageshow) | A separate skill for faux product UI stills and animated cursor demos on landing-page heroes and feature cards |
|
package/package.json
CHANGED
|
@@ -1429,7 +1429,9 @@ threads remain in the total but do not count as dismissed. Only comments ending
|
|
|
1429
1429
|
counted, so PRs posted before the marker existed score zero comments.
|
|
1430
1430
|
**Eval harness.** `evals/manifest.json` lists PRs with a known outcome (`expect:
|
|
1431
1431
|
"buggy"` with `knownBug: "file:line description"`, or `"clean"`). Run lekker on
|
|
1432
|
-
each, save the `findings` arrays
|
|
1432
|
+
each, save the `findings` arrays under each case's key (`id`, else
|
|
1433
|
+
`<repo>#<pr>@<sha>` when the entry has `sha`, else `<repo>#<pr>`), for example
|
|
1434
|
+
`{"<repo>#<pr>": [...]}`; a value may be an array of runs to get a hit rate, then
|
|
1433
1435
|
`node "${SKILL_ROOT}/scripts/eval-runner.mjs" --manifest "${SKILL_ROOT}/evals/manifest.json" --results <results.json>`
|
|
1434
1436
|
prints recall on buggy PRs and the false-positive count (critical or important
|
|
1435
1437
|
findings) on clean ones. The shipped manifest holds two placeholders; choosing
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"note": "PLACEHOLDERS ONLY. Replace with real PRs chosen by a human: known-buggy PRs with the bug located, and clean PRs that merged without follow-up fixes.",
|
|
2
|
+
"note": "PLACEHOLDERS ONLY. Replace with real PRs chosen by a human: known-buggy PRs with the bug located, and clean PRs that merged without follow-up fixes. Add id or sha to an entry to list the same PR at two commits (buggy head, merged head); a results value may be an array of runs.",
|
|
3
3
|
"prs": [
|
|
4
4
|
{
|
|
5
5
|
"repo": "example-org/example-repo",
|
|
@@ -4,15 +4,20 @@
|
|
|
4
4
|
// Usage:
|
|
5
5
|
// node eval-runner.mjs --manifest evals/manifest.json --results <results.json> [--json]
|
|
6
6
|
//
|
|
7
|
-
// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description" }] }
|
|
8
|
-
//
|
|
9
|
-
//
|
|
7
|
+
// manifest.json: { prs: [{ repo, pr, expect: "buggy"|"clean", knownBug?: "file:line + description", id?, sha? }] }
|
|
8
|
+
// The case key is `id` when present, else "<repo>#<pr>@<sha>" when `sha` is
|
|
9
|
+
// present, else "<repo>#<pr>". Use id or sha to list one PR at two commits.
|
|
10
|
+
// results.json: { "<case key>": [ <finding>, ... ] } the findings of one lekker run per case
|
|
11
|
+
// (the `findings` array the workflow returns), or an array of such arrays for
|
|
12
|
+
// repeated runs, which adds per-case hitRate and per-run counts to separate
|
|
13
|
+
// model noise from a real difference. A case missing from results is
|
|
10
14
|
// reported as unscored, never as zero findings.
|
|
11
15
|
//
|
|
12
16
|
// A buggy PR is HIT when some finding is on the knownBug file within LINE_WINDOW lines
|
|
13
17
|
// of the knownBug line. Known limitation: the match is by location only, so a
|
|
14
18
|
// same-file finding near the line counts even if it describes a different problem;
|
|
15
|
-
// judging "same bug" needs an LLM or a description match and is not attempted. Recall = hits /
|
|
19
|
+
// judging "same bug" needs an LLM or a description match and is not attempted. Recall = hits /
|
|
20
|
+
// scored buggy runs (one run per case unless a case has repeated runs). On a clean PR every
|
|
16
21
|
// verdict-affecting finding (critical or important) is a false positive.
|
|
17
22
|
|
|
18
23
|
import { readFileSync } from 'node:fs';
|
|
@@ -38,6 +43,15 @@ export function isHit(findings, knownBug) {
|
|
|
38
43
|
&& Math.abs(f.line - spot.line) <= LINE_WINDOW);
|
|
39
44
|
}
|
|
40
45
|
|
|
46
|
+
export function caseKey(e) {
|
|
47
|
+
if (typeof e.id === 'string' && e.id) return e.id;
|
|
48
|
+
return typeof e.sha === 'string' && e.sha ? `${e.repo}#${e.pr}@${e.sha}` : `${e.repo}#${e.pr}`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const SHA = /^[0-9a-f]{7,40}$/i;
|
|
52
|
+
|
|
53
|
+
const isMultiRun = (v) => Array.isArray(v) && v.length > 0 && v.every(Array.isArray);
|
|
54
|
+
|
|
41
55
|
export function validateManifest(manifest) {
|
|
42
56
|
const prs = manifest && Array.isArray(manifest.prs) ? manifest.prs : null;
|
|
43
57
|
if (!prs) throw new Error('manifest needs a "prs" array');
|
|
@@ -45,7 +59,9 @@ export function validateManifest(manifest) {
|
|
|
45
59
|
prs.forEach((e, i) => {
|
|
46
60
|
if (!e.repo || !Number.isInteger(e.pr)) throw new Error(`prs[${i}]: repo and integer pr are required`);
|
|
47
61
|
if (e.expect !== 'buggy' && e.expect !== 'clean') throw new Error(`prs[${i}]: expect must be "buggy" or "clean"`);
|
|
48
|
-
|
|
62
|
+
if ('id' in e && (typeof e.id !== 'string' || !e.id)) throw new Error(`prs[${i}]: id must be a non-empty string`);
|
|
63
|
+
if ('sha' in e && (typeof e.sha !== 'string' || !SHA.test(e.sha))) throw new Error(`prs[${i}]: sha must be 7 to 40 hex characters`);
|
|
64
|
+
const key = caseKey(e);
|
|
49
65
|
if (seen.has(key)) throw new Error(`prs[${i}]: duplicate entry ${key}`);
|
|
50
66
|
seen.add(key);
|
|
51
67
|
if (e.expect === 'buggy' && !parseKnownBug(e.knownBug)) {
|
|
@@ -57,25 +73,42 @@ export function validateManifest(manifest) {
|
|
|
57
73
|
|
|
58
74
|
export function score(manifest, results) {
|
|
59
75
|
const rows = validateManifest(manifest).map((e) => {
|
|
60
|
-
const key =
|
|
61
|
-
const
|
|
62
|
-
if (!Array.isArray(
|
|
63
|
-
|
|
76
|
+
const key = caseKey(e);
|
|
77
|
+
const value = results[key];
|
|
78
|
+
if (!Array.isArray(value)) return { key, expect: e.expect, scored: false };
|
|
79
|
+
const runs = isMultiRun(value) ? value : null;
|
|
80
|
+
if (e.expect === 'buggy') {
|
|
81
|
+
if (!runs) return { key, expect: 'buggy', scored: true, hit: isHit(value, e.knownBug) };
|
|
82
|
+
const hits = runs.filter((r) => isHit(r, e.knownBug)).length;
|
|
83
|
+
return { key, expect: 'buggy', scored: true, hit: hits > 0, runs: runs.length, hits, hitRate: hits / runs.length };
|
|
84
|
+
}
|
|
85
|
+
const fp = (r) => r.filter((f) => BLOCKING.has(f.severity)).length;
|
|
86
|
+
if (!runs) return { key, expect: 'clean', scored: true, falsePositives: fp(value) };
|
|
87
|
+
const perRun = runs.map(fp);
|
|
64
88
|
return {
|
|
65
|
-
key, expect: 'clean', scored: true,
|
|
66
|
-
|
|
89
|
+
key, expect: 'clean', scored: true, falsePositives: perRun.reduce((n, x) => n + x, 0),
|
|
90
|
+
runs: runs.length, falsePositivesPerRun: perRun,
|
|
67
91
|
};
|
|
68
92
|
});
|
|
69
93
|
const buggy = rows.filter((r) => r.expect === 'buggy' && r.scored);
|
|
70
94
|
const clean = rows.filter((r) => r.expect === 'clean' && r.scored);
|
|
71
|
-
|
|
72
|
-
|
|
95
|
+
const buggyRuns = buggy.reduce((n, r) => n + (r.runs || 1), 0);
|
|
96
|
+
const cleanRuns = clean.reduce((n, r) => n + (r.runs || 1), 0);
|
|
97
|
+
const falsePositives = clean.reduce((n, r) => n + r.falsePositives, 0);
|
|
98
|
+
const out = {
|
|
99
|
+
recall: buggyRuns === 0 ? null : buggy.reduce((n, r) => n + (r.runs ? r.hits : Number(r.hit)), 0) / buggyRuns,
|
|
73
100
|
buggyScored: buggy.length,
|
|
74
|
-
falsePositives
|
|
101
|
+
falsePositives,
|
|
75
102
|
cleanScored: clean.length,
|
|
76
103
|
unscored: rows.filter((r) => !r.scored).map((r) => r.key),
|
|
77
104
|
rows,
|
|
78
105
|
};
|
|
106
|
+
if (rows.some((r) => r.runs)) {
|
|
107
|
+
Object.assign(out, {
|
|
108
|
+
buggyRuns, cleanRuns, falsePositivesPerCleanRun: cleanRuns === 0 ? null : falsePositives / cleanRuns,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
return out;
|
|
79
112
|
}
|
|
80
113
|
|
|
81
114
|
function main(argv) {
|
|
@@ -86,8 +119,13 @@ function main(argv) {
|
|
|
86
119
|
}
|
|
87
120
|
const out = score(JSON.parse(readFileSync(flag('manifest'), 'utf8')), JSON.parse(readFileSync(flag('results'), 'utf8')));
|
|
88
121
|
if (argv.includes('--json')) { console.log(JSON.stringify(out)); return; }
|
|
89
|
-
const pct =
|
|
90
|
-
|
|
122
|
+
const pct = (x) => (x === null ? 'n/a' : `${Math.round(x * 100)}%`);
|
|
123
|
+
if (out.buggyRuns === undefined) {
|
|
124
|
+
console.log(`recall ${pct(out.recall)} (${out.buggyScored} buggy scored), false positives ${out.falsePositives} (${out.cleanScored} clean scored), unscored ${out.unscored.length}`);
|
|
125
|
+
return;
|
|
126
|
+
}
|
|
127
|
+
const perRun = out.falsePositivesPerCleanRun === null ? 'n/a' : out.falsePositivesPerCleanRun.toFixed(2);
|
|
128
|
+
console.log(`recall ${pct(out.recall)} per run (${out.buggyRuns} runs over ${out.buggyScored} buggy cases), false positives ${out.falsePositives} (${out.cleanRuns} runs over ${out.cleanScored} clean cases, ${perRun} per run), unscored ${out.unscored.length}`);
|
|
91
129
|
}
|
|
92
130
|
|
|
93
131
|
if (process.argv[1] === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
|
|
@@ -33,4 +33,10 @@ Seed: [Aman Singh's resource post](https://x.com/amananimates/status/21118486758
|
|
|
33
33
|
|
|
34
34
|
For each selected candidate, inspect its exact documentation and source, not only its homepage. Verify framework/version requirements, dependencies, responsive states, keyboard behavior, reduced-motion handling where relevant, and reuse terms. Paid sections and free sections may have different conditions.
|
|
35
35
|
|
|
36
|
-
Use the source that answers the task. A native HTML app can implement an original CSS interaction; a chart library is unnecessary when a comparison table is clearer. If no candidate fits, retain local components and derive distinctiveness from composition, content and typography.
|
|
36
|
+
Use the source that answers the task. A native HTML app can implement an original CSS interaction; a chart library is unnecessary when a comparison table is clearer. If no candidate fits, retain local components and derive distinctiveness from composition, content and typography.
|
|
37
|
+
|
|
38
|
+
Seed: [Benji Taylor's Stageshow post](https://x.com/benjitaylor/status/2116593247854453244), retrieved on 2026-11-05 through [FxTwitter](https://api.fxtwitter.com/status/2116593247854453244) after X returned 403. This entry is for marketing illustration, not for building a product screen. Its mock content is invented by design, so never use it in place of real product content. No license was detected on the repository; check the `stageshow/` folder's terms before installing or copying anything.
|
|
39
|
+
|
|
40
|
+
| Source | Start here when exploring |
|
|
41
|
+
| --- | --- |
|
|
42
|
+
| [Stageshow](https://github.com/benjitaylor/skills/tree/main/stageshow) | A separate skill for faux product UI stills and animated cursor demos on landing-page heroes and feature cards |
|