@giamme/forge 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/SKILL.md +4 -3
- package/VERSION +1 -1
- package/docs/artifacts.md +11 -0
- package/docs/commands.md +8 -3
- package/docs/dataset.md +13 -0
- package/docs/decomposed-runs.md +13 -3
- package/docs/forge-ui.md +8 -3
- package/docs/memory.md +4 -2
- package/docs/reviews.md +73 -2
- package/forge +16 -1
- package/package.json +1 -1
- package/references/dataset.md +86 -4
- package/references/decompose.md +24 -6
- package/references/harnesses.md +43 -1
- package/references/ui.md +4 -1
- package/scripts/forge-config.py +5 -2
- package/scripts/forge-control.py +4 -0
- package/scripts/forge-dispatch.sh +235 -121
- package/scripts/forge-issues.py +344 -16
- package/scripts/forge-lib-plans.sh +6 -2
- package/scripts/forge-lib-review.sh +158 -3
- package/scripts/forge-lib-task.sh +132 -14
- package/scripts/forge-parallel.sh +96 -11
- package/scripts/forge-prompt.py +61 -4
- package/scripts/forge-route.py +72 -12
- package/scripts/forge-runtime.py +47 -1
- package/scripts/forge_dataset/__init__.py +7 -2
- package/scripts/forge_dataset/cli.py +44 -1
- package/scripts/forge_dataset/export.py +36 -3
- package/scripts/forge_dataset/forecast.py +355 -0
- package/scripts/forge_dataset/harvest.py +23 -2
- package/scripts/forge_dataset/importer.py +20 -8
- package/scripts/forge_dataset/reviewers.py +145 -0
- package/scripts/forge_dataset/routing.py +1 -0
- package/scripts/forge_jev/__init__.py +7 -3
- package/scripts/forge_registry.py +15 -10
- package/ui/README.md +4 -3
- package/ui/docs/api.md +75 -15
- package/ui/forge_ui/control.py +12 -0
- package/ui/forge_ui/reader.py +184 -11
- package/ui/forge_ui/static/app.js +24 -10
- package/ui/forge_ui/static/cloud3d.js +4 -1
- package/ui/forge_ui/static/cloudmodel.js +1 -1
- package/ui/forge_ui/static/cloudui.js +13 -4
- package/ui/forge_ui/static/findingsview.js +29 -3
- package/ui/forge_ui/static/graph.js +6 -5
- package/ui/forge_ui/static/leaderboard.js +18 -0
- package/ui/forge_ui/static/modelstats.js +19 -1
- package/ui/forge_ui/static/panel.js +52 -10
- package/ui/forge_ui/static/runmodel.js +128 -13
- package/ui/forge_ui/static/scene3d.js +32 -15
- package/ui/forge_ui/static/style.css +7 -3
|
@@ -19,7 +19,7 @@ window.ForgeApp = (() => {
|
|
|
19
19
|
replay: null, playing: null, speed: store.get('speed', 60), stage: null, stageFailed: null, stageLoading: false, timer: null,
|
|
20
20
|
lastStatuses: new Map(), announced: 0, rendered: null, cloud: null, cloudUI: null, cloudPrefs: store.get('cloud', {}),
|
|
21
21
|
settings: false, lastKey: null, settingsRun: null, settingsPickedAt: null, settingsView: null,
|
|
22
|
-
models: false, modelsData: null, modelsText: null, modelsError: null, modelsPrefs: store.get('models', {}),
|
|
22
|
+
models: false, modelsData: null, modelsText: null, modelsError: null, reviewerOutcomes: null, modelsPrefs: store.get('models', {}),
|
|
23
23
|
files: new Map(), findingsCount: new Map(), filesPrefs: store.get('files', {}), runSort: store.get('run-sort', 'updated'), tablePrefs: store.get('tables', {}),
|
|
24
24
|
findings: new Map() };
|
|
25
25
|
state.stageMode = state.mode === 'exec' ? 'exec' : 'plan'; // the stage's own mode, kept while Files is shown
|
|
@@ -106,9 +106,9 @@ window.ForgeApp = (() => {
|
|
|
106
106
|
function link(text, key, sel) { const a = el('a', text); a.href = hashFor(key, sel); return a; }
|
|
107
107
|
|
|
108
108
|
const statuses = new Set(['merged', 'pass', 'running', 'pending', 'fail', 'error', 'infra',
|
|
109
|
-
'timeout', 'unknown', 'invalidated', 'blocked', 'interrupted', 'nochanges', 'conflict']);
|
|
110
|
-
function chip(value) {
|
|
111
|
-
const status = String(value || 'unknown').toLowerCase();
|
|
109
|
+
'timeout', 'unknown', 'invalidated', 'blocked', 'interrupted', 'nochanges', 'conflict', 'accepted', 'outdated']);
|
|
110
|
+
function chip(value, cls) {
|
|
111
|
+
const status = String(cls || value || 'unknown').toLowerCase();
|
|
112
112
|
return el('span', value || '—', 'chip st-' + (statuses.has(status) ? status : 'unknown'));
|
|
113
113
|
}
|
|
114
114
|
function time(value) {
|
|
@@ -251,7 +251,8 @@ window.ForgeApp = (() => {
|
|
|
251
251
|
if (!solo) {
|
|
252
252
|
const verification = el('span', undefined, 'verification');
|
|
253
253
|
verification.append(document.createTextNode('Verification '), chip(run.verification_status));
|
|
254
|
-
|
|
254
|
+
const reviewStatus = RM.runReviewStatus(run);
|
|
255
|
+
if (reviewStatus) verification.append(document.createTextNode(' Review '), chip(reviewStatus));
|
|
255
256
|
item.append(verification);
|
|
256
257
|
}
|
|
257
258
|
$('run-list').append(item);
|
|
@@ -338,7 +339,7 @@ window.ForgeApp = (() => {
|
|
|
338
339
|
add('Kind', sel('kind', [['', 'any'], ['plan', 'plan'], ['solo', 'solo']]));
|
|
339
340
|
add('Model', sel('model', [['', 'any'], ...opts.models.map(m => [m, m])]));
|
|
340
341
|
add('Age', sel('ageDays', [[0, 'any'], [1, 'last day'], [7, 'last week'], [30, 'last month']]));
|
|
341
|
-
add('Review', sel('verdict', [['', 'any'], ['PASS', 'passed'], ['FAIL', 'failed'], ['none', 'no verdict']]));
|
|
342
|
+
add('Review', sel('verdict', [['', 'any'], ['PASS', 'passed'], ['FAIL', 'failed'], ['OUTDATED', 'outdated'], ['none', 'no verdict']]));
|
|
342
343
|
const all = el('input'); all.type = 'checkbox'; all.checked = state.showAll; all.dataset.f = 'showAll';
|
|
343
344
|
const allL = el('label', undefined, 'filter check'); allL.append(all, document.createTextNode(' older runs too')); bar.append(allL);
|
|
344
345
|
const reset = el('button', 'Reset'); reset.type = 'button'; reset.addEventListener('click', () => { Object.assign(state.mapFilter, { text: '', state: '', repo: '', kind: '', model: '', ageDays: 0, verdict: '' }); state.showAll = false; filterChanged(true); });
|
|
@@ -368,7 +369,12 @@ window.ForgeApp = (() => {
|
|
|
368
369
|
card.className = `map-card st-${({ live: 'running', failed: 'fail', done: 'merged', pending: 'pending' })[m.state]}${m.match ? '' : ' dim'}`;
|
|
369
370
|
card.append(el('strong', String(r.goal || 'Untitled run').split('\n')[0]), el('span', `${r.run_id || ''}${r.kind === 'solo' ? ' · solo' : ''}`, 'mono muted'));
|
|
370
371
|
const leds = el('span', undefined, 'leds');
|
|
371
|
-
for (const t of m.tasks.slice(0, 60)) {
|
|
372
|
+
for (const t of m.tasks.slice(0, 60)) {
|
|
373
|
+
const status = RM.taskStatus(t);
|
|
374
|
+
const l = el('i', undefined, 'led st-' + String(status || 'pending').toLowerCase());
|
|
375
|
+
l.title = `${t.id}: ${status}`;
|
|
376
|
+
leds.append(l);
|
|
377
|
+
}
|
|
372
378
|
card.append(leds, el('span', relative(r.updated), 'muted'));
|
|
373
379
|
grid.append(card);
|
|
374
380
|
}
|
|
@@ -474,15 +480,23 @@ window.ForgeApp = (() => {
|
|
|
474
480
|
runs: state.runs, prefs: state.modelsPrefs, now: () => Date.now() / 1000,
|
|
475
481
|
savePrefs: p => { state.modelsPrefs = p; store.set('models', p); },
|
|
476
482
|
tablePrefs: id => state.tablePrefs[id], saveTablePrefs: (id, p) => { state.tablePrefs = { ...state.tablePrefs, [id]: p }; store.set('tables', state.tablePrefs); },
|
|
477
|
-
modelHref: alias => '#/cloud/models/' + encodeURIComponent(alias),
|
|
483
|
+
modelHref: alias => '#/cloud/models/' + encodeURIComponent(alias), reviewerOutcomes: state.reviewerOutcomes,
|
|
478
484
|
});
|
|
479
485
|
}
|
|
486
|
+
// How each reviewer's PASSed work held up (forge dataset reviewers). Advisory and optional: read once per
|
|
487
|
+
// change of the models data, and a failure (an export, no dataset) only leaves the column empty.
|
|
488
|
+
function loadReviewerOutcomes() {
|
|
489
|
+
if (SNAP) return;
|
|
490
|
+
fetchJSON('/api/models/reviewers').then(result => { state.reviewerOutcomes = result.data; },
|
|
491
|
+
() => { state.reviewerOutcomes = null; }).then(() => { if (state.models) renderModels(); });
|
|
492
|
+
}
|
|
480
493
|
function loadModels() {
|
|
481
494
|
return request('/api/models', result => {
|
|
482
495
|
state.modelsError = null;
|
|
483
496
|
if (result.text === state.modelsText) return;
|
|
484
497
|
state.modelsText = result.text; state.modelsData = result.data;
|
|
485
498
|
if (state.models) renderModels();
|
|
499
|
+
loadReviewerOutcomes();
|
|
486
500
|
}, 'Could not refresh the model results', error => { state.modelsError = error.message; if (state.models) renderModels(); });
|
|
487
501
|
}
|
|
488
502
|
|
|
@@ -741,11 +755,11 @@ window.ForgeApp = (() => {
|
|
|
741
755
|
{ id: 'title', label: 'Title', value: t => t.title },
|
|
742
756
|
{ id: 'difficulty', label: 'Difficulty', value: t => DIFF_RANK[t.difficulty] ?? null, cell: t => display(t.difficulty) },
|
|
743
757
|
{ id: 'wave', label: 'Wave', value: t => Number.isFinite(Number(t.wave)) && t.wave !== null && t.wave !== '' ? Number(t.wave) : null, cell: t => display(t.wave) },
|
|
744
|
-
{ id: 'status', label: 'Status', value: t => t
|
|
758
|
+
{ id: 'status', label: 'Status', value: t => RM.taskStatus(t), cell: t => chip(RM.taskStatus(t)) },
|
|
745
759
|
{ id: 'dwarf', label: 'Dwarf', value: t => t.dwarf, cell: t => window.ForgeIcons.modelPlate(t.dwarf) },
|
|
746
760
|
{ id: 'qa', label: 'QA', value: t => t.qa, cell: t => window.ForgeIcons.modelPlate(t.qa) },
|
|
747
761
|
{ id: 'attempt', label: 'Attempt', value: t => Number(t.attempt) || null, cell: t => display(t.attempt) },
|
|
748
|
-
{ id: 'verdict', label: 'Verdict', value: t => t.
|
|
762
|
+
{ id: 'verdict', label: 'Verdict', value: t => RM.taskVerdict(t).text, cell: t => { const v = RM.taskVerdict(t); return chip(v.text, v.status); } },
|
|
749
763
|
{ id: 'findings', label: 'Findings', title: 'The worst confirmed severity and how many; ~n plausible only',
|
|
750
764
|
value: t => t.findings && t.findings.worst ? SEV_RANK[t.findings.worst] * 1000 - (t.findings[t.findings.worst] || 0) : t.findings && t.findings.plausible ? 4000 : null,
|
|
751
765
|
cell: t => { const f = t.findings, v = sev(f); if (!f || !f.worst) return v; const n = el('span', v, 'sev-chip sev-' + f.worst); return n; } },
|
|
@@ -427,9 +427,12 @@ export function createCloud(E, host, cb = {}) {
|
|
|
427
427
|
function info(el) {
|
|
428
428
|
const f = el.f, r = f.run, counts = Object.entries(r.counts || {}).map(([s, n]) => `${s.toLowerCase()} ${n}`).join(' · ');
|
|
429
429
|
const proj = st.L && st.L.projects.find(p => p.id === f.project);
|
|
430
|
+
const verdict = r.final_review && typeof r.final_review === 'object' ? r.final_review.verdict : r.final_review;
|
|
431
|
+
const isOutdated = verdict && RM.runReviewStatus(r) === 'OUTDATED';
|
|
432
|
+
const reviewText = verdict ? (isOutdated ? `${verdict} (outdated)` : verdict) : null;
|
|
430
433
|
return [['meta', `${proj ? proj.name : RM.basename(r.repo)} · ${r.run_id || ''} · ${r.kind}`], ['title', String(r.goal || 'Untitled run').split('\n')[0]],
|
|
431
434
|
['body', `${f.n} task${f.n > 1 ? 's' : ''} · ${counts}`], ['body', `${[...f.dwarf].join(', ') || '—'} → ${[...f.qa].join(', ') || '—'}`],
|
|
432
|
-
['foot', `started ${new Date(f.t * 1000).toLocaleString()}${
|
|
435
|
+
['foot', `started ${new Date(f.t * 1000).toLocaleString()}${reviewText ? ' · review ' + reviewText : ''}${r.verification_status ? ' · verify ' + r.verification_status : ''}`],
|
|
433
436
|
['foot', (store.layouts[st.layout] || {})[el.key] ? 'moved by you · drag to move, Shift = depth, Alt = height' : 'drag to move · double-click to open']];
|
|
434
437
|
}
|
|
435
438
|
function showTip(el, cx, cy) {
|
|
@@ -63,7 +63,7 @@
|
|
|
63
63
|
const aliases = specs => new Set(list(specs).map(RM.alias).filter(Boolean));
|
|
64
64
|
return { key: run.key, run, project: hashId(run.repo || '?'), n, bucket: bucket(n), diff, dominant, state, t: startOf(run),
|
|
65
65
|
dwarf: aliases(run.dwarf_models || run.models), qa: aliases(run.qa_models || []), all: aliases(run.models),
|
|
66
|
-
review: run
|
|
66
|
+
review: RM.runReviewStatus(run) || null, verification: run.verification_status || null, decompose: run.decompose_level || null, kind: run.kind };
|
|
67
67
|
}
|
|
68
68
|
|
|
69
69
|
/* ---------- filters: the cloud shows exactly the runs that match ---------- */
|
|
@@ -254,7 +254,7 @@ window.ForgeCloudUI = (() => {
|
|
|
254
254
|
}
|
|
255
255
|
const more = el('fieldset', undefined, 'cloud-field cloud-more'); more.append(el('legend', 'More'));
|
|
256
256
|
more.append(select('Active', 'active', [[0, 'any time'], [1, 'in the last day'], [7, 'in the last week'], [30, 'in the last month'], [90, 'in the last 3 months']], () => f.activeDays, v => { f.activeDays = Number(v) || 0; }));
|
|
257
|
-
more.append(select('Final review', 'review', [['', 'any'], ['PASS', 'passed'], ['FAIL', 'failed'], ['none', 'no verdict']], () => f.review, v => { f.review = v; }));
|
|
257
|
+
more.append(select('Final review', 'review', [['', 'any'], ['PASS', 'passed'], ['FAIL', 'failed'], ['OUTDATED', 'outdated'], ['none', 'no verdict']], () => f.review, v => { f.review = v; }));
|
|
258
258
|
more.append(select('Verification', 'verification', [['', 'any'], ['PASS', 'passed'], ['FAIL', 'failed'], ['none', 'not run']], () => f.verification, v => { f.verification = v; }));
|
|
259
259
|
const dec = Object.keys(O.decompose);
|
|
260
260
|
if (dec.length > 1) more.append(chips('Decompose', dec.map(k => [k, k === 'none' ? 'off' : k, O.decompose[k]]), () => f.decompose, v => { f.decompose = v; }));
|
|
@@ -290,13 +290,22 @@ window.ForgeCloudUI = (() => {
|
|
|
290
290
|
head.append(shut);
|
|
291
291
|
card.append(head, el('h2', String(r.goal || 'Untitled run').split('\n')[0]), el('p', r.run_id || '', 'mono muted'));
|
|
292
292
|
const leds = el('span', undefined, 'leds');
|
|
293
|
-
for (const t of list(r.task_statuses).slice(0, 120)) {
|
|
293
|
+
for (const t of list(r.task_statuses).slice(0, 120)) {
|
|
294
|
+
const st = RM.taskStatus(t);
|
|
295
|
+
const l = el('i', undefined, 'led st-' + String(st || 'pending').toLowerCase());
|
|
296
|
+
l.title = `${t.id}: ${st}${t.title ? ' · ' + t.title : ''}`;
|
|
297
|
+
leds.append(l);
|
|
298
|
+
}
|
|
294
299
|
card.append(leds);
|
|
295
300
|
const dl = el('dl', undefined, 'fields');
|
|
296
301
|
const mix = ['high', 'medium', 'low', 'none'].filter(k => f.diff[k]).map(k => `${f.diff[k]} ${k === 'none' ? 'unset' : k}`).join(' · ');
|
|
297
302
|
const plates = specs => { if (!specs.length || !window.ForgeIcons) return specs.join(', ') || '—'; const s = el('span', undefined, 'plates'); for (const x of specs) s.append(window.ForgeIcons.modelPlate(x)); return s; };
|
|
303
|
+
const verdict = r.final_review && typeof r.final_review === 'object' ? r.final_review.verdict : r.final_review;
|
|
304
|
+
const isOutdated = verdict && RM.runReviewStatus(r) === 'OUTDATED';
|
|
305
|
+
const reviewText = verdict ? (isOutdated ? `${verdict} (outdated)` : verdict) : '—';
|
|
306
|
+
const reviewVal = isOutdated ? el('span', reviewText, 'chip st-outdated') : reviewText;
|
|
298
307
|
for (const [k, v, mono] of [['Tasks', `${f.n}${mix ? ' · ' + mix : ''}`], ['Implementers', plates([...f.dwarf])], ['Reviewers', plates([...f.qa])],
|
|
299
|
-
['Started', when(r.created)], ['Updated', when(r.updated)], ['Final review',
|
|
308
|
+
['Started', when(r.created)], ['Updated', when(r.updated)], ['Final review', reviewVal], ['Verification', r.verification_status || '—'], ['Decompose', r.decompose_level || 'off']]) {
|
|
300
309
|
const dd = el('dd', undefined, mono ? 'mono' : ''); dd.append(v instanceof Node ? v : document.createTextNode(String(v))); dl.append(el('dt', k), dd);
|
|
301
310
|
}
|
|
302
311
|
card.append(dl);
|
|
@@ -345,7 +354,7 @@ window.ForgeCloudUI = (() => {
|
|
|
345
354
|
a.href = '#/runs/' + encodeURIComponent(r.key);
|
|
346
355
|
a.append(el('strong', String(r.goal || 'Untitled run').split('\n')[0]), el('span', `${r.run_id || ''} · ${f.n} task${f.n === 1 ? '' : 's'}${r.kind === 'solo' ? ' · solo' : ''}`, 'mono muted'));
|
|
347
356
|
const leds = el('span', undefined, 'leds');
|
|
348
|
-
for (const t of list(r.task_statuses).slice(0, 60)) leds.append(el('i', undefined, 'led st-' + String(t
|
|
357
|
+
for (const t of list(r.task_statuses).slice(0, 60)) leds.append(el('i', undefined, 'led st-' + String(RM.taskStatus(t) || 'pending').toLowerCase()));
|
|
349
358
|
a.append(leds); grid.append(a);
|
|
350
359
|
}
|
|
351
360
|
g.append(grid); listHost.append(g);
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
const MAX_FPS = 25, MAX_LABELS = 5, MAX_LABEL = 50, MAX_REASON = 500;
|
|
17
17
|
const list = v => Array.isArray(v) ? v : [];
|
|
18
18
|
const SEVERITIES = ['P0', 'P1', 'P2', 'P3'];
|
|
19
|
-
const ORIGINS = ['tolerated', 'review', 'plausible'];
|
|
19
|
+
const ORIGINS = ['tolerated', 'accepted', 'review', 'plausible'];
|
|
20
20
|
const STATUSES = ['open', 'filed', 'fixed', 'dismissed'];
|
|
21
21
|
|
|
22
22
|
/* ---------- pure ---------- */
|
|
@@ -106,6 +106,20 @@
|
|
|
106
106
|
}
|
|
107
107
|
// An untrusted URL becomes a link only when it is a plain https address.
|
|
108
108
|
const safeUrl = u => typeof u === 'string' && /^https:\/\/[^\s"'<>]+$/.test(u) ? u : null;
|
|
109
|
+
// The marker for a finding whose whole-run review is outdated, or null. The lamp is the hollow unlit
|
|
110
|
+
// `.st-outdated`. `title` is the same sentence the issue body adds, ids cut after 5, so a row can show it
|
|
111
|
+
// without building HTML. A current review, and any finding that is not from a whole-run review, has none.
|
|
112
|
+
function reviewMarker(row) {
|
|
113
|
+
const review = row && row.review;
|
|
114
|
+
if (!review || review.outdated !== true) return null;
|
|
115
|
+
const ids = list(review.merged_since).map(id => String(id));
|
|
116
|
+
const shown = ids.slice(0, 5), extra = ids.length - shown.length;
|
|
117
|
+
const names = shown.join(', ') + (extra > 0 ? ', +' + extra + ' more' : '');
|
|
118
|
+
const n = ids.length;
|
|
119
|
+
const sentence = `The review is outdated: ${n} ${n === 1 ? 'task' : 'tasks'} merged into the integration branch after it`
|
|
120
|
+
+ (names ? ` (${names})` : '') + ', so check whether they fixed it first.';
|
|
121
|
+
return { lamp: 'st-outdated', label: 'outdated review', title: sentence, sentence };
|
|
122
|
+
}
|
|
109
123
|
|
|
110
124
|
/* ---------- DOM ---------- */
|
|
111
125
|
function render(host, ctx) {
|
|
@@ -198,6 +212,16 @@
|
|
|
198
212
|
}
|
|
199
213
|
|
|
200
214
|
// cells
|
|
215
|
+
const originCell = r => {
|
|
216
|
+
const marker = reviewMarker(r);
|
|
217
|
+
if (!marker) return r.origin || '—';
|
|
218
|
+
const box = el('span', undefined, 'fnd-status');
|
|
219
|
+
const lamp = el('i', undefined, 'lamp ' + marker.lamp);
|
|
220
|
+
lamp.title = marker.title;
|
|
221
|
+
box.title = marker.title;
|
|
222
|
+
box.append(lamp, el('span', r.origin || '—'), el('span', marker.label));
|
|
223
|
+
return box;
|
|
224
|
+
};
|
|
201
225
|
const statusCell = r => {
|
|
202
226
|
const st = statusOf(r), box = el('span', undefined, 'fnd-status fnd-' + st);
|
|
203
227
|
box.append(el('i', undefined, 'lamp ' + (st === 'filed' ? 'fnd-amber' : st === 'fixed' ? 'st-pass' : 'lamp-off')));
|
|
@@ -223,6 +247,8 @@
|
|
|
223
247
|
more.append(el('span', '▸', 'fnd-caret'));
|
|
224
248
|
const detail = el('div', undefined, 'fnd-detail'); detail.hidden = true;
|
|
225
249
|
detail.append(el('pre', r.text || r.summary || '', 'fnd-text'));
|
|
250
|
+
const marker = reviewMarker(r);
|
|
251
|
+
if (marker) detail.append(el('p', marker.sentence, 'muted'));
|
|
226
252
|
const meta = el('dl', undefined, 'fnd-meta');
|
|
227
253
|
for (const [k, v] of [['Reviewer', r.reviewer], ['Commit', r.commit], ['Fingerprint', r.fp], ['Plan', r.plan_dir || r.plan],
|
|
228
254
|
['Dismissed', r.dismissed ? `${r.dismissed.at || ''} ${r.dismissed.reason || ''}`.trim() : null]]) {
|
|
@@ -245,7 +271,7 @@
|
|
|
245
271
|
cols.push(
|
|
246
272
|
{ id: 'severity', label: 'Severity', value: r => SEVERITIES.indexOf(r.severity), cell: sevCell },
|
|
247
273
|
{ id: 'confidence', label: 'Confidence', value: r => r.confidence, cell: r => r.confidence || '—' },
|
|
248
|
-
{ id: 'origin', label: 'Origin', value: r => r.origin, cell:
|
|
274
|
+
{ id: 'origin', label: 'Origin', value: r => r.origin, cell: originCell },
|
|
249
275
|
{ id: 'task', label: 'Task', value: r => r.task, mono: true, cell: r => r.task || 'whole run' });
|
|
250
276
|
if (s.scope === 'repo') cols.push({ id: 'plan', label: 'Plan', value: r => r.plan, mono: true, cell: r => r.plan || '—' });
|
|
251
277
|
cols.push(
|
|
@@ -408,5 +434,5 @@
|
|
|
408
434
|
}
|
|
409
435
|
|
|
410
436
|
return { render, statusOf, filterRows, mergeFiled, selectable, selection, issuesUrl, issuesPath, dismissPath, fileBody, dismissBody,
|
|
411
|
-
parseLabels, resultLines, safeUrl, MAX_FPS };
|
|
437
|
+
parseLabels, resultLines, safeUrl, reviewMarker, ORIGINS, MAX_FPS };
|
|
412
438
|
});
|
|
@@ -201,8 +201,9 @@
|
|
|
201
201
|
if (drawn[t.id]) return;
|
|
202
202
|
drawn[t.id] = true;
|
|
203
203
|
var p = pos[t.id];
|
|
204
|
-
var status =
|
|
205
|
-
var
|
|
204
|
+
var status = window.ForgeRunModel.taskStatus(t);
|
|
205
|
+
var verdict = window.ForgeRunModel.taskVerdict(t);
|
|
206
|
+
var stClass = 'st-' + String(status || 'pending').toLowerCase().replace(/[^a-z0-9_-]/g, '');
|
|
206
207
|
var selected = options.selected === t.id;
|
|
207
208
|
var id = String(t.id);
|
|
208
209
|
|
|
@@ -217,7 +218,7 @@
|
|
|
217
218
|
|
|
218
219
|
var tip = [str(t.title, id), 'status: ' + status,
|
|
219
220
|
'dwarf: ' + str(t.dwarf, '—') + ' / qa: ' + str(t.qa, '—')];
|
|
220
|
-
if (
|
|
221
|
+
if (verdict.text) tip.push('verdict: ' + verdict.text);
|
|
221
222
|
if (t.explored) tip.push('explored');
|
|
222
223
|
var title = el('title', null, g);
|
|
223
224
|
title.textContent = tip.join('\n');
|
|
@@ -238,8 +239,8 @@
|
|
|
238
239
|
x: 12, y: 36, 'font-size': 12, fill: 'currentColor', 'fill-opacity': 0.75
|
|
239
240
|
});
|
|
240
241
|
|
|
241
|
-
if (
|
|
242
|
-
var pass =
|
|
242
|
+
if (verdict.mark === 'PASS' || verdict.mark === 'FAIL') {
|
|
243
|
+
var pass = verdict.mark === 'PASS';
|
|
243
244
|
// a drawn tick or cross, in the verdict's colour
|
|
244
245
|
el('path', { 'class': 'fg-verdict' + (pass ? ' fg-pass' : ''), d: pass ? 'M' + (NODE_W - 20) + ' 12l4 4 7-8' : 'M' + (NODE_W - 19) + ' 8l8 8m0-8l-8 8',
|
|
245
246
|
fill: 'none', 'stroke-width': 1.8, 'stroke-linecap': 'round', 'stroke-linejoin': 'round' }, g);
|
|
@@ -149,6 +149,7 @@ window.ForgeLeaderboard = (() => {
|
|
|
149
149
|
['Verdicts', '', 'PASS and FAIL verdicts over every review'],
|
|
150
150
|
...MS.SEVERITIES.map(s => [s, 'lb-num', SEV_TITLE(s)]),
|
|
151
151
|
['Shadow agreement', 'lb-num', 'How often this model\'s shadow verdict matched the primary reviewer\'s (both PASS or FAIL)'],
|
|
152
|
+
['Did not hold', 'lb-num', 'Of the work this model PASSed that was integrated, the share later reverted or with under 80% of its lines kept (forge dataset reviewers, every run in the dataset: the filters do not apply). Advisory: routing never reads it'],
|
|
152
153
|
['Median time', 'lb-num', 'Median duration of a review'],
|
|
153
154
|
['Tokens in / out', 'lb-num', 'Summed over the reviews that recorded tokens'],
|
|
154
155
|
];
|
|
@@ -182,7 +183,23 @@ window.ForgeLeaderboard = (() => {
|
|
|
182
183
|
td.title = `${a.k} of ${a.n} shadow verdicts matched the primary's; 95% Wilson interval ${pct(a.lo)}–${pct(a.hi)}${a.unknown ? ` · ${a.unknown} without two comparable verdicts` : ''}`;
|
|
183
184
|
return td;
|
|
184
185
|
}
|
|
186
|
+
function heldCell(h) {
|
|
187
|
+
const days = h.horizon ? ` ${h.horizon} days after integrate` : '';
|
|
188
|
+
if (h.state === 'unavailable') return numCell('—', 'The run dataset could not be read (forge dataset reviewers)');
|
|
189
|
+
if (h.state === 'none') return numCell('—', `No work this model PASSed has a survival outcome${days} yet (forge dataset outcomes records them)`);
|
|
190
|
+
const detail = `${h.reverted} reverted, ${h.rewritten} rewritten, of ${h.n} PASSed and integrated${days}`;
|
|
191
|
+
if (h.state === 'few') {
|
|
192
|
+
const td = numCell('not enough data', `${detail}; a rate needs ${h.need || 'more'} outcomes`);
|
|
193
|
+
td.classList.add('muted');
|
|
194
|
+
return td;
|
|
195
|
+
}
|
|
196
|
+
const td = el('td', undefined, 'lb-num');
|
|
197
|
+
td.append(el('strong', pct(h.p)), document.createTextNode(' '), el('span', `${h.k}/${h.n}`, 'muted'));
|
|
198
|
+
td.title = `${detail}; 95% Wilson interval ${pct(h.lo)}–${pct(h.hi)}`;
|
|
199
|
+
return td;
|
|
200
|
+
}
|
|
185
201
|
function reviewerTable(rows, ctx) {
|
|
202
|
+
const held = r => MS.heldUp(ctx.reviewerOutcomes, r.alias);
|
|
186
203
|
const columns = colsOf(REV_COLS, [
|
|
187
204
|
{ id: 'model', value: r => r.alias, cell: r => modelCell(r.alias, ctx, '') },
|
|
188
205
|
{ id: 'primary', value: r => r.reviews.primary, cell: r => numCell(r.reviews.primary) },
|
|
@@ -191,6 +208,7 @@ window.ForgeLeaderboard = (() => {
|
|
|
191
208
|
{ id: 'verdicts', value: r => { const v = r.verdicts, t = v.PASS + v.FAIL + v.other; return t ? v.PASS / t : null; }, cell: r => verdictCell(r.verdicts) },
|
|
192
209
|
...MS.SEVERITIES.map(sv => ({ id: 'sev-' + sv, value: r => r.findings[sv], cell: r => findingCell(r, sv) })),
|
|
193
210
|
{ id: 'agreement', value: r => r.agreement.n ? r.agreement.p : null, cell: r => agreementCell(r.agreement) },
|
|
211
|
+
{ id: 'held', value: r => { const h = held(r); return h.state === 'rate' ? h.p : null; }, cell: r => heldCell(held(r)) },
|
|
194
212
|
{ id: 'time', value: r => num(r.medianDurationS), cell: r => numCell(dur(r.medianDurationS)) },
|
|
195
213
|
{ id: 'tokens', value: r => (r.tokens.attempts ?? r.tokens.reviews) ? (r.tokens.input || 0) + (r.tokens.output || 0) : null, cell: r => tokenCell(r.tokens, r.reviews.total, 'review') },
|
|
196
214
|
]);
|
|
@@ -245,5 +245,23 @@
|
|
|
245
245
|
return { implementers: imp.rows, reviewers: rev.rows, byDifficulty, totals, caveats, filter: f };
|
|
246
246
|
}
|
|
247
247
|
|
|
248
|
-
|
|
248
|
+
// How the work a reviewer PASSed held up, from GET /api/models/reviewers (`forge dataset reviewers --json`):
|
|
249
|
+
// { state: 'unavailable' } without a report, 'none' when the alias has no landed outcome, 'few' below the
|
|
250
|
+
// report's minimum (no rate is shown then), else 'rate' with the share that did not hold and its interval.
|
|
251
|
+
function heldUp(report, alias) {
|
|
252
|
+
const r = obj(report);
|
|
253
|
+
if (!r || !Array.isArray(r.reviewers)) return { state: 'unavailable' };
|
|
254
|
+
const horizon = num(r.horizon_days), row = r.reviewers.find(x => obj(x) && x.reviewer === alias);
|
|
255
|
+
if (!row) return { state: 'none', horizon };
|
|
256
|
+
const n = num(row.landed), k = num(row.did_not_hold);
|
|
257
|
+
if (n === null || k === null) return { state: 'unavailable' };
|
|
258
|
+
const base = { n, k, reverted: num(row.reverted) || 0, rewritten: num(row.rewritten) || 0, horizon };
|
|
259
|
+
const ci = list(row.ci);
|
|
260
|
+
if (row.enough !== true || num(row.rate) === null || num(ci[0]) === null || num(ci[1]) === null) {
|
|
261
|
+
return { state: 'few', need: num(r.min_landed), ...base };
|
|
262
|
+
}
|
|
263
|
+
return { state: 'rate', p: row.rate, lo: ci[0], hi: ci[1], ...base };
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
return { DIFFS, KINDS, SEVERITIES, DEFAULT_MIN_N, wilson, emptyFilter, normFilter, taskMatch, options, leaderboard, heldUp };
|
|
249
267
|
});
|
|
@@ -9,7 +9,7 @@ window.ForgePanel = (() => {
|
|
|
9
9
|
const num = v => typeof v === 'number' && Number.isFinite(v) ? v : null;
|
|
10
10
|
const display = v => v === null || v === undefined || v === '' ? '—' : typeof v === 'object' ? JSON.stringify(v, null, 2) : String(v);
|
|
11
11
|
const STATUSES = new Set(['merged', 'pass', 'running', 'pending', 'fail', 'error', 'infra', 'timeout', 'unknown', 'invalidated', 'blocked',
|
|
12
|
-
'interrupted', 'nochanges', 'conflict', 'accepted', 'none']);
|
|
12
|
+
'interrupted', 'nochanges', 'conflict', 'accepted', 'none', 'outdated']);
|
|
13
13
|
const cache = new Map(); // path → {data, error, at}
|
|
14
14
|
function el(tag, text, className) {
|
|
15
15
|
const node = document.createElement(tag);
|
|
@@ -51,6 +51,7 @@ window.ForgePanel = (() => {
|
|
|
51
51
|
function section(parent, title) { const s = el('section', undefined, 'insp-section'); s.append(el('h3', title)); parent.append(s); return s; }
|
|
52
52
|
function details(parent, title, value, key, { open = false, cls = '' } = {}) {
|
|
53
53
|
const d = el('details', undefined, cls); d.dataset.keep = key; if (open) d.open = true;
|
|
54
|
+
if (key && String(key).endsWith('known_issues')) d.id = key;
|
|
54
55
|
const pre = el('pre', value); pre.dataset.scroll = key + '-text';
|
|
55
56
|
d.append(el('summary', title), pre); parent.append(d); return d;
|
|
56
57
|
}
|
|
@@ -60,6 +61,21 @@ window.ForgePanel = (() => {
|
|
|
60
61
|
return a;
|
|
61
62
|
}
|
|
62
63
|
function muted(parent, text) { parent.append(el('p', text, 'muted')); }
|
|
64
|
+
// The acceptance chip is a key to the Known issues text in this panel, not another route.
|
|
65
|
+
function acceptedChip(task) {
|
|
66
|
+
const over = task.accepted_over === 'FAIL' || task.accepted_over === 'UNKNOWN' ? task.accepted_over : null;
|
|
67
|
+
const id = 'task-' + task.id + '-known_issues';
|
|
68
|
+
const a = el('a', over ? 'accepted over ' + over : 'accepted', 'chip st-accepted');
|
|
69
|
+
a.href = '#' + id;
|
|
70
|
+
a.addEventListener('click', event => {
|
|
71
|
+
event.preventDefault();
|
|
72
|
+
const node = document.getElementById(id);
|
|
73
|
+
if (!node) return;
|
|
74
|
+
node.open = true;
|
|
75
|
+
node.scrollIntoView({ block: 'nearest' });
|
|
76
|
+
});
|
|
77
|
+
return a;
|
|
78
|
+
}
|
|
63
79
|
// who decided an escalation: policy, a human's retry --dwarf (retry) or a pin; older rows read as policy
|
|
64
80
|
const sourcePlate = source => el('span', source || 'policy', 'plate src-plate src-' + String(source || 'policy').replace(/[^a-z]/g, ''));
|
|
65
81
|
// the write controls of a plan run (task undefined) or of one of its tasks; a page without controls.js says it is read-only
|
|
@@ -218,11 +234,23 @@ window.ForgePanel = (() => {
|
|
|
218
234
|
}
|
|
219
235
|
const f = section(host, 'Finish');
|
|
220
236
|
const F = RM.finishState(run);
|
|
221
|
-
if (run.kind === 'solo')
|
|
237
|
+
if (run.kind === 'solo') {
|
|
238
|
+
const soloTask = list(run.tasks)[0];
|
|
239
|
+
const verdict = soloTask ? RM.taskVerdict(soloTask) : { text: null, status: 'unknown' };
|
|
240
|
+
fields(f, [['Verdict', chip(verdict.text, verdict.status)]]);
|
|
241
|
+
}
|
|
222
242
|
else {
|
|
243
|
+
const finalSel = { kind: 'final' }, openFinal = () => ctx.onSelect(finalSel);
|
|
244
|
+
let finalValue;
|
|
245
|
+
if (F.final.status === 'OUTDATED') {
|
|
246
|
+
finalValue = el('a'); finalValue.href = ctx.hash(finalSel);
|
|
247
|
+
finalValue.addEventListener('click', e => { e.preventDefault(); openFinal(); });
|
|
248
|
+
finalValue.append(chip('OUTDATED'));
|
|
249
|
+
} else finalValue = link(F.final.status === 'NONE' ? 'not requested' : F.final.status, ctx.hash(finalSel), openFinal);
|
|
223
250
|
fields(f, [['Integration', link(`${F.integration.merged}/${F.integration.total} merged`, ctx.hash({ kind: 'integration' }), () => ctx.onSelect({ kind: 'integration' }))],
|
|
224
251
|
['Verification', link(F.verification.status, ctx.hash({ kind: 'verification' }), () => ctx.onSelect({ kind: 'verification' }))],
|
|
225
|
-
['Final review',
|
|
252
|
+
['Final review', finalValue]]);
|
|
253
|
+
if (F.final.status === 'OUTDATED') muted(f, RM.reviewSinceLine(F.final));
|
|
226
254
|
if (F.final.findings) findingCounts(f, F.final.findings);
|
|
227
255
|
}
|
|
228
256
|
if (list(run.deferrals).length) details(host, `Deferred tasks (${run.deferrals.length})`, run.deferrals.join('\n'), 'deferrals');
|
|
@@ -236,10 +264,13 @@ window.ForgePanel = (() => {
|
|
|
236
264
|
const head = el('div', undefined, 'insp-head');
|
|
237
265
|
head.append(el('h2', solo ? (task.title || 'Solo run') : `${task.id} · ${task.title || 'Untitled task'}`), el('div', solo ? 'solo task' : `task · wave ${display(task.wave)}`, 'insp-kind'));
|
|
238
266
|
const chips = el('div', undefined, 'chips');
|
|
239
|
-
chips.append(
|
|
240
|
-
|
|
267
|
+
if (RM.taskStatus(task) === 'ACCEPTED') chips.append(acceptedChip(task));
|
|
268
|
+
else {
|
|
269
|
+
chips.append(chip(RM.taskStatus(task)));
|
|
270
|
+
if (task.accepted === true) chips.append(acceptedChip(task));
|
|
271
|
+
else if (task.verdict) chips.append(chip('review ' + task.verdict, task.verdict.toLowerCase()));
|
|
272
|
+
}
|
|
241
273
|
if (task.explored) chips.append(el('span', '* explored', 'tag'));
|
|
242
|
-
if (task.accepted_by) chips.append(chip('accepted over FAIL', 'nochanges'));
|
|
243
274
|
head.append(chips);
|
|
244
275
|
const dl = diffLink(ctx, solo ? 'View the diff →' : 'View its diff, findings pinned to their lines →', { kind: 'task', task: task.id, source: 'current' }); if (dl) head.append(dl);
|
|
245
276
|
host.append(head);
|
|
@@ -270,7 +301,7 @@ window.ForgePanel = (() => {
|
|
|
270
301
|
for (const z of tp.zones) {
|
|
271
302
|
const box = el('div', undefined, 'attempt');
|
|
272
303
|
const row = list(task.attempts).filter(a => a.attempt === z.attempt).pop();
|
|
273
|
-
const h = el('div', undefined, 'attempt-head'); h.append(el('strong', z.name.toLowerCase())); if (row) h.append(chip(row.status), el('span', time(row.at), 'muted'));
|
|
304
|
+
const h = el('div', undefined, 'attempt-head'); h.append(el('strong', z.name.toLowerCase())); if (row) h.append(chip(RM.attemptStatus(task, row.attempt, row.status)), el('span', time(row.at), 'muted'));
|
|
274
305
|
box.append(h);
|
|
275
306
|
for (const pid of z.parts) {
|
|
276
307
|
const p = tp.parts.find(x => x.id === pid); if (!p || p.planned) continue;
|
|
@@ -329,7 +360,8 @@ window.ForgePanel = (() => {
|
|
|
329
360
|
const cs = section(host, 'Clef predictions');
|
|
330
361
|
for (const [label, p] of preds) {
|
|
331
362
|
const n = Number(String(label).match(/\d+/) || 1), outcome = list(task.attempts).filter(a => a.attempt === n).pop();
|
|
332
|
-
|
|
363
|
+
const happened = outcome ? RM.attemptStatus(task, outcome.attempt, outcome.status) : 'not finished';
|
|
364
|
+
fields(cs, [[label, `p(pass) = ${display(p.p)} → ${happened}`], ['Assignment', p.assignment ? plates([p.assignment.dwarf, p.assignment.qa]) : '—']]);
|
|
333
365
|
}
|
|
334
366
|
}
|
|
335
367
|
// the prompts and the final messages
|
|
@@ -338,10 +370,18 @@ window.ForgePanel = (() => {
|
|
|
338
370
|
else {
|
|
339
371
|
const t = detail.data.texts || {};
|
|
340
372
|
for (const [title, key] of [['Requirements (prompt.md)', 'prompt'], ['Intended approach', 'approach'], ['Retry findings', 'retry_findings'],
|
|
341
|
-
['Dwarf prompt as assembled (dwarf.input)', 'dwarf_input'], ['Review prompt as assembled (qa.input)', 'qa_input']
|
|
373
|
+
['Dwarf prompt as assembled (dwarf.input)', 'dwarf_input'], ['Review prompt as assembled (qa.input)', 'qa_input']]) {
|
|
342
374
|
if (t[key] != null) details(ps, title, t[key], `task-${task.id}-${key}`);
|
|
343
375
|
}
|
|
344
376
|
}
|
|
377
|
+
// Known issues is the text the acceptance chip opens. Accepted by keeps the reason, who, and when.
|
|
378
|
+
const texts = detail.data && detail.data.texts || null;
|
|
379
|
+
if (task.accepted === true || (texts && texts.known_issues != null)) {
|
|
380
|
+
const known = texts ? texts.known_issues : null;
|
|
381
|
+
details(ps, 'Known issues', known != null ? known : (detail.data ? '—' : 'Loading…'), `task-${task.id}-known_issues`);
|
|
382
|
+
}
|
|
383
|
+
const acceptedBy = texts && texts.accepted_by != null ? texts.accepted_by : task.accepted_by;
|
|
384
|
+
if (acceptedBy != null) details(ps, 'Accepted by', acceptedBy, `task-${task.id}-accepted_by`);
|
|
345
385
|
if (task.dwarf_tail != null) details(ps, 'Dwarf final message', task.dwarf_tail, `task-${task.id}-dwarf_tail`);
|
|
346
386
|
if (task.qa_tail != null) details(ps, 'Reviewer final message', task.qa_tail, `task-${task.id}-qa_tail`);
|
|
347
387
|
}
|
|
@@ -475,10 +515,12 @@ window.ForgePanel = (() => {
|
|
|
475
515
|
return;
|
|
476
516
|
}
|
|
477
517
|
const chips = el('div', undefined, 'chips');
|
|
478
|
-
|
|
518
|
+
const status = RM.finalStatus(pick);
|
|
519
|
+
chips.append(chip(status));
|
|
479
520
|
if (pick.published) chips.append(el('span', 'published', 'tag'));
|
|
480
521
|
chips.append(el('span', pick.source === 'integrate' ? 'during integrate' : 'forge review', 'tag'));
|
|
481
522
|
head.append(chips);
|
|
523
|
+
if (status === 'OUTDATED') muted(host, RM.reviewSinceLine(pick));
|
|
482
524
|
fields(host, [['Reviewer', plate(pick.spec)], ['Threshold', pick.threshold], ['Reviewed', pick.base || pick.tip ? `${String(pick.base || '?').slice(0, 10)}..${String(pick.tip || '?').slice(0, 10)}` : '—', true],
|
|
483
525
|
['Started', time(pick.started ?? pick.at)], ['Ended', time(pick.ended)], ['Duration', RM.fmtDur(pick.duration_s)]]);
|
|
484
526
|
if (pick.execution) { const s = { kind: 'execution', task: null, exec: pick.execution }; host.append(link('Model, timings and exact prompt →', ctx.hash(s), () => ctx.onSelect(s))); }
|