deepline 0.2.41 → 0.2.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +9 -6
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +10 -0
- package/dist/cli/index.js +3 -1
- package/dist/cli/index.mjs +3 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/viewer/viewer.css +366 -88
- package/dist/viewer/viewer.js +404 -37
- package/package.json +1 -1
package/dist/viewer/viewer.js
CHANGED
|
@@ -871,20 +871,33 @@ function processRawSession(raw) {
|
|
|
871
871
|
|
|
872
872
|
const toolCalls = timeline.filter(e => e.kind === 'tool_call').length;
|
|
873
873
|
const toolErrors = timeline.filter(e => e.kind === 'tool_call' && e.is_error).length;
|
|
874
|
+
const canonicalPerformance = raw.transcript_unavailable === true
|
|
875
|
+
&& raw.performance && typeof raw.performance === 'object'
|
|
876
|
+
? raw.performance
|
|
877
|
+
: null;
|
|
878
|
+
const canonicalNumber = (key, fallback) => {
|
|
879
|
+
const value = canonicalPerformance && canonicalPerformance[key];
|
|
880
|
+
if (value == null || !Number.isFinite(Number(value))) return fallback;
|
|
881
|
+
return Number(value);
|
|
882
|
+
};
|
|
874
883
|
|
|
875
884
|
return {
|
|
876
885
|
label: raw.label,
|
|
886
|
+
transcript_available: raw.transcript_unavailable !== true,
|
|
877
887
|
meta: meta,
|
|
878
888
|
prompt: prompt,
|
|
879
889
|
result: result,
|
|
880
890
|
timeline: timeline,
|
|
881
891
|
timing_mode: timingMode,
|
|
882
892
|
stats: {
|
|
883
|
-
duration_s: totalDurationS,
|
|
884
|
-
tool_calls: toolCalls,
|
|
885
|
-
tool_errors: toolErrors,
|
|
886
|
-
num_turns: numTurns,
|
|
887
|
-
cost_usd: (
|
|
893
|
+
duration_s: canonicalNumber('duration_s', totalDurationS),
|
|
894
|
+
tool_calls: canonicalNumber('tool_calls', toolCalls),
|
|
895
|
+
tool_errors: canonicalNumber('tool_errors', toolErrors),
|
|
896
|
+
num_turns: canonicalNumber('turns', numTurns),
|
|
897
|
+
cost_usd: canonicalNumber(
|
|
898
|
+
'cost_usd',
|
|
899
|
+
(result || {}).total_cost_usd != null ? result.total_cost_usd : null,
|
|
900
|
+
),
|
|
888
901
|
tool_breakdown: toolBreakdown,
|
|
889
902
|
file_touches: fileTouches,
|
|
890
903
|
loop_groups: analysis.loopGroups,
|
|
@@ -905,6 +918,135 @@ let currentSession = 0;
|
|
|
905
918
|
let filterErrors = false;
|
|
906
919
|
let isLive = false;
|
|
907
920
|
let liveIntervalId = null;
|
|
921
|
+
let currentEval = '';
|
|
922
|
+
let currentSelection = 'all';
|
|
923
|
+
let sliceBy = 'run';
|
|
924
|
+
let runFilter = '';
|
|
925
|
+
let agentFilter = 'all';
|
|
926
|
+
let modelFilter = 'all';
|
|
927
|
+
let skillFilter = 'all';
|
|
928
|
+
let visibleEvalKeys = [];
|
|
929
|
+
let visibleSelectionKeys = [];
|
|
930
|
+
|
|
931
|
+
function readEvalMetrics() {
|
|
932
|
+
const el = document.getElementById('eval-harness-metrics');
|
|
933
|
+
if (!el || !el.textContent) return [];
|
|
934
|
+
try {
|
|
935
|
+
const parsed = JSON.parse(el.textContent);
|
|
936
|
+
return Array.isArray(parsed) ? parsed : [];
|
|
937
|
+
} catch(e) { return []; }
|
|
938
|
+
}
|
|
939
|
+
|
|
940
|
+
function firstPresent(values, fallback) {
|
|
941
|
+
for (const value of values) {
|
|
942
|
+
if (value != null && String(value).trim()) return String(value).trim();
|
|
943
|
+
}
|
|
944
|
+
return fallback;
|
|
945
|
+
}
|
|
946
|
+
|
|
947
|
+
function inferEval(label) {
|
|
948
|
+
return String(label || 'Session')
|
|
949
|
+
.replace(/\s+run\s+\d+.*$/i, '')
|
|
950
|
+
.replace(/\s+[·|].*$/, '')
|
|
951
|
+
.trim() || 'Session';
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
function inferRun(label, index) {
|
|
955
|
+
const match = String(label || '').match(/\brun\s+(\d+)/i);
|
|
956
|
+
return match ? match[1] : String(index + 1);
|
|
957
|
+
}
|
|
958
|
+
|
|
959
|
+
function inferAgent(label, meta) {
|
|
960
|
+
const match = String(label || '').match(/[·|]\s*([^/·|]+)\//);
|
|
961
|
+
const env = (meta || {}).environment || {};
|
|
962
|
+
return firstPresent([match && match[1], env.agent_runtime], 'Unknown agent');
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
function inferModel(label, meta) {
|
|
966
|
+
const match = String(label || '').match(/[·|]\s*[^/·|]+\/([^·|]+)/);
|
|
967
|
+
return firstPresent([match && match[1], (meta || {}).model], 'Unknown model');
|
|
968
|
+
}
|
|
969
|
+
|
|
970
|
+
function inferSkill(session) {
|
|
971
|
+
const promptMatch = String(session.prompt || '').match(/\b(deepline-(?:plays|gtm))\b/i);
|
|
972
|
+
if (promptMatch) return promptMatch[1].toLowerCase();
|
|
973
|
+
for (const entry of session.timeline || []) {
|
|
974
|
+
if (entry.kind !== 'tool_call' || entry.tool !== 'Skill') continue;
|
|
975
|
+
const value = String(entry.command || (entry.input || {}).skill || '');
|
|
976
|
+
const match = value.match(/\b(deepline-[\w-]+)\b/i);
|
|
977
|
+
if (match) return match[1].toLowerCase();
|
|
978
|
+
}
|
|
979
|
+
return 'Unspecified';
|
|
980
|
+
}
|
|
981
|
+
|
|
982
|
+
function attachEvalMetadata() {
|
|
983
|
+
const metrics = readEvalMetrics();
|
|
984
|
+
const bySession = new Map(metrics.filter(m => m && m.session_id).map(m => [String(m.session_id), m]));
|
|
985
|
+
const byLabel = new Map(metrics.filter(m => m && m.label).map(m => [String(m.label), m]));
|
|
986
|
+
SESSIONS.forEach((session, index) => {
|
|
987
|
+
const metric = bySession.get(String(session.meta.session_id || '')) || byLabel.get(String(session.label || '')) || null;
|
|
988
|
+
session.evalMetric = metric;
|
|
989
|
+
session.sourceIndex = index;
|
|
990
|
+
session.dimensions = {
|
|
991
|
+
eval: firstPresent([metric && metric.eval_id], inferEval(session.label)),
|
|
992
|
+
run: firstPresent([metric && metric.run], inferRun(session.label, index)),
|
|
993
|
+
agent: firstPresent([metric && metric.agent], inferAgent(session.label, session.meta)),
|
|
994
|
+
model: firstPresent([metric && metric.model_id, metric && metric.requested_model, metric && metric.model], inferModel(session.label, session.meta)),
|
|
995
|
+
skill: firstPresent([metric && metric.invoke_skill], inferSkill(session)),
|
|
996
|
+
};
|
|
997
|
+
});
|
|
998
|
+
}
|
|
999
|
+
|
|
1000
|
+
function uniqueValues(values) {
|
|
1001
|
+
return Array.from(new Set(values.filter(value => value != null && String(value).trim()).map(String)));
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
function sessionsForCurrentEval() {
|
|
1005
|
+
return SESSIONS.filter(s => s.dimensions.eval === currentEval);
|
|
1006
|
+
}
|
|
1007
|
+
|
|
1008
|
+
function selectionDimensions() {
|
|
1009
|
+
const dimensions = ['run', 'agent', 'model', 'skill'];
|
|
1010
|
+
return sliceBy === 'none' ? dimensions : dimensions.filter(dimension => dimension !== sliceBy);
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
function selectionKey(session) {
|
|
1014
|
+
return selectionDimensions().map(dimension => session.dimensions[dimension]).join('\u001f');
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
function selectionLabel(session) {
|
|
1018
|
+
const labels = {
|
|
1019
|
+
run: value => 'Run ' + value,
|
|
1020
|
+
agent: value => value,
|
|
1021
|
+
model: value => value,
|
|
1022
|
+
skill: value => value,
|
|
1023
|
+
};
|
|
1024
|
+
return selectionDimensions().map(dimension => labels[dimension](session.dimensions[dimension])).join(' · ') || 'All transcripts';
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
function matchesTopFilters(session) {
|
|
1028
|
+
const query = runFilter.trim().toLowerCase();
|
|
1029
|
+
const d = session.dimensions;
|
|
1030
|
+
if (agentFilter !== 'all' && d.agent !== agentFilter) return false;
|
|
1031
|
+
if (modelFilter !== 'all' && d.model !== modelFilter) return false;
|
|
1032
|
+
if (skillFilter !== 'all' && d.skill !== skillFilter) return false;
|
|
1033
|
+
return !query || [session.label, d.eval, d.run, d.agent, d.model, d.skill].join(' ').toLowerCase().includes(query);
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
function matchingConfigurations() {
|
|
1037
|
+
const groups = new Map();
|
|
1038
|
+
for (const session of sessionsForCurrentEval().filter(matchesTopFilters)) {
|
|
1039
|
+
const key = selectionKey(session);
|
|
1040
|
+
if (!groups.has(key)) groups.set(key, {key, label: selectionLabel(session), sessions: []});
|
|
1041
|
+
groups.get(key).sessions.push(session);
|
|
1042
|
+
}
|
|
1043
|
+
return Array.from(groups.values());
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
function filteredSessions() {
|
|
1047
|
+
return sessionsForCurrentEval().filter(session => matchesTopFilters(session)
|
|
1048
|
+
&& (currentSelection === 'all' || selectionKey(session) === currentSelection));
|
|
1049
|
+
}
|
|
908
1050
|
|
|
909
1051
|
function saveViewerState() {
|
|
910
1052
|
const main = document.getElementById('main-content');
|
|
@@ -925,6 +1067,13 @@ function saveViewerState() {
|
|
|
925
1067
|
expandedText: expandedText,
|
|
926
1068
|
filterErrors: filterErrors,
|
|
927
1069
|
search: searchVal,
|
|
1070
|
+
currentEval: currentEval,
|
|
1071
|
+
currentSelection: currentSelection,
|
|
1072
|
+
sliceBy: sliceBy,
|
|
1073
|
+
runFilter: runFilter,
|
|
1074
|
+
agentFilter: agentFilter,
|
|
1075
|
+
modelFilter: modelFilter,
|
|
1076
|
+
skillFilter: skillFilter,
|
|
928
1077
|
}));
|
|
929
1078
|
}
|
|
930
1079
|
|
|
@@ -953,33 +1102,47 @@ function toggleLive() {
|
|
|
953
1102
|
|
|
954
1103
|
function init() {
|
|
955
1104
|
SESSIONS = RAW_SESSIONS.map(processRawSession);
|
|
1105
|
+
attachEvalMetadata();
|
|
1106
|
+
ensureRunSidebar();
|
|
956
1107
|
|
|
957
1108
|
const saved = restoreViewerState();
|
|
958
1109
|
const initialSession = (saved && saved.session != null && saved.session < SESSIONS.length && saved.session >= -1) ? saved.session : 0;
|
|
959
1110
|
currentSession = initialSession;
|
|
1111
|
+
visibleEvalKeys = uniqueValues(SESSIONS.map(s => s.dimensions.eval));
|
|
1112
|
+
currentEval = saved && visibleEvalKeys.includes(saved.currentEval)
|
|
1113
|
+
? saved.currentEval
|
|
1114
|
+
: ((SESSIONS[Math.max(0, initialSession)] || {}).dimensions || {}).eval || visibleEvalKeys[0] || '';
|
|
1115
|
+
sliceBy = saved && ['none', 'run', 'agent', 'model', 'skill'].includes(saved.sliceBy)
|
|
1116
|
+
? saved.sliceBy
|
|
1117
|
+
: (SESSIONS.length > 1 ? 'run' : 'none');
|
|
1118
|
+
currentSelection = (saved && (saved.currentSelection || saved.currentRun)) || 'all';
|
|
1119
|
+
runFilter = (saved && saved.runFilter) || '';
|
|
1120
|
+
agentFilter = (saved && saved.agentFilter) || 'all';
|
|
1121
|
+
modelFilter = (saved && saved.modelFilter) || 'all';
|
|
1122
|
+
skillFilter = (saved && saved.skillFilter) || 'all';
|
|
960
1123
|
|
|
961
1124
|
if (SESSIONS.length > 1) {
|
|
962
1125
|
document.getElementById('sidebar').classList.remove('hidden');
|
|
1126
|
+
document.getElementById('run-sidebar').classList.remove('hidden');
|
|
963
1127
|
renderSidebar();
|
|
1128
|
+
renderRunSidebar();
|
|
964
1129
|
document.addEventListener('keydown', function(e) {
|
|
965
|
-
if (e.target.tagName === 'INPUT') return;
|
|
1130
|
+
if (e.target.tagName === 'INPUT' || e.target.tagName === 'SELECT') return;
|
|
966
1131
|
if (e.key === 'ArrowDown' || e.key === 'j') {
|
|
967
1132
|
e.preventDefault();
|
|
968
|
-
|
|
1133
|
+
const index = Math.max(0, visibleSelectionKeys.indexOf(currentSelection));
|
|
1134
|
+
if (index < visibleSelectionKeys.length - 1) switchSelection(index + 1);
|
|
969
1135
|
} else if (e.key === 'ArrowUp' || e.key === 'k') {
|
|
970
1136
|
e.preventDefault();
|
|
971
|
-
|
|
1137
|
+
const index = Math.max(0, visibleSelectionKeys.indexOf(currentSelection));
|
|
1138
|
+
if (index > 0) switchSelection(index - 1);
|
|
972
1139
|
} else if (e.key === 'q' || e.key === 'Q') {
|
|
973
1140
|
switchToCompare();
|
|
974
1141
|
}
|
|
975
1142
|
});
|
|
976
1143
|
}
|
|
977
1144
|
|
|
978
|
-
|
|
979
|
-
renderComparison();
|
|
980
|
-
} else {
|
|
981
|
-
renderSession(Math.max(0, initialSession));
|
|
982
|
-
}
|
|
1145
|
+
renderWorkspace();
|
|
983
1146
|
|
|
984
1147
|
if (saved) {
|
|
985
1148
|
if (saved.filterErrors) {
|
|
@@ -1032,33 +1195,226 @@ function init() {
|
|
|
1032
1195
|
|
|
1033
1196
|
function renderSidebar() {
|
|
1034
1197
|
const el = document.getElementById('sidebar');
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1198
|
+
el.classList.remove('hidden');
|
|
1199
|
+
visibleEvalKeys = uniqueValues(SESSIONS.map(s => s.dimensions.eval));
|
|
1200
|
+
let html = '<div class="rail-heading"><span>Evaluations</span><strong>' + visibleEvalKeys.length + '</strong></div>';
|
|
1201
|
+
html += visibleEvalKeys.map((evalName, i) => {
|
|
1202
|
+
const sessions = SESSIONS.filter(s => s.dimensions.eval === evalName);
|
|
1203
|
+
const runs = uniqueValues(sessions.map(s => s.dimensions.run)).length;
|
|
1204
|
+
return `<button class="sidebar-item ${evalName === currentEval ? 'active' : ''}" onclick="switchEval(${i})">
|
|
1205
|
+
<span class="label">${esc(evalName)}</span>
|
|
1206
|
+
<span class="sub">${runs} run${runs === 1 ? '' : 's'} · ${sessions.length} session${sessions.length === 1 ? '' : 's'}</span>
|
|
1207
|
+
</button>`;
|
|
1208
|
+
}).join('');
|
|
1209
|
+
el.innerHTML = html;
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
function ensureRunSidebar() {
|
|
1213
|
+
if (document.getElementById('run-sidebar')) return;
|
|
1214
|
+
const main = document.getElementById('main-content');
|
|
1215
|
+
if (!main || !main.parentElement) return;
|
|
1216
|
+
const rail = document.createElement('div');
|
|
1217
|
+
rail.id = 'run-sidebar';
|
|
1218
|
+
rail.className = 'run-sidebar hidden';
|
|
1219
|
+
main.parentElement.insertBefore(rail, main);
|
|
1220
|
+
}
|
|
1221
|
+
|
|
1222
|
+
function renderRunSidebar() {
|
|
1223
|
+
const el = document.getElementById('run-sidebar');
|
|
1224
|
+
if (!el) return;
|
|
1225
|
+
el.classList.remove('hidden');
|
|
1226
|
+
const groups = matchingConfigurations();
|
|
1227
|
+
visibleSelectionKeys = ['all'].concat(groups.map(group => group.key));
|
|
1228
|
+
if (!visibleSelectionKeys.includes(currentSelection)) currentSelection = 'all';
|
|
1229
|
+
const dimensions = selectionDimensions();
|
|
1230
|
+
const heading = dimensions.length ? 'Compare setup' : 'Transcripts';
|
|
1231
|
+
let html = `<div class="rail-heading"><span>${heading}</span><strong>${groups.length}</strong></div>`;
|
|
1232
|
+
html += `<button class="run-sidebar-item ${currentSelection === 'all' ? 'active' : ''}" onclick="switchSelection(0)">
|
|
1233
|
+
<span class="run-number">All matching</span><span class="run-meta">${sessionsForCurrentEval().filter(matchesTopFilters).length} transcripts · ${sliceBy === 'none' ? 'stacked' : 'sliced by ' + sliceBy}</span>
|
|
1234
|
+
</button>`;
|
|
1235
|
+
html += groups.map((group, index) => {
|
|
1236
|
+
const selectedDimensions = selectionDimensions();
|
|
1237
|
+
const muted = selectedDimensions.map(dimension => dimension[0].toUpperCase() + dimension.slice(1)).join(' · ');
|
|
1238
|
+
return `<button class="run-sidebar-item ${currentSelection === group.key ? 'active' : ''}" onclick="switchSelection(${index + 1})">
|
|
1239
|
+
<span class="run-number">${esc(group.label)}</span>
|
|
1240
|
+
<span class="run-meta">${esc(muted)} · ${group.sessions.length} transcript${group.sessions.length === 1 ? '' : 's'}</span>
|
|
1241
|
+
<span class="run-skill">${sliceBy === 'none' ? 'single transcript' : 'compare ' + esc(sliceBy)}</span>
|
|
1242
|
+
</button>`;
|
|
1048
1243
|
}).join('');
|
|
1049
1244
|
el.innerHTML = html;
|
|
1050
1245
|
}
|
|
1051
1246
|
|
|
1247
|
+
function switchEval(index) {
|
|
1248
|
+
currentEval = visibleEvalKeys[index] || visibleEvalKeys[0] || '';
|
|
1249
|
+
currentSelection = 'all';
|
|
1250
|
+
currentSession = (sessionsForCurrentEval()[0] || {}).sourceIndex || 0;
|
|
1251
|
+
agentFilter = 'all';
|
|
1252
|
+
modelFilter = 'all';
|
|
1253
|
+
skillFilter = 'all';
|
|
1254
|
+
renderSidebar();
|
|
1255
|
+
renderRunSidebar();
|
|
1256
|
+
renderWorkspace();
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1259
|
+
function switchSelection(index) {
|
|
1260
|
+
currentSelection = visibleSelectionKeys[index] || 'all';
|
|
1261
|
+
const selected = filteredSessions();
|
|
1262
|
+
if (selected.length) currentSession = selected[0].sourceIndex;
|
|
1263
|
+
renderRunSidebar();
|
|
1264
|
+
renderWorkspace();
|
|
1265
|
+
}
|
|
1266
|
+
|
|
1052
1267
|
function switchSession(i) {
|
|
1053
1268
|
currentSession = i;
|
|
1269
|
+
currentEval = SESSIONS[i].dimensions.eval;
|
|
1270
|
+
currentSelection = selectionKey(SESSIONS[i]);
|
|
1054
1271
|
renderSidebar();
|
|
1272
|
+
renderRunSidebar();
|
|
1055
1273
|
renderSession(i);
|
|
1056
1274
|
}
|
|
1057
1275
|
|
|
1058
1276
|
function switchToCompare() {
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1277
|
+
currentSelection = 'all';
|
|
1278
|
+
sliceBy = 'run';
|
|
1279
|
+
renderRunSidebar();
|
|
1280
|
+
renderWorkspace();
|
|
1281
|
+
}
|
|
1282
|
+
|
|
1283
|
+
function optionList(values, selected, allLabel) {
|
|
1284
|
+
let html = `<option value="all" ${selected === 'all' ? 'selected' : ''}>${esc(allLabel)}</option>`;
|
|
1285
|
+
html += values.map(value => `<option value="${esc(value)}" ${selected === value ? 'selected' : ''}>${esc(value)}</option>`).join('');
|
|
1286
|
+
return html;
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
function renderWorkspaceControls() {
|
|
1290
|
+
const sessions = sessionsForCurrentEval();
|
|
1291
|
+
const agents = uniqueValues(sessions.map(s => s.dimensions.agent));
|
|
1292
|
+
const models = uniqueValues(sessions.map(s => s.dimensions.model));
|
|
1293
|
+
const skills = uniqueValues(sessions.map(s => s.dimensions.skill));
|
|
1294
|
+
return `<section class="workspace-controls" aria-label="Transcript filters">
|
|
1295
|
+
<div class="workspace-context"><span class="eyebrow">Evaluation</span><strong>${esc(currentEval)}</strong></div>
|
|
1296
|
+
<label class="control-field control-search"><span>Filter runs</span><input id="run-filter" value="${esc(runFilter)}" placeholder="model, skill, label…" oninput="runFilter=this.value" onkeydown="if(event.key==='Enter'){applyWorkspaceFilters()}"></label>
|
|
1297
|
+
<label class="control-field"><span>Agent</span><select id="agent-filter" onchange="agentFilter=this.value;applyWorkspaceFilters()">${optionList(agents, agentFilter, 'All agents')}</select></label>
|
|
1298
|
+
<label class="control-field"><span>Model</span><select id="model-filter" onchange="modelFilter=this.value;applyWorkspaceFilters()">${optionList(models, modelFilter, 'All models')}</select></label>
|
|
1299
|
+
<label class="control-field"><span>Skill</span><select id="skill-filter" onchange="skillFilter=this.value;applyWorkspaceFilters()">${optionList(skills, skillFilter, 'All skills')}</select></label>
|
|
1300
|
+
<label class="control-field slice-control"><span>Slice by</span><select id="slice-by" onchange="changeSlice(this.value)">
|
|
1301
|
+
<option value="none" ${sliceBy === 'none' ? 'selected' : ''}>No slicing</option>
|
|
1302
|
+
<option value="run" ${sliceBy === 'run' ? 'selected' : ''}>Run number</option>
|
|
1303
|
+
<option value="agent" ${sliceBy === 'agent' ? 'selected' : ''}>Agent</option>
|
|
1304
|
+
<option value="model" ${sliceBy === 'model' ? 'selected' : ''}>Model</option>
|
|
1305
|
+
<option value="skill" ${sliceBy === 'skill' ? 'selected' : ''}>Skill</option>
|
|
1306
|
+
</select></label>
|
|
1307
|
+
<button class="apply-filter-btn" onclick="applyWorkspaceFilters()">Apply</button>
|
|
1308
|
+
<button class="reset-filter-btn" onclick="resetWorkspaceFilters()">Reset</button>
|
|
1309
|
+
</section>`;
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
function applyWorkspaceFilters() {
|
|
1313
|
+
const input = document.getElementById('run-filter');
|
|
1314
|
+
if (input) runFilter = input.value;
|
|
1315
|
+
renderRunSidebar();
|
|
1316
|
+
renderWorkspace();
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1319
|
+
function resetWorkspaceFilters() {
|
|
1320
|
+
runFilter = '';
|
|
1321
|
+
agentFilter = 'all';
|
|
1322
|
+
modelFilter = 'all';
|
|
1323
|
+
skillFilter = 'all';
|
|
1324
|
+
renderRunSidebar();
|
|
1325
|
+
renderWorkspace();
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
function changeSlice(value) {
|
|
1329
|
+
sliceBy = value;
|
|
1330
|
+
currentSelection = 'all';
|
|
1331
|
+
renderRunSidebar();
|
|
1332
|
+
renderWorkspace();
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
function renderExpectationMiniTable(session) {
|
|
1336
|
+
const metric = session.evalMetric;
|
|
1337
|
+
const expectations = metric && Array.isArray(metric.expectations) ? metric.expectations : [];
|
|
1338
|
+
const summary = metric && metric.expectation_summary;
|
|
1339
|
+
if (!expectations.length) {
|
|
1340
|
+
return `<div class="expectations-empty"><span>Eval results</span><strong>Pending or unavailable</strong></div>`;
|
|
1341
|
+
}
|
|
1342
|
+
const passed = summary ? summary.passed : expectations.filter(item => String(item.result || '').toUpperCase() === 'PASS').length;
|
|
1343
|
+
const total = summary ? summary.total : expectations.length;
|
|
1344
|
+
const failed = summary ? summary.failed : total - passed;
|
|
1345
|
+
const statusClass = failed > 0 ? 'has-failures' : 'all-passed';
|
|
1346
|
+
const rows = expectations.map(item => {
|
|
1347
|
+
const result = String(item.result || '—').toUpperCase();
|
|
1348
|
+
const resultClass = result === 'PASS' ? 'pass' : (result === 'FAIL' ? 'fail' : 'unknown');
|
|
1349
|
+
return `<tr title="${esc(item.reason || '')}"><td>${esc(item.expectation || '—')}</td><td>${esc(item.category || 'other')}</td><td><span class="expectation-result ${resultClass}">${esc(result)}</span></td></tr>`;
|
|
1350
|
+
}).join('');
|
|
1351
|
+
return `<div class="expectations-mini ${statusClass}">
|
|
1352
|
+
<div class="expectations-mini-heading"><span>Eval results</span><strong>${passed}/${total} passed</strong></div>
|
|
1353
|
+
<div class="expectations-table-wrap"><table><thead><tr><th>Expectation</th><th>Category</th><th>Result</th></tr></thead><tbody>${rows}</tbody></table></div>
|
|
1354
|
+
</div>`;
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
function renderLaneSession(session) {
|
|
1358
|
+
const i = session.sourceIndex;
|
|
1359
|
+
const d = session.dimensions;
|
|
1360
|
+
const prefix = 'lane-' + i;
|
|
1361
|
+
const errorRate = session.stats.tool_calls > 0 ? Math.round(session.stats.tool_errors / session.stats.tool_calls * 100) : 0;
|
|
1362
|
+
let html = `<article class="lane-session">
|
|
1363
|
+
<header class="lane-session-header">
|
|
1364
|
+
<div class="lane-title-row"><div><span class="run-kicker">Run ${esc(d.run)}</span><h3>${esc(d.agent)} / ${esc(d.model)}</h3></div><button class="download-btn" onclick="downloadJsonl(${i})">JSONL</button></div>
|
|
1365
|
+
<div class="dimension-chips"><span>${esc(d.skill)}</span><span>${formatDuration(session.stats.duration_s)}</span><span>${session.stats.tool_calls} calls</span><span class="${session.stats.tool_errors ? 'chip-error' : ''}">${session.stats.tool_errors} errors${errorRate ? ' · ' + errorRate + '%' : ''}</span></div>
|
|
1366
|
+
${renderExpectationMiniTable(session)}
|
|
1367
|
+
</header>`;
|
|
1368
|
+
if (!session.transcript_available) {
|
|
1369
|
+
html += '<div class="transcript-unavailable">No transcript was retained for this completed eval. Its scored expectations remain included in this comparison.</div>';
|
|
1370
|
+
}
|
|
1371
|
+
if (session.prompt) {
|
|
1372
|
+
html += `<details class="lane-prompt"><summary>Prompt</summary><div class="prompt-text">${esc(session.prompt)}</div></details>`;
|
|
1373
|
+
}
|
|
1374
|
+
html += `<div class="timeline lane-timeline">${renderTimeline(session.timeline, prefix)}</div>`;
|
|
1375
|
+
if (session.result && session.result.result_text) {
|
|
1376
|
+
html += `<details class="lane-result"><summary>Final output</summary><div class="md-rendered">${renderMarkdown(session.result.result_text)}</div></details>`;
|
|
1377
|
+
}
|
|
1378
|
+
return html + '</article>';
|
|
1379
|
+
}
|
|
1380
|
+
|
|
1381
|
+
function renderSlicedTranscripts(sessions) {
|
|
1382
|
+
const main = document.getElementById('main-content');
|
|
1383
|
+
main.classList.add('full-width');
|
|
1384
|
+
let html = renderWorkspaceControls();
|
|
1385
|
+
html += `<div class="timeline-filter-bar"><input type="text" id="search-input" placeholder="Filter visible transcripts…" oninput="applyFilter()"><button class="filter-btn ${filterErrors ? 'active' : ''}" onclick="toggleErrorFilter()">Errors Only</button></div>`;
|
|
1386
|
+
if (!sessions.length) {
|
|
1387
|
+
main.innerHTML = html + '<div class="empty-workspace"><strong>No runs match these filters.</strong><span>Reset the filters or select another run.</span></div>';
|
|
1388
|
+
return;
|
|
1389
|
+
}
|
|
1390
|
+
const dimension = sliceBy === 'none' ? null : sliceBy;
|
|
1391
|
+
const groups = [];
|
|
1392
|
+
if (!dimension) {
|
|
1393
|
+
groups.push({label: 'Selected transcripts', sessions});
|
|
1394
|
+
} else {
|
|
1395
|
+
for (const value of uniqueValues(sessions.map(s => s.dimensions[dimension]))) {
|
|
1396
|
+
groups.push({label: dimension === 'run' ? 'Run ' + value : value, sessions: sessions.filter(s => s.dimensions[dimension] === value)});
|
|
1397
|
+
}
|
|
1398
|
+
}
|
|
1399
|
+
html += `<div class="slice-summary"><span>${sessions.length} transcript${sessions.length === 1 ? '' : 's'}</span><span>${groups.length} ${groups.length === 1 ? 'column' : 'columns'}</span><span>Sliced by ${dimension || 'none'}</span></div>`;
|
|
1400
|
+
const sliceMinWidth = groups.length * 460 + Math.max(0, groups.length - 1) * 12;
|
|
1401
|
+
html += `<div class="slice-grid" style="--lane-count:${groups.length};--slice-min-width:${sliceMinWidth}px">`;
|
|
1402
|
+
for (const group of groups) {
|
|
1403
|
+
html += `<section class="transcript-lane"><div class="transcript-lane-heading"><span>${esc(group.label)}</span><strong>${group.sessions.length}</strong></div>${group.sessions.map(renderLaneSession).join('')}</section>`;
|
|
1404
|
+
}
|
|
1405
|
+
main.innerHTML = html + '</div>';
|
|
1406
|
+
applyFilter();
|
|
1407
|
+
}
|
|
1408
|
+
|
|
1409
|
+
function renderWorkspace() {
|
|
1410
|
+
window.__evalHarnessActive = false;
|
|
1411
|
+
const sessions = filteredSessions();
|
|
1412
|
+
if (sliceBy === 'none' && sessions.length === 1) {
|
|
1413
|
+
currentSession = sessions[0].sourceIndex;
|
|
1414
|
+
renderSession(currentSession);
|
|
1415
|
+
return;
|
|
1416
|
+
}
|
|
1417
|
+
renderSlicedTranscripts(sessions);
|
|
1062
1418
|
}
|
|
1063
1419
|
|
|
1064
1420
|
function downloadJsonl(i) {
|
|
@@ -1076,14 +1432,19 @@ function downloadJsonl(i) {
|
|
|
1076
1432
|
function renderSession(i) {
|
|
1077
1433
|
const s = SESSIONS[i];
|
|
1078
1434
|
const main = document.getElementById('main-content');
|
|
1435
|
+
main.classList.add('full-width');
|
|
1079
1436
|
|
|
1080
1437
|
// Header
|
|
1081
1438
|
const timingBadge = s.timing_mode === 'exact'
|
|
1082
1439
|
? '<span class="timing-badge exact">exact timestamps</span>'
|
|
1083
1440
|
: '<span class="timing-badge estimated">estimated timestamps</span>';
|
|
1084
1441
|
const liveBadge = isLive ? '<span class="live-badge" onclick="toggleLive()" title="Click to pause/resume auto-refresh">LIVE</span>' : '';
|
|
1085
|
-
let html =
|
|
1086
|
-
html +=
|
|
1442
|
+
let html = renderWorkspaceControls();
|
|
1443
|
+
html += `<div class="header"><div class="header-title-row"><h1>${esc(s.label)}</h1>${timingBadge}${liveBadge}<button class="download-btn" onclick="downloadJsonl(${i})" title="Download raw JSONL">Download JSONL</button></div><div class="header-grid">`;
|
|
1444
|
+
html += item('Agent', s.dimensions.agent);
|
|
1445
|
+
html += item('Model', s.dimensions.model || s.meta.model || '\u2014');
|
|
1446
|
+
html += item('Skill', s.dimensions.skill);
|
|
1447
|
+
html += item('Run', s.dimensions.run);
|
|
1087
1448
|
html += item('Session', (s.meta.session_id || '\u2014').slice(0, 12));
|
|
1088
1449
|
if (s.meta.uploaded_by) html += item('Uploaded by', s.meta.uploaded_by);
|
|
1089
1450
|
const env = s.meta.environment || {};
|
|
@@ -1101,6 +1462,10 @@ function renderSession(i) {
|
|
|
1101
1462
|
if (s.stats.loop_groups > 0) html += item('Retry Loops', s.stats.loop_groups);
|
|
1102
1463
|
if (s.stats.error_streak_groups > 0) html += item('Error Streaks', s.stats.error_streak_groups + ' (max ' + s.stats.max_error_streak + ')');
|
|
1103
1464
|
html += '</div>';
|
|
1465
|
+
html += renderExpectationMiniTable(s);
|
|
1466
|
+
if (!s.transcript_available) {
|
|
1467
|
+
html += '<div class="transcript-unavailable">No transcript was retained for this completed eval. Its scored expectations remain included in this comparison.</div>';
|
|
1468
|
+
}
|
|
1104
1469
|
|
|
1105
1470
|
// Tool breakdown
|
|
1106
1471
|
const tbEntries = Object.entries(s.stats.tool_breakdown).sort((a, b) => b[1] - a[1]);
|
|
@@ -1158,7 +1523,7 @@ function renderSession(i) {
|
|
|
1158
1523
|
|
|
1159
1524
|
// Timeline
|
|
1160
1525
|
html += '<div class="timeline" id="timeline">';
|
|
1161
|
-
html += renderTimeline(s.timeline);
|
|
1526
|
+
html += renderTimeline(s.timeline, 'session-' + i);
|
|
1162
1527
|
html += '</div>';
|
|
1163
1528
|
|
|
1164
1529
|
// Result card
|
|
@@ -1273,12 +1638,14 @@ function compareRow(label, displayValues, numericValues, betterIs) {
|
|
|
1273
1638
|
return html;
|
|
1274
1639
|
}
|
|
1275
1640
|
|
|
1276
|
-
function renderTimeline(timeline) {
|
|
1641
|
+
function renderTimeline(timeline, idPrefix) {
|
|
1277
1642
|
let html = '';
|
|
1643
|
+
const prefix = idPrefix ? String(idPrefix).replace(/[^a-zA-Z0-9_-]/g, '-') + '-' : '';
|
|
1278
1644
|
|
|
1279
1645
|
for (let i = 0; i < timeline.length; i++) {
|
|
1280
1646
|
const e = timeline[i];
|
|
1281
|
-
const id = 'entry-' + i;
|
|
1647
|
+
const id = prefix + 'entry-' + i;
|
|
1648
|
+
const detailId = prefix + 'detail-' + i;
|
|
1282
1649
|
|
|
1283
1650
|
if (e.kind === 'user_message') {
|
|
1284
1651
|
const long = e.text.length > 300;
|
|
@@ -1322,7 +1689,7 @@ function renderTimeline(timeline) {
|
|
|
1322
1689
|
if (e.is_loop) entryClasses.push('in-loop');
|
|
1323
1690
|
|
|
1324
1691
|
html += `<div class="${entryClasses.join(' ')}" data-searchtext="${esc(searchContent)}">
|
|
1325
|
-
<div class="tool-row" onclick="toggleDetail('
|
|
1692
|
+
<div class="tool-row" onclick="toggleDetail('${detailId}', this)">
|
|
1326
1693
|
<span class="tool-step">#${e.step}</span>
|
|
1327
1694
|
<span class="tool-elapsed">${formatElapsed(e.elapsed)}</span>
|
|
1328
1695
|
<span class="tool-status ${statusClass}">${statusLabel}</span>
|
|
@@ -1331,10 +1698,10 @@ function renderTimeline(timeline) {
|
|
|
1331
1698
|
${e.parallel ? '<span class="parallel-badge">parallel</span>' : ''}
|
|
1332
1699
|
${e.is_loop ? '<span class="loop-badge">loop \u00d7' + e.loop_count + '</span>' : ''}
|
|
1333
1700
|
</div>
|
|
1334
|
-
<div class="tool-detail" id="
|
|
1701
|
+
<div class="tool-detail" id="${detailId}">
|
|
1335
1702
|
<div class="tool-detail-section">
|
|
1336
1703
|
<div class="label">Input</div>
|
|
1337
|
-
${formatToolInputHtml(e.tool, e.input, i)}
|
|
1704
|
+
${formatToolInputHtml(e.tool, e.input, prefix + i)}
|
|
1338
1705
|
</div>
|
|
1339
1706
|
${e.result_content != null ? `<div class="tool-detail-section">
|
|
1340
1707
|
<div class="label">Result</div>
|