deepline 0.3.234 → 0.3.235
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +7 -1
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +78 -13
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/dist/viewer/viewer.css +2 -1
- package/dist/viewer/viewer.js +64 -9
- package/package.json +1 -1
|
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
|
|
|
200
200
|
// getters keep their established compatibility behavior.
|
|
201
201
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
202
202
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
203
|
-
version: '0.3.
|
|
203
|
+
version: '0.3.235',
|
|
204
204
|
updateSummary:
|
|
205
205
|
'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
|
|
206
206
|
packageCapabilities: {
|
|
@@ -43,6 +43,7 @@ export type PlayRunRecoveryReason =
|
|
|
43
43
|
| 'modal_runner_start_failed_before_execution'
|
|
44
44
|
| 'daytona_sandbox_transition_pending_before_execution'
|
|
45
45
|
| 'daytona_sandbox_acquisition_unavailable_before_execution'
|
|
46
|
+
| 'runtime_scheduler_unavailable_before_execution'
|
|
46
47
|
| 'runtime_initialization_transport_before_execution'
|
|
47
48
|
| 'runtime_traffic_hold_before_execution'
|
|
48
49
|
| 'runtime_traffic_throttled_before_execution';
|
|
@@ -211,6 +212,27 @@ export function recoveryForRuntimeTrafficPolicy(input: {
|
|
|
211
212
|
});
|
|
212
213
|
}
|
|
213
214
|
|
|
215
|
+
/**
|
|
216
|
+
* Scheduler admission must be durable before a provider create. If that
|
|
217
|
+
* control-plane read/write is temporarily unavailable, defer through the
|
|
218
|
+
* recovery ledger instead of turning an unstarted Play into a runner failure.
|
|
219
|
+
*/
|
|
220
|
+
export function recoveryForRuntimeSchedulerAdmissionFailure(
|
|
221
|
+
error: unknown,
|
|
222
|
+
): PlayRunRecoveryError {
|
|
223
|
+
return new PlayRunRecoveryError({
|
|
224
|
+
decision: {
|
|
225
|
+
stage: 'pre_code',
|
|
226
|
+
action: 'defer',
|
|
227
|
+
reason: 'runtime_scheduler_unavailable_before_execution',
|
|
228
|
+
dependency: 'runtime',
|
|
229
|
+
},
|
|
230
|
+
message:
|
|
231
|
+
'Runtime scheduler admission is temporarily unavailable before customer execution began.',
|
|
232
|
+
cause: error,
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
|
|
214
236
|
/**
|
|
215
237
|
* Daytona acknowledged a named sandbox but has not finished creating it.
|
|
216
238
|
* Resume that durable provider operation after a bounded defer; another
|
|
@@ -38,7 +38,10 @@ import {
|
|
|
38
38
|
type OneShotDaytonaSandboxLifecycle,
|
|
39
39
|
validateDaytonaExecutionContext,
|
|
40
40
|
} from './daytona-lifecycle';
|
|
41
|
-
import {
|
|
41
|
+
import {
|
|
42
|
+
isPlayRunRecoveryError,
|
|
43
|
+
recoveryForDaytonaSandboxTransitionPending,
|
|
44
|
+
} from '../../play-run-recovery-policy';
|
|
42
45
|
import { stageDaytonaRunnerPayload } from './daytona-payload-transport';
|
|
43
46
|
import {
|
|
44
47
|
DaytonaRunnerInitializationError,
|
|
@@ -1819,6 +1822,9 @@ export const daytonaPlayRunnerBackend: PlayRunnerBackend = {
|
|
|
1819
1822
|
if (error instanceof DaytonaSandboxAcquisitionUnavailableError) {
|
|
1820
1823
|
throw error;
|
|
1821
1824
|
}
|
|
1825
|
+
if (isPlayRunRecoveryError(error)) {
|
|
1826
|
+
throw error;
|
|
1827
|
+
}
|
|
1822
1828
|
emitDaytonaStage(callbacks, config.context, 'execute:error', {
|
|
1823
1829
|
sandboxId: sandboxCleanup.currentSandbox()?.id ?? null,
|
|
1824
1830
|
runnerAttempt: Math.max(0, Math.floor(config.context.runAttempt ?? 0)),
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts
CHANGED
|
@@ -50,6 +50,7 @@ import { runtimeLimitExceededCause } from '../run-failure';
|
|
|
50
50
|
import {
|
|
51
51
|
recoveryForDaytonaSandboxAcquisitionUnavailable,
|
|
52
52
|
recoveryForRuntimeInitializationTransportFailure,
|
|
53
|
+
recoveryForRuntimeSchedulerAdmissionFailure,
|
|
53
54
|
recoveryForRuntimeTrafficPolicy,
|
|
54
55
|
} from '../play-run-recovery-policy';
|
|
55
56
|
import type { PlayCheckpoint } from '../ctx-types';
|
|
@@ -208,9 +209,10 @@ import {
|
|
|
208
209
|
} from './absurd-worker-required-env';
|
|
209
210
|
import {
|
|
210
211
|
acquirePgNotifyHub,
|
|
212
|
+
isConnectionCategoryError,
|
|
211
213
|
isRuntimeSchedulerCapacityError,
|
|
214
|
+
isRuntimeSchedulerQueryCancellationError,
|
|
212
215
|
isRuntimeSchedulerRetryableTransactionError,
|
|
213
|
-
isConnectionCategoryError,
|
|
214
216
|
parseRuntimeSchedulerCapacityError,
|
|
215
217
|
RuntimeSchedulerCapacityError,
|
|
216
218
|
runtimeSchedulerCapacityRetryAfterMs,
|
|
@@ -3490,22 +3492,85 @@ async function executeAssignedAbsurdAttempt(input: {
|
|
|
3490
3492
|
);
|
|
3491
3493
|
}
|
|
3492
3494
|
const mode = runtimeSandboxCapacityAdmissionMode();
|
|
3493
|
-
const
|
|
3494
|
-
|
|
3495
|
-
|
|
3495
|
+
const reservationInput = {
|
|
3496
|
+
orgId: launch.orgId,
|
|
3497
|
+
runId,
|
|
3498
|
+
provider,
|
|
3499
|
+
mode,
|
|
3500
|
+
attempt: generation,
|
|
3501
|
+
environment: 'production' as const,
|
|
3502
|
+
resumePendingDaytonaOperation: provider === 'daytona',
|
|
3503
|
+
operationKey: `sandbox-operation:${runId}:${generation}:${provider}:${providerAttempt ?? 1}`,
|
|
3504
|
+
};
|
|
3505
|
+
const reserve = () =>
|
|
3506
|
+
deps.runWithClient((client) =>
|
|
3507
|
+
reserveRuntimeSandboxOperation(
|
|
3508
|
+
client,
|
|
3509
|
+
reservationInput,
|
|
3510
|
+
options,
|
|
3511
|
+
),
|
|
3512
|
+
);
|
|
3513
|
+
const deferForSchedulerFailure = (error: unknown): never => {
|
|
3514
|
+
console.warn(
|
|
3515
|
+
'[absurd-runtime.worker] sandbox_capacity_admission_deferred',
|
|
3496
3516
|
{
|
|
3497
|
-
orgId: launch.orgId,
|
|
3498
3517
|
runId,
|
|
3499
3518
|
provider,
|
|
3500
|
-
|
|
3501
|
-
|
|
3502
|
-
|
|
3503
|
-
|
|
3504
|
-
|
|
3519
|
+
providerAttempt: providerAttempt ?? 1,
|
|
3520
|
+
error:
|
|
3521
|
+
error instanceof Error
|
|
3522
|
+
? error.message
|
|
3523
|
+
: String(error),
|
|
3505
3524
|
},
|
|
3506
|
-
|
|
3507
|
-
)
|
|
3508
|
-
|
|
3525
|
+
);
|
|
3526
|
+
throw recoveryForRuntimeSchedulerAdmissionFailure(error);
|
|
3527
|
+
};
|
|
3528
|
+
const isRetryableSchedulerFailure = (error: unknown) =>
|
|
3529
|
+
isRuntimeSchedulerCapacityError(error) ||
|
|
3530
|
+
isConnectionCategoryError(error) ||
|
|
3531
|
+
isRuntimeSchedulerRetryableTransactionError(error) ||
|
|
3532
|
+
isRuntimeSchedulerQueryCancellationError(error);
|
|
3533
|
+
let reserved: Awaited<ReturnType<typeof reserve>> | null =
|
|
3534
|
+
null;
|
|
3535
|
+
try {
|
|
3536
|
+
reserved = await reserve();
|
|
3537
|
+
} catch (error) {
|
|
3538
|
+
if (isRuntimeSchedulerQueryCancellationError(error)) {
|
|
3539
|
+
// A query timeout has an ambiguous outcome. The stable
|
|
3540
|
+
// operation key makes this reservation safe to repeat:
|
|
3541
|
+
// a committed first attempt is read back, and a missing
|
|
3542
|
+
// one is inserted once.
|
|
3543
|
+
console.warn(
|
|
3544
|
+
'[absurd-runtime.worker] sandbox_capacity_admission_retry',
|
|
3545
|
+
{
|
|
3546
|
+
runId,
|
|
3547
|
+
provider,
|
|
3548
|
+
providerAttempt: providerAttempt ?? 1,
|
|
3549
|
+
error:
|
|
3550
|
+
error instanceof Error
|
|
3551
|
+
? error.message
|
|
3552
|
+
: String(error),
|
|
3553
|
+
},
|
|
3554
|
+
);
|
|
3555
|
+
try {
|
|
3556
|
+
reserved = await reserve();
|
|
3557
|
+
} catch (retryError) {
|
|
3558
|
+
if (!isRetryableSchedulerFailure(retryError)) {
|
|
3559
|
+
throw retryError;
|
|
3560
|
+
}
|
|
3561
|
+
deferForSchedulerFailure(retryError);
|
|
3562
|
+
}
|
|
3563
|
+
} else if (isRetryableSchedulerFailure(error)) {
|
|
3564
|
+
deferForSchedulerFailure(error);
|
|
3565
|
+
} else {
|
|
3566
|
+
throw error;
|
|
3567
|
+
}
|
|
3568
|
+
}
|
|
3569
|
+
if (reserved === null) {
|
|
3570
|
+
throw new Error(
|
|
3571
|
+
'Runtime scheduler sandbox capacity reservation returned no result.',
|
|
3572
|
+
);
|
|
3573
|
+
}
|
|
3509
3574
|
if (!reserved.granted) {
|
|
3510
3575
|
throw new RuntimeSandboxCapacityLimitError({
|
|
3511
3576
|
orgId: launch.orgId,
|
package/dist/cli/index.js
CHANGED
|
@@ -3454,7 +3454,7 @@ var SDK_RELEASE = {
|
|
|
3454
3454
|
// getters keep their established compatibility behavior.
|
|
3455
3455
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3456
3456
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3457
|
-
version: "0.3.
|
|
3457
|
+
version: "0.3.235",
|
|
3458
3458
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3459
3459
|
packageCapabilities: {
|
|
3460
3460
|
updatePreferences: 1
|
package/dist/cli/index.mjs
CHANGED
|
@@ -3449,7 +3449,7 @@ var SDK_RELEASE = {
|
|
|
3449
3449
|
// getters keep their established compatibility behavior.
|
|
3450
3450
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3451
3451
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3452
|
-
version: "0.3.
|
|
3452
|
+
version: "0.3.235",
|
|
3453
3453
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3454
3454
|
packageCapabilities: {
|
|
3455
3455
|
updatePreferences: 1
|
package/dist/index.js
CHANGED
|
@@ -1267,7 +1267,7 @@ var SDK_RELEASE = {
|
|
|
1267
1267
|
// getters keep their established compatibility behavior.
|
|
1268
1268
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
1269
1269
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
1270
|
-
version: "0.3.
|
|
1270
|
+
version: "0.3.235",
|
|
1271
1271
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
1272
1272
|
packageCapabilities: {
|
|
1273
1273
|
updatePreferences: 1
|
package/dist/index.mjs
CHANGED
|
@@ -1171,7 +1171,7 @@ var SDK_RELEASE = {
|
|
|
1171
1171
|
// getters keep their established compatibility behavior.
|
|
1172
1172
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
1173
1173
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
1174
|
-
version: "0.3.
|
|
1174
|
+
version: "0.3.235",
|
|
1175
1175
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
1176
1176
|
packageCapabilities: {
|
|
1177
1177
|
updatePreferences: 1
|
package/dist/release.d.mts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.235";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.d.ts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.235";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.js
CHANGED
|
@@ -74,7 +74,7 @@ var SDK_RELEASE = {
|
|
|
74
74
|
// getters keep their established compatibility behavior.
|
|
75
75
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
76
76
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
77
|
-
version: "0.3.
|
|
77
|
+
version: "0.3.235",
|
|
78
78
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
79
79
|
packageCapabilities: {
|
|
80
80
|
updatePreferences: 1
|
package/dist/release.mjs
CHANGED
|
@@ -48,7 +48,7 @@ var SDK_RELEASE = {
|
|
|
48
48
|
// getters keep their established compatibility behavior.
|
|
49
49
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
50
50
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
51
|
-
version: "0.3.
|
|
51
|
+
version: "0.3.235",
|
|
52
52
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
53
53
|
packageCapabilities: {
|
|
54
54
|
updatePreferences: 1
|
package/dist/viewer/viewer.css
CHANGED
|
@@ -737,7 +737,7 @@ body {
|
|
|
737
737
|
top: -24px;
|
|
738
738
|
z-index: 20;
|
|
739
739
|
display: grid;
|
|
740
|
-
grid-template-columns: minmax(180px, 1.2fr) minmax(170px, 1.2fr) repeat(
|
|
740
|
+
grid-template-columns: minmax(180px, 1.2fr) minmax(170px, 1.2fr) repeat(5, minmax(120px, .75fr)) auto auto;
|
|
741
741
|
gap: 8px;
|
|
742
742
|
align-items: end;
|
|
743
743
|
margin: -24px -32px 18px;
|
|
@@ -902,6 +902,7 @@ body {
|
|
|
902
902
|
.expectation-result.pass { color: var(--green); }
|
|
903
903
|
.expectation-result.fail { color: var(--red); }
|
|
904
904
|
.expectation-result.unknown { color: var(--yellow); }
|
|
905
|
+
.grading-detail { margin-top: 3px; color: var(--text-dim); white-space: pre-wrap; }
|
|
905
906
|
.expectation-failure-reason {
|
|
906
907
|
margin-top: 5px;
|
|
907
908
|
color: color-mix(in srgb, var(--red) 80%, var(--text-dim));
|
package/dist/viewer/viewer.js
CHANGED
|
@@ -1371,11 +1371,12 @@ let isLive = false;
|
|
|
1371
1371
|
let liveIntervalId = null;
|
|
1372
1372
|
let currentEval = '';
|
|
1373
1373
|
let currentSelection = 'all';
|
|
1374
|
-
let sliceBy = '
|
|
1374
|
+
let sliceBy = 'none';
|
|
1375
1375
|
let runFilter = '';
|
|
1376
1376
|
let agentFilter = 'all';
|
|
1377
1377
|
let modelFilter = 'all';
|
|
1378
1378
|
let skillFilter = 'all';
|
|
1379
|
+
let treatmentFilter = 'all';
|
|
1379
1380
|
let visibleEvalKeys = [];
|
|
1380
1381
|
let visibleSelectionKeys = [];
|
|
1381
1382
|
|
|
@@ -1444,6 +1445,7 @@ function attachEvalMetadata() {
|
|
|
1444
1445
|
agent: firstPresent([metric && metric.agent], inferAgent(session.label, session.meta)),
|
|
1445
1446
|
model: firstPresent([metric && metric.model_id, metric && metric.requested_model, metric && metric.model], inferModel(session.label, session.meta)),
|
|
1446
1447
|
skill: firstPresent([metric && metric.invoke_skill], inferSkill(session)),
|
|
1448
|
+
treatment: firstPresent([metric && metric.variant_id], 'Unrecorded treatment'),
|
|
1447
1449
|
};
|
|
1448
1450
|
});
|
|
1449
1451
|
}
|
|
@@ -1457,7 +1459,7 @@ function sessionsForCurrentEval() {
|
|
|
1457
1459
|
}
|
|
1458
1460
|
|
|
1459
1461
|
function selectionDimensions() {
|
|
1460
|
-
const dimensions = ['run', 'agent', 'model', 'skill'];
|
|
1462
|
+
const dimensions = ['run', 'agent', 'model', 'skill', 'treatment'];
|
|
1461
1463
|
return sliceBy === 'none' ? dimensions : dimensions.filter(dimension => dimension !== sliceBy);
|
|
1462
1464
|
}
|
|
1463
1465
|
|
|
@@ -1471,6 +1473,7 @@ function selectionLabel(session) {
|
|
|
1471
1473
|
agent: value => value,
|
|
1472
1474
|
model: value => value,
|
|
1473
1475
|
skill: value => value,
|
|
1476
|
+
treatment: value => value,
|
|
1474
1477
|
};
|
|
1475
1478
|
return selectionDimensions().map(dimension => labels[dimension](session.dimensions[dimension])).join(' · ') || 'All transcripts';
|
|
1476
1479
|
}
|
|
@@ -1481,7 +1484,8 @@ function matchesTopFilters(session) {
|
|
|
1481
1484
|
if (agentFilter !== 'all' && d.agent !== agentFilter) return false;
|
|
1482
1485
|
if (modelFilter !== 'all' && d.model !== modelFilter) return false;
|
|
1483
1486
|
if (skillFilter !== 'all' && d.skill !== skillFilter) return false;
|
|
1484
|
-
|
|
1487
|
+
if (treatmentFilter !== 'all' && d.treatment !== treatmentFilter) return false;
|
|
1488
|
+
return !query || [session.label, d.eval, d.run, d.agent, d.model, d.skill, d.treatment].join(' ').toLowerCase().includes(query);
|
|
1485
1489
|
}
|
|
1486
1490
|
|
|
1487
1491
|
function matchingConfigurations() {
|
|
@@ -1525,6 +1529,7 @@ function saveViewerState() {
|
|
|
1525
1529
|
agentFilter: agentFilter,
|
|
1526
1530
|
modelFilter: modelFilter,
|
|
1527
1531
|
skillFilter: skillFilter,
|
|
1532
|
+
treatmentFilter: treatmentFilter,
|
|
1528
1533
|
}));
|
|
1529
1534
|
}
|
|
1530
1535
|
|
|
@@ -1563,14 +1568,15 @@ function init() {
|
|
|
1563
1568
|
currentEval = saved && visibleEvalKeys.includes(saved.currentEval)
|
|
1564
1569
|
? saved.currentEval
|
|
1565
1570
|
: ((SESSIONS[Math.max(0, initialSession)] || {}).dimensions || {}).eval || visibleEvalKeys[0] || '';
|
|
1566
|
-
sliceBy = saved && ['none', 'run', 'agent', 'model', 'skill'].includes(saved.sliceBy)
|
|
1571
|
+
sliceBy = saved && ['none', 'run', 'agent', 'model', 'skill', 'treatment'].includes(saved.sliceBy)
|
|
1567
1572
|
? saved.sliceBy
|
|
1568
|
-
:
|
|
1573
|
+
: 'none';
|
|
1569
1574
|
currentSelection = (saved && (saved.currentSelection || saved.currentRun)) || 'all';
|
|
1570
1575
|
runFilter = (saved && saved.runFilter) || '';
|
|
1571
1576
|
agentFilter = (saved && saved.agentFilter) || 'all';
|
|
1572
1577
|
modelFilter = (saved && saved.modelFilter) || 'all';
|
|
1573
1578
|
skillFilter = (saved && saved.skillFilter) || 'all';
|
|
1579
|
+
treatmentFilter = (saved && saved.treatmentFilter) || 'all';
|
|
1574
1580
|
|
|
1575
1581
|
if (SESSIONS.length > 1) {
|
|
1576
1582
|
document.getElementById('sidebar').classList.remove('hidden');
|
|
@@ -1720,6 +1726,7 @@ function switchEval(index) {
|
|
|
1720
1726
|
agentFilter = 'all';
|
|
1721
1727
|
modelFilter = 'all';
|
|
1722
1728
|
skillFilter = 'all';
|
|
1729
|
+
treatmentFilter = 'all';
|
|
1723
1730
|
renderSidebar();
|
|
1724
1731
|
renderRunSidebar();
|
|
1725
1732
|
renderWorkspace();
|
|
@@ -1744,7 +1751,7 @@ function switchSession(i) {
|
|
|
1744
1751
|
|
|
1745
1752
|
function switchToCompare() {
|
|
1746
1753
|
currentSelection = 'all';
|
|
1747
|
-
sliceBy = '
|
|
1754
|
+
sliceBy = 'none';
|
|
1748
1755
|
renderRunSidebar();
|
|
1749
1756
|
renderWorkspace();
|
|
1750
1757
|
}
|
|
@@ -1760,14 +1767,17 @@ function renderWorkspaceControls() {
|
|
|
1760
1767
|
const agents = uniqueValues(sessions.map(s => s.dimensions.agent));
|
|
1761
1768
|
const models = uniqueValues(sessions.map(s => s.dimensions.model));
|
|
1762
1769
|
const skills = uniqueValues(sessions.map(s => s.dimensions.skill));
|
|
1770
|
+
const treatments = uniqueValues(sessions.map(s => s.dimensions.treatment));
|
|
1763
1771
|
return `<section class="workspace-controls" aria-label="Transcript filters">
|
|
1764
1772
|
<div class="workspace-context"><span class="eyebrow">Evaluation</span><strong>${esc(currentEval)}</strong></div>
|
|
1765
1773
|
<label class="control-field control-search"><span>Filter runs</span><input id="run-filter" value="${esc(runFilter)}" placeholder="model, skill, label…" oninput="runFilter=this.value" onkeydown="if(event.key==='Enter'){applyWorkspaceFilters()}"></label>
|
|
1766
1774
|
<label class="control-field"><span>Agent</span><select id="agent-filter" onchange="agentFilter=this.value;applyWorkspaceFilters()">${optionList(agents, agentFilter, 'All agents')}</select></label>
|
|
1767
1775
|
<label class="control-field"><span>Model</span><select id="model-filter" onchange="modelFilter=this.value;applyWorkspaceFilters()">${optionList(models, modelFilter, 'All models')}</select></label>
|
|
1768
1776
|
<label class="control-field"><span>Skill</span><select id="skill-filter" onchange="skillFilter=this.value;applyWorkspaceFilters()">${optionList(skills, skillFilter, 'All skills')}</select></label>
|
|
1777
|
+
<label class="control-field"><span>Treatment</span><select id="treatment-filter" onchange="treatmentFilter=this.value;applyWorkspaceFilters()">${optionList(treatments, treatmentFilter, 'All treatments')}</select></label>
|
|
1769
1778
|
<label class="control-field slice-control"><span>Slice by</span><select id="slice-by" onchange="changeSlice(this.value)">
|
|
1770
1779
|
<option value="none" ${sliceBy === 'none' ? 'selected' : ''}>No slicing</option>
|
|
1780
|
+
<option value="treatment" ${sliceBy === 'treatment' ? 'selected' : ''}>Treatment</option>
|
|
1771
1781
|
<option value="run" ${sliceBy === 'run' ? 'selected' : ''}>Run number</option>
|
|
1772
1782
|
<option value="agent" ${sliceBy === 'agent' ? 'selected' : ''}>Agent</option>
|
|
1773
1783
|
<option value="model" ${sliceBy === 'model' ? 'selected' : ''}>Model</option>
|
|
@@ -1790,6 +1800,9 @@ function resetWorkspaceFilters() {
|
|
|
1790
1800
|
agentFilter = 'all';
|
|
1791
1801
|
modelFilter = 'all';
|
|
1792
1802
|
skillFilter = 'all';
|
|
1803
|
+
treatmentFilter = 'all';
|
|
1804
|
+
sliceBy = 'none';
|
|
1805
|
+
currentSelection = 'all';
|
|
1793
1806
|
renderRunSidebar();
|
|
1794
1807
|
renderWorkspace();
|
|
1795
1808
|
}
|
|
@@ -1801,12 +1814,43 @@ function changeSlice(value) {
|
|
|
1801
1814
|
renderWorkspace();
|
|
1802
1815
|
}
|
|
1803
1816
|
|
|
1817
|
+
function renderSavedGrading(grading) {
|
|
1818
|
+
if (!grading || typeof grading !== 'object') return '';
|
|
1819
|
+
const outcome = grading.result || {};
|
|
1820
|
+
const graders = Array.isArray(grading.graders) ? grading.graders : [];
|
|
1821
|
+
if (!grading.result && !graders.length) return '';
|
|
1822
|
+
// Quality is an outcome, not a pass/fail check. In particular, zero is a
|
|
1823
|
+
// real Low score; missing evidence and judge errors must remain ungraded.
|
|
1824
|
+
const quality = outcome.score != null
|
|
1825
|
+
? `${outcome.score}/${outcome.max_score ?? 2} (${outcome.grade || 'scored'})`
|
|
1826
|
+
: `Not scored (${outcome.status || 'unknown'})`;
|
|
1827
|
+
const checks = outcome.checks_status || 'unknown';
|
|
1828
|
+
const rows = graders.map(item => {
|
|
1829
|
+
const status = item.status || 'unknown';
|
|
1830
|
+
const result = item.score != null
|
|
1831
|
+
? `${item.score} (${item.grade || status})`
|
|
1832
|
+
: status;
|
|
1833
|
+
const resultClass = item.type === 'deterministic' && status === 'passed'
|
|
1834
|
+
? 'pass' : (status === 'failed' || status === 'error' ? 'fail' : (item.score != null ? '' : 'unknown'));
|
|
1835
|
+
const errors = Array.isArray(item.evidence_errors) ? item.evidence_errors : [];
|
|
1836
|
+
const details = [item.reason, item.error_kind, ...errors.map(error => [error.code, error.detail].filter(Boolean).join(': '))].filter(Boolean);
|
|
1837
|
+
const reason = details.length ? `<div class="${status === 'failed' || status === 'error' ? 'expectation-failure-reason' : 'grading-detail'}">${esc(details.join(' · '))}</div>` : '';
|
|
1838
|
+
return `<tr><td>${esc(item.id || 'Unnamed grader')}${reason}</td><td>${esc(item.type || 'unknown')}</td><td><span class="expectation-result ${resultClass}">${esc(result)}</span></td></tr>`;
|
|
1839
|
+
}).join('');
|
|
1840
|
+
return `<div class="expectations-mini${checks === 'failed' || checks === 'error' ? ' has-failures' : ''}">
|
|
1841
|
+
<div class="expectations-mini-heading"><span>Outcome quality</span><strong>${esc(quality)}</strong></div>
|
|
1842
|
+
<div class="expectations-mini-heading"><span>Required checks: ${esc(checks)}</span><span>Grading: ${esc(outcome.status || 'unknown')}</span></div>
|
|
1843
|
+
${rows ? `<div class="expectations-table-wrap"><table><thead><tr><th>Grader</th><th>Type</th><th>Result</th></tr></thead><tbody>${rows}</tbody></table></div>` : ''}
|
|
1844
|
+
</div>`;
|
|
1845
|
+
}
|
|
1846
|
+
|
|
1804
1847
|
function renderExpectationMiniTable(session) {
|
|
1805
1848
|
const metric = session.evalMetric;
|
|
1849
|
+
const grading = renderSavedGrading(metric && metric.grading);
|
|
1806
1850
|
const expectations = metric && Array.isArray(metric.expectations) ? metric.expectations : [];
|
|
1807
1851
|
const summary = metric && metric.expectation_summary;
|
|
1808
1852
|
if (!expectations.length) {
|
|
1809
|
-
return `<div class="expectations-empty"><span>Eval results</span><strong>Pending or unavailable</strong></div>`;
|
|
1853
|
+
return grading || `<div class="expectations-empty"><span>Eval results</span><strong>Pending or unavailable</strong></div>`;
|
|
1810
1854
|
}
|
|
1811
1855
|
const passed = summary ? summary.passed : expectations.filter(item => String(item.result || '').toUpperCase() === 'PASS').length;
|
|
1812
1856
|
const total = summary ? summary.total : expectations.length;
|
|
@@ -1820,7 +1864,7 @@ function renderExpectationMiniTable(session) {
|
|
|
1820
1864
|
: '';
|
|
1821
1865
|
return `<tr><td>${esc(item.expectation || '—')}${reason}</td><td>${esc(item.category || 'other')}</td><td><span class="expectation-result ${resultClass}">${esc(result)}</span></td></tr>`;
|
|
1822
1866
|
}).join('');
|
|
1823
|
-
return `<div class="expectations-mini ${statusClass}">
|
|
1867
|
+
return grading + `<div class="expectations-mini ${statusClass}">
|
|
1824
1868
|
<div class="expectations-mini-heading"><span>Eval results</span><strong>${passed}/${total} passed</strong></div>
|
|
1825
1869
|
<div class="expectations-table-wrap"><table><thead><tr><th>Expectation</th><th>Category</th><th>Result</th></tr></thead><tbody>${rows}</tbody></table></div>
|
|
1826
1870
|
</div>`;
|
|
@@ -1887,6 +1931,14 @@ function renderFinalArtifacts(session) {
|
|
|
1887
1931
|
</section>`;
|
|
1888
1932
|
}
|
|
1889
1933
|
|
|
1934
|
+
function renderSourceProvenance(metric) {
|
|
1935
|
+
if (!metric.bundle_digest) return '';
|
|
1936
|
+
const fields = { 'Source run': metric.source_run, 'Bundle': metric.bundle_digest, 'Assignment': metric.assignment_id };
|
|
1937
|
+
return '<details class="lane-prompt"><summary>Source provenance</summary><div class="prompt-text">'
|
|
1938
|
+
+ Object.entries(fields).filter(([, value]) => value).map(([key, value]) => esc(key + ': ' + value)).join('<br>')
|
|
1939
|
+
+ '</div></details>';
|
|
1940
|
+
}
|
|
1941
|
+
|
|
1890
1942
|
function renderLaneSession(session) {
|
|
1891
1943
|
const i = session.sourceIndex;
|
|
1892
1944
|
const d = session.dimensions;
|
|
@@ -1900,9 +1952,10 @@ function renderLaneSession(session) {
|
|
|
1900
1952
|
let html = `<article class="lane-session">
|
|
1901
1953
|
<header class="lane-session-header">
|
|
1902
1954
|
<div class="lane-title-row"><div><span class="run-kicker">Run ${esc(d.run)}</span><h3>${esc(d.agent)} / ${esc(d.model)}</h3></div><button class="download-btn" onclick="downloadJsonl(${i})">JSONL</button></div>
|
|
1903
|
-
<div class="dimension-chips"><span>${esc(d.skill)}</span><span>${formatDuration(session.stats.duration_s)}</span><span>${session.stats.tool_calls} calls</span><span class="${session.stats.tool_errors ? 'chip-error' : ''}">${session.stats.tool_errors} errors${errorRate ? ' · ' + errorRate + '%' : ''}</span>${embeddedFailures ? `<span class="chip-error">${embeddedFailures} embedded failures</span>` : ''}${infraAffected ? `<span class="chip-error">infra affected${infraCodes ? ' · ' + esc(infraCodes) : ''}</span>` : ''}</div>
|
|
1955
|
+
<div class="dimension-chips"><span>${esc(d.treatment)}</span><span>${esc(d.skill)}</span><span>${formatDuration(session.stats.duration_s)}</span><span>${session.stats.tool_calls} calls</span><span class="${session.stats.tool_errors ? 'chip-error' : ''}">${session.stats.tool_errors} errors${errorRate ? ' · ' + errorRate + '%' : ''}</span>${embeddedFailures ? `<span class="chip-error">${embeddedFailures} embedded failures</span>` : ''}${infraAffected ? `<span class="chip-error">infra affected${infraCodes ? ' · ' + esc(infraCodes) : ''}</span>` : ''}</div>
|
|
1904
1956
|
${renderExpectationMiniTable(session)}
|
|
1905
1957
|
${renderInfraSignals(metric)}
|
|
1958
|
+
${renderSourceProvenance(metric)}
|
|
1906
1959
|
</header>`;
|
|
1907
1960
|
if (!session.transcript_available) {
|
|
1908
1961
|
html += '<div class="transcript-unavailable">No transcript was retained for this completed eval. Its scored expectations remain included in this comparison.</div>';
|
|
@@ -1985,6 +2038,7 @@ function renderSession(i) {
|
|
|
1985
2038
|
html += item('Agent', s.dimensions.agent);
|
|
1986
2039
|
html += item('Model', s.dimensions.model || s.meta.model || '\u2014');
|
|
1987
2040
|
html += item('Skill', s.dimensions.skill);
|
|
2041
|
+
html += item('Treatment', s.dimensions.treatment);
|
|
1988
2042
|
html += item('Run', s.dimensions.run);
|
|
1989
2043
|
html += item('Session', (s.meta.session_id || '\u2014').slice(0, 12));
|
|
1990
2044
|
if (s.meta.uploaded_by) html += item('Uploaded by', s.meta.uploaded_by);
|
|
@@ -2013,6 +2067,7 @@ function renderSession(i) {
|
|
|
2013
2067
|
if (s.stats.loop_groups > 0) html += item('Retry Loops', s.stats.loop_groups);
|
|
2014
2068
|
if (s.stats.error_streak_groups > 0) html += item('Error Streaks', s.stats.error_streak_groups + ' (max ' + s.stats.max_error_streak + ')');
|
|
2015
2069
|
html += '</div>';
|
|
2070
|
+
html += renderSourceProvenance(metric);
|
|
2016
2071
|
html += renderExpectationMiniTable(s);
|
|
2017
2072
|
html += renderInfraSignals(metric);
|
|
2018
2073
|
if (!s.transcript_available) {
|