deepline 0.2.47 → 0.2.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +13 -0
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +5 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +1 -0
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/viewer/viewer.css +15 -0
- package/dist/viewer/viewer.js +35 -1
- package/package.json +1 -1
|
@@ -160,7 +160,7 @@ export const SDK_RELEASE = {
|
|
|
160
160
|
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
161
161
|
// exposed storage-dependent synchronous access. This deliberate minor
|
|
162
162
|
// release keeps lazy paging semantics independent of row residency.
|
|
163
|
-
version: '0.2.
|
|
163
|
+
version: '0.2.48',
|
|
164
164
|
contracts: {
|
|
165
165
|
api: {
|
|
166
166
|
name: 'sdk-http-api',
|
|
@@ -1102,6 +1102,7 @@ function durableCtxKey(input: {
|
|
|
1102
1102
|
id: string;
|
|
1103
1103
|
semanticKey?: string | null;
|
|
1104
1104
|
staleAfterSeconds?: number | null;
|
|
1105
|
+
cacheEpochMs?: number;
|
|
1105
1106
|
}): string {
|
|
1106
1107
|
if (input.operation === 'tool') {
|
|
1107
1108
|
throw new Error('Tool calls use tool receipt keys.');
|
|
@@ -1113,6 +1114,7 @@ function durableCtxKey(input: {
|
|
|
1113
1114
|
id: input.id,
|
|
1114
1115
|
semanticKey: input.semanticKey,
|
|
1115
1116
|
staleAfterSeconds: input.staleAfterSeconds,
|
|
1117
|
+
cacheEpochMs: input.cacheEpochMs,
|
|
1116
1118
|
});
|
|
1117
1119
|
}
|
|
1118
1120
|
|
|
@@ -1391,6 +1393,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
1391
1393
|
private fixtureProviderPacingBypassLogged = false;
|
|
1392
1394
|
private fixtureProviderPacingEnforcementLogged = false;
|
|
1393
1395
|
private checkpoint: PlayCheckpoint;
|
|
1396
|
+
private readonly durableCallCacheEpochMs: number;
|
|
1394
1397
|
/**
|
|
1395
1398
|
* Durable tool receipts are the replay/cache authority for the execution
|
|
1396
1399
|
* paths the host supports. Keeping the same completed result in this
|
|
@@ -1664,6 +1667,14 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
1664
1667
|
};
|
|
1665
1668
|
this.#options = options;
|
|
1666
1669
|
this.checkpoint = options.checkpoint ?? emptyCheckpoint();
|
|
1670
|
+
const checkpointCacheEpochMs = this.checkpoint.durableCallCacheEpochMs;
|
|
1671
|
+
this.durableCallCacheEpochMs =
|
|
1672
|
+
typeof checkpointCacheEpochMs === 'number' &&
|
|
1673
|
+
Number.isFinite(checkpointCacheEpochMs) &&
|
|
1674
|
+
checkpointCacheEpochMs >= 0
|
|
1675
|
+
? checkpointCacheEpochMs
|
|
1676
|
+
: Date.now();
|
|
1677
|
+
this.checkpoint.durableCallCacheEpochMs = this.durableCallCacheEpochMs;
|
|
1667
1678
|
this.durableMappedToolResultsBackedByReceipts = Boolean(
|
|
1668
1679
|
(options.claimRuntimeStepReceipt || options.claimRuntimeStepReceipts) &&
|
|
1669
1680
|
(options.getRuntimeStepReceipt || options.getRuntimeStepReceipts) &&
|
|
@@ -3104,6 +3115,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3104
3115
|
}),
|
|
3105
3116
|
providerActionVersion,
|
|
3106
3117
|
staleAfterSeconds: input.staleAfterSeconds,
|
|
3118
|
+
cacheEpochMs: this.durableCallCacheEpochMs,
|
|
3107
3119
|
});
|
|
3108
3120
|
}
|
|
3109
3121
|
|
|
@@ -3286,6 +3298,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
|
|
|
3286
3298
|
id,
|
|
3287
3299
|
semanticKey: opts.semanticKey,
|
|
3288
3300
|
staleAfterSeconds: stalePolicy.staleAfterSeconds,
|
|
3301
|
+
cacheEpochMs: this.durableCallCacheEpochMs,
|
|
3289
3302
|
});
|
|
3290
3303
|
return await executeWithDurableRuntimeReceipt<T>({
|
|
3291
3304
|
operation,
|
|
@@ -779,6 +779,11 @@ export interface ContextOptions {
|
|
|
779
779
|
}
|
|
780
780
|
|
|
781
781
|
export interface PlayCheckpoint {
|
|
782
|
+
/**
|
|
783
|
+
* Clock captured when this logical run first creates its context. All
|
|
784
|
+
* staleAfterSeconds receipt buckets use this value across durable resumes.
|
|
785
|
+
*/
|
|
786
|
+
durableCallCacheEpochMs?: number;
|
|
782
787
|
/** Waterfall batches that have completed: key = `${toolName}:${provider}`, value = results array. */
|
|
783
788
|
completedBatches: Record<string, BatchResult[]>;
|
|
784
789
|
/** Tool call batches that have completed: key = toolId, value = sparse row-cache-key -> result map. */
|
|
@@ -42,6 +42,8 @@ export function buildDurableToolCallCacheKey(input: {
|
|
|
42
42
|
providerActionVersion?: string | null;
|
|
43
43
|
cachePolicyVersion?: string | null;
|
|
44
44
|
staleAfterSeconds?: number | null;
|
|
45
|
+
/** Run-stable clock used to choose the stale bucket. */
|
|
46
|
+
cacheEpochMs?: number;
|
|
45
47
|
playLocalScope?: string | null;
|
|
46
48
|
}): string {
|
|
47
49
|
const orgId = input.orgId?.trim() || 'org';
|
|
@@ -70,6 +72,7 @@ export function buildDurableToolCallCacheKey(input: {
|
|
|
70
72
|
input.cachePolicyVersion ?? DURABLE_CALL_CACHE_POLICY_VERSION,
|
|
71
73
|
staleBucket: durableCacheStaleBucket({
|
|
72
74
|
staleAfterSeconds: input.staleAfterSeconds,
|
|
75
|
+
nowMs: input.cacheEpochMs,
|
|
73
76
|
}),
|
|
74
77
|
}),
|
|
75
78
|
);
|
|
@@ -84,6 +87,8 @@ export function buildDurableCtxCallCacheKey(input: {
|
|
|
84
87
|
semanticKey?: string | null;
|
|
85
88
|
cachePolicyVersion?: string | null;
|
|
86
89
|
staleAfterSeconds?: number | null;
|
|
90
|
+
/** Run-stable clock used to choose the stale bucket. */
|
|
91
|
+
cacheEpochMs?: number;
|
|
87
92
|
}): string {
|
|
88
93
|
const orgId = input.orgId?.trim() || 'org';
|
|
89
94
|
const playId = input.playId?.trim() || 'play';
|
|
@@ -104,6 +109,7 @@ export function buildDurableCtxCallCacheKey(input: {
|
|
|
104
109
|
input.cachePolicyVersion ?? DURABLE_CALL_CACHE_POLICY_VERSION,
|
|
105
110
|
staleBucket: durableCacheStaleBucket({
|
|
106
111
|
staleAfterSeconds: input.staleAfterSeconds,
|
|
112
|
+
nowMs: input.cacheEpochMs,
|
|
107
113
|
}),
|
|
108
114
|
}),
|
|
109
115
|
);
|
package/dist/cli/index.js
CHANGED
|
@@ -1044,7 +1044,7 @@ var SDK_RELEASE = {
|
|
|
1044
1044
|
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
1045
1045
|
// exposed storage-dependent synchronous access. This deliberate minor
|
|
1046
1046
|
// release keeps lazy paging semantics independent of row residency.
|
|
1047
|
-
version: "0.2.
|
|
1047
|
+
version: "0.2.48",
|
|
1048
1048
|
contracts: {
|
|
1049
1049
|
api: {
|
|
1050
1050
|
name: "sdk-http-api",
|
package/dist/cli/index.mjs
CHANGED
|
@@ -1030,7 +1030,7 @@ var SDK_RELEASE = {
|
|
|
1030
1030
|
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
1031
1031
|
// exposed storage-dependent synchronous access. This deliberate minor
|
|
1032
1032
|
// release keeps lazy paging semantics independent of row residency.
|
|
1033
|
-
version: "0.2.
|
|
1033
|
+
version: "0.2.48",
|
|
1034
1034
|
contracts: {
|
|
1035
1035
|
api: {
|
|
1036
1036
|
name: "sdk-http-api",
|
package/dist/index.js
CHANGED
|
@@ -763,7 +763,7 @@ var SDK_RELEASE = {
|
|
|
763
763
|
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
764
764
|
// exposed storage-dependent synchronous access. This deliberate minor
|
|
765
765
|
// release keeps lazy paging semantics independent of row residency.
|
|
766
|
-
version: "0.2.
|
|
766
|
+
version: "0.2.48",
|
|
767
767
|
contracts: {
|
|
768
768
|
api: {
|
|
769
769
|
name: "sdk-http-api",
|
package/dist/index.mjs
CHANGED
|
@@ -689,7 +689,7 @@ var SDK_RELEASE = {
|
|
|
689
689
|
// 0.2.0 makes Dataset Handles uniformly async-only after 0.1.320 briefly
|
|
690
690
|
// exposed storage-dependent synchronous access. This deliberate minor
|
|
691
691
|
// release keeps lazy paging semantics independent of row residency.
|
|
692
|
-
version: "0.2.
|
|
692
|
+
version: "0.2.48",
|
|
693
693
|
contracts: {
|
|
694
694
|
api: {
|
|
695
695
|
name: "sdk-http-api",
|
package/dist/viewer/viewer.css
CHANGED
|
@@ -881,6 +881,21 @@ body {
|
|
|
881
881
|
.expectation-result.pass { color: var(--green); }
|
|
882
882
|
.expectation-result.fail { color: var(--red); }
|
|
883
883
|
.expectation-result.unknown { color: var(--yellow); }
|
|
884
|
+
.infra-signals {
|
|
885
|
+
margin-top: 10px;
|
|
886
|
+
border: 1px solid color-mix(in srgb, var(--red) 45%, var(--viewer-border));
|
|
887
|
+
border-radius: var(--viewer-radius-md);
|
|
888
|
+
background: color-mix(in srgb, var(--red) 5%, var(--bg));
|
|
889
|
+
color: var(--text-dim);
|
|
890
|
+
font-size: 10px;
|
|
891
|
+
}
|
|
892
|
+
.infra-signals-heading { padding: 7px 9px; border-bottom: 1px solid var(--viewer-border); color: var(--red); font-weight: 700; letter-spacing: .04em; text-transform: uppercase; }
|
|
893
|
+
.infra-signals ul { margin: 0; padding: 0; list-style: none; }
|
|
894
|
+
.infra-signals li { padding: 7px 9px; }
|
|
895
|
+
.infra-signals li + li { border-top: 1px solid var(--viewer-border); }
|
|
896
|
+
.infra-signals strong { color: var(--text); font-family: var(--viewer-font-mono); }
|
|
897
|
+
.infra-signals span { margin-left: 7px; color: var(--text-dimmer); font-family: var(--viewer-font-mono); }
|
|
898
|
+
.infra-signals p { margin: 4px 0 0; overflow-wrap: anywhere; }
|
|
884
899
|
.lane-prompt, .lane-result { border-bottom: 1px solid var(--viewer-border); }
|
|
885
900
|
.lane-prompt summary, .lane-result summary { cursor: pointer; padding: 8px 13px; color: var(--text-dim); font-size: 11px; font-weight: 650; }
|
|
886
901
|
.lane-prompt .prompt-text, .lane-result .md-rendered { padding: 0 13px 12px; font-size: 12px; }
|
package/dist/viewer/viewer.js
CHANGED
|
@@ -1354,16 +1354,38 @@ function renderExpectationMiniTable(session) {
|
|
|
1354
1354
|
</div>`;
|
|
1355
1355
|
}
|
|
1356
1356
|
|
|
1357
|
+
function normalizedInfraSignals(metric) {
|
|
1358
|
+
if (!metric || !Array.isArray(metric.infra_signals)) return [];
|
|
1359
|
+
return metric.infra_signals.filter(signal => signal && typeof signal === 'object').map(signal => ({
|
|
1360
|
+
code: String(signal.code || 'unknown_infrastructure_error'),
|
|
1361
|
+
source: String(signal.source || 'unknown'),
|
|
1362
|
+
message: String(signal.message || 'No detail was recorded.'),
|
|
1363
|
+
}));
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
function renderInfraSignals(metric) {
|
|
1367
|
+
const signals = normalizedInfraSignals(metric);
|
|
1368
|
+
if (!signals.length) return '';
|
|
1369
|
+
const rows = signals.map(signal => `<li><strong>${esc(signal.code)}</strong><span>${esc(signal.source)}</span><p>${esc(signal.message)}</p></li>`).join('');
|
|
1370
|
+
return `<div class="infra-signals"><div class="infra-signals-heading">Infrastructure signals</div><ul>${rows}</ul></div>`;
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1357
1373
|
function renderLaneSession(session) {
|
|
1358
1374
|
const i = session.sourceIndex;
|
|
1359
1375
|
const d = session.dimensions;
|
|
1360
1376
|
const prefix = 'lane-' + i;
|
|
1361
1377
|
const errorRate = session.stats.tool_calls > 0 ? Math.round(session.stats.tool_errors / session.stats.tool_calls * 100) : 0;
|
|
1378
|
+
const metric = session.evalMetric || {};
|
|
1379
|
+
const infraAffected = metric.sample_validity === 'infra_affected';
|
|
1380
|
+
const infraSignals = normalizedInfraSignals(metric);
|
|
1381
|
+
const infraCodes = [...new Set(infraSignals.map(signal => signal.code))].join(', ');
|
|
1382
|
+
const embeddedFailures = Number(metric.embedded_failure_count || 0);
|
|
1362
1383
|
let html = `<article class="lane-session">
|
|
1363
1384
|
<header class="lane-session-header">
|
|
1364
1385
|
<div class="lane-title-row"><div><span class="run-kicker">Run ${esc(d.run)}</span><h3>${esc(d.agent)} / ${esc(d.model)}</h3></div><button class="download-btn" onclick="downloadJsonl(${i})">JSONL</button></div>
|
|
1365
|
-
<div class="dimension-chips"><span>${esc(d.skill)}</span><span>${formatDuration(session.stats.duration_s)}</span><span>${session.stats.tool_calls} calls</span><span class="${session.stats.tool_errors ? 'chip-error' : ''}">${session.stats.tool_errors} errors${errorRate ? ' · ' + errorRate + '%' : ''}</span
|
|
1386
|
+
<div class="dimension-chips"><span>${esc(d.skill)}</span><span>${formatDuration(session.stats.duration_s)}</span><span>${session.stats.tool_calls} calls</span><span class="${session.stats.tool_errors ? 'chip-error' : ''}">${session.stats.tool_errors} errors${errorRate ? ' · ' + errorRate + '%' : ''}</span>${embeddedFailures ? `<span class="chip-error">${embeddedFailures} embedded failures</span>` : ''}${infraAffected ? `<span class="chip-error">infra affected${infraCodes ? ' · ' + esc(infraCodes) : ''}</span>` : ''}</div>
|
|
1366
1387
|
${renderExpectationMiniTable(session)}
|
|
1388
|
+
${renderInfraSignals(metric)}
|
|
1367
1389
|
</header>`;
|
|
1368
1390
|
if (!session.transcript_available) {
|
|
1369
1391
|
html += '<div class="transcript-unavailable">No transcript was retained for this completed eval. Its scored expectations remain included in this comparison.</div>';
|
|
@@ -1431,6 +1453,7 @@ function downloadJsonl(i) {
|
|
|
1431
1453
|
|
|
1432
1454
|
function renderSession(i) {
|
|
1433
1455
|
const s = SESSIONS[i];
|
|
1456
|
+
const metric = s.evalMetric || {};
|
|
1434
1457
|
const main = document.getElementById('main-content');
|
|
1435
1458
|
main.classList.add('full-width');
|
|
1436
1459
|
|
|
@@ -1458,11 +1481,22 @@ function renderSession(i) {
|
|
|
1458
1481
|
html += item('Turns', s.stats.num_turns || '\u2014');
|
|
1459
1482
|
html += item('Tool Calls', s.stats.tool_calls);
|
|
1460
1483
|
html += item('Errors', s.stats.tool_errors + (s.stats.tool_calls > 0 ? ' (' + Math.round(s.stats.tool_errors / s.stats.tool_calls * 100) + '%)' : ''));
|
|
1484
|
+
if (Number(metric.embedded_failure_count || 0) > 0) {
|
|
1485
|
+
html += item('Embedded Failures', Number(metric.embedded_failure_count));
|
|
1486
|
+
}
|
|
1487
|
+
if (metric.sample_validity === 'infra_affected') {
|
|
1488
|
+
html += item('Sample Validity', 'infra affected');
|
|
1489
|
+
}
|
|
1490
|
+
const infraSignals = normalizedInfraSignals(metric);
|
|
1491
|
+
if (infraSignals.length) {
|
|
1492
|
+
html += item('Infra Signal Codes', [...new Set(infraSignals.map(signal => signal.code))].join(', '));
|
|
1493
|
+
}
|
|
1461
1494
|
if (s.stats.cost_usd != null) html += item('Cost', '$' + s.stats.cost_usd.toFixed(2));
|
|
1462
1495
|
if (s.stats.loop_groups > 0) html += item('Retry Loops', s.stats.loop_groups);
|
|
1463
1496
|
if (s.stats.error_streak_groups > 0) html += item('Error Streaks', s.stats.error_streak_groups + ' (max ' + s.stats.max_error_streak + ')');
|
|
1464
1497
|
html += '</div>';
|
|
1465
1498
|
html += renderExpectationMiniTable(s);
|
|
1499
|
+
html += renderInfraSignals(metric);
|
|
1466
1500
|
if (!s.transcript_available) {
|
|
1467
1501
|
html += '<div class="transcript-unavailable">No transcript was retained for this completed eval. Its scored expectations remain included in this comparison.</div>';
|
|
1468
1502
|
}
|