@oneharness/sdk 0.6.8 → 0.6.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/generated/contracts.d.ts +41 -1
- package/dist/generated/history-line.d.ts +19 -13
- package/dist/generated/history-records.d.ts +18 -12
- package/dist/generated/history-stream-envelope.d.ts +19 -13
- package/dist/generated/history.d.ts +17 -11
- package/dist/generated/run-stream-envelope.d.ts +41 -1
- package/dist/generated/schemas.json +454 -27
- package/dist/generated/zod.d.ts +2 -1
- package/dist/generated/zod.js +186 -37
- package/package.json +2 -2
|
@@ -25,7 +25,39 @@ export type OutputFormat = "text" | "json" | "stream-json";
|
|
|
25
25
|
/**
|
|
26
26
|
* The outcome of attempting to run one harness.
|
|
27
27
|
*/
|
|
28
|
-
export type Status = "ok" | "nonzero" | "timeout" | "spawn-error" | "skipped" | "planned";
|
|
28
|
+
export type Status = "ok" | "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped" | "planned";
|
|
29
|
+
/**
|
|
30
|
+
* Measured execution telemetry for one harness run: when the invocation ran,
|
|
31
|
+
* and — when the harness's transcript supports it — how its wall clock split
|
|
32
|
+
* between provider latency and tool work.
|
|
33
|
+
*
|
|
34
|
+
* Carried from the runner/parser to the history writer *and*, since report
|
|
35
|
+
* schema `0.5`, serialized on [`RunResult::telemetry`]. Exposing it there is
|
|
36
|
+
* what lets a consumer read the numbers off the run it just made instead of
|
|
37
|
+
* re-opening the history file the same run wrote.
|
|
38
|
+
*
|
|
39
|
+
* Internally tagged by `source`, so the variant is a value a consumer switches
|
|
40
|
+
* on rather than a shape it has to sniff. Every variant states only what was
|
|
41
|
+
* actually measured — there is no variant meaning "no telemetry"; that is a
|
|
42
|
+
* `null` field.
|
|
43
|
+
*/
|
|
44
|
+
export type ExecutionTelemetry = {
|
|
45
|
+
finished_at: string | null;
|
|
46
|
+
model_ms: number | null;
|
|
47
|
+
source: "provider_measured";
|
|
48
|
+
started_at: string;
|
|
49
|
+
time_to_first_token_ms: number | null;
|
|
50
|
+
tool_ms: number | null;
|
|
51
|
+
[k: string]: unknown;
|
|
52
|
+
} | {
|
|
53
|
+
source: "partial_invocation";
|
|
54
|
+
started_at: string;
|
|
55
|
+
[k: string]: unknown;
|
|
56
|
+
} | {
|
|
57
|
+
source: "stdout_observed";
|
|
58
|
+
tool_ms: number;
|
|
59
|
+
[k: string]: unknown;
|
|
60
|
+
};
|
|
29
61
|
/**
|
|
30
62
|
* The top-level `run` report written to stdout.
|
|
31
63
|
*/
|
|
@@ -321,6 +353,14 @@ export interface RunResult {
|
|
|
321
353
|
* see what the harness produced.
|
|
322
354
|
*/
|
|
323
355
|
structured: unknown;
|
|
356
|
+
/**
|
|
357
|
+
* Measured execution telemetry for this run: the invocation bounds and, when
|
|
358
|
+
* the harness's transcript carried a trace to read them from, the
|
|
359
|
+
* model/tool split. `null` when nothing was measured — never estimated.
|
|
360
|
+
* Added in report schema `0.5`; before that a consumer had to re-read the
|
|
361
|
+
* history file for numbers the run itself already had.
|
|
362
|
+
*/
|
|
363
|
+
telemetry: ExecutionTelemetry | null;
|
|
324
364
|
/**
|
|
325
365
|
* Best-effort final assistant text; `null` when extraction is impossible.
|
|
326
366
|
*/
|
|
@@ -6,7 +6,7 @@ export type HistoryLine = ({
|
|
|
6
6
|
harness: string;
|
|
7
7
|
harness_id?: string | null | undefined;
|
|
8
8
|
run_id: string;
|
|
9
|
-
schema_version: "1.2" | "1.3";
|
|
9
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
10
10
|
type: "event";
|
|
11
11
|
variant?: string | null | undefined;
|
|
12
12
|
[k: string]: unknown;
|
|
@@ -27,15 +27,21 @@ export type HistoryLine = ({
|
|
|
27
27
|
[k: string]: unknown;
|
|
28
28
|
} | ({
|
|
29
29
|
error: string;
|
|
30
|
-
schema_version?: "1.3" | undefined;
|
|
30
|
+
schema_version?: "1.3" | "1.4" | undefined;
|
|
31
31
|
[k: string]: unknown;
|
|
32
32
|
} & ({
|
|
33
|
-
status?: "nonzero" | "timeout" | "spawn-error" | "skipped" | undefined;
|
|
33
|
+
status?: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped" | undefined;
|
|
34
34
|
[k: string]: unknown;
|
|
35
35
|
} | {
|
|
36
36
|
failure_kind: "tool_deferred";
|
|
37
37
|
[k: string]: unknown;
|
|
38
38
|
}))) & ({
|
|
39
|
+
status?: "ok" | "nonzero" | "timeout" | "spawn-error" | "skipped" | "planned" | undefined;
|
|
40
|
+
[k: string]: unknown;
|
|
41
|
+
} | {
|
|
42
|
+
schema_version?: "1.4" | undefined;
|
|
43
|
+
[k: string]: unknown;
|
|
44
|
+
}) & ({
|
|
39
45
|
duration_ms: number;
|
|
40
46
|
/**
|
|
41
47
|
* Normalized failure text for a run that did not succeed (see
|
|
@@ -56,7 +62,7 @@ export type HistoryLine = ({
|
|
|
56
62
|
permission_mode: PermissionMode;
|
|
57
63
|
project: string;
|
|
58
64
|
prompt: string;
|
|
59
|
-
schema_version: "1.0" | "1.1" | "1.2" | "1.3";
|
|
65
|
+
schema_version: "1.0" | "1.1" | "1.2" | "1.3" | "1.4";
|
|
60
66
|
session: string;
|
|
61
67
|
session_id: string | null;
|
|
62
68
|
started_at: string;
|
|
@@ -91,11 +97,11 @@ export type HistoryLine = ({
|
|
|
91
97
|
permission_mode: PermissionMode;
|
|
92
98
|
project: string;
|
|
93
99
|
prompt: string;
|
|
94
|
-
schema_version: "1.0" | "1.1" | "1.2" | "1.3";
|
|
100
|
+
schema_version: "1.0" | "1.1" | "1.2" | "1.3" | "1.4";
|
|
95
101
|
session: string;
|
|
96
102
|
session_id: string | null;
|
|
97
103
|
started_at: string;
|
|
98
|
-
status: "timeout" | "spawn-error" | "skipped" | "planned";
|
|
104
|
+
status: "timeout" | "cancelled" | "spawn-error" | "skipped" | "planned";
|
|
99
105
|
text: string | null;
|
|
100
106
|
text_source: string | null;
|
|
101
107
|
time_to_first_token_ms?: number | null | undefined;
|
|
@@ -126,7 +132,7 @@ export type HistoryLine = ({
|
|
|
126
132
|
permission_mode: PermissionMode;
|
|
127
133
|
project: string;
|
|
128
134
|
prompt: string;
|
|
129
|
-
schema_version: "1.2" | "1.3";
|
|
135
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
130
136
|
session: string;
|
|
131
137
|
session_id: string | null;
|
|
132
138
|
started_at?: never | undefined;
|
|
@@ -161,11 +167,11 @@ export type HistoryLine = ({
|
|
|
161
167
|
permission_mode: PermissionMode;
|
|
162
168
|
project: string;
|
|
163
169
|
prompt: string;
|
|
164
|
-
schema_version: "1.0" | "1.1" | "1.2" | "1.3";
|
|
170
|
+
schema_version: "1.0" | "1.1" | "1.2" | "1.3" | "1.4";
|
|
165
171
|
session: string;
|
|
166
172
|
session_id: string | null;
|
|
167
173
|
started_at?: never | undefined;
|
|
168
|
-
status: "nonzero" | "timeout" | "spawn-error" | "skipped";
|
|
174
|
+
status: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped";
|
|
169
175
|
text: string | null;
|
|
170
176
|
text_source: string | null;
|
|
171
177
|
time_to_first_token_ms?: never | undefined;
|
|
@@ -196,11 +202,11 @@ export type HistoryLine = ({
|
|
|
196
202
|
permission_mode: PermissionMode;
|
|
197
203
|
project: string;
|
|
198
204
|
prompt: string;
|
|
199
|
-
schema_version: "1.3";
|
|
205
|
+
schema_version: "1.3" | "1.4";
|
|
200
206
|
session: string;
|
|
201
207
|
session_id: string | null;
|
|
202
208
|
started_at: string;
|
|
203
|
-
status: "nonzero" | "timeout" | "spawn-error" | "skipped";
|
|
209
|
+
status: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped";
|
|
204
210
|
text: string | null;
|
|
205
211
|
text_source: string | null;
|
|
206
212
|
time_to_first_token_ms?: number | null | undefined;
|
|
@@ -231,7 +237,7 @@ export type HistoryLine = ({
|
|
|
231
237
|
permission_mode: PermissionMode;
|
|
232
238
|
project: string;
|
|
233
239
|
prompt: string;
|
|
234
|
-
schema_version: "1.0" | "1.1" | "1.2" | "1.3";
|
|
240
|
+
schema_version: "1.0" | "1.1" | "1.2" | "1.3" | "1.4";
|
|
235
241
|
session: string;
|
|
236
242
|
session_id: string | null;
|
|
237
243
|
started_at?: never | undefined;
|
|
@@ -272,7 +278,7 @@ export type PermissionMode = "read-only" | "plan" | "default" | "edit" | "auto"
|
|
|
272
278
|
/**
|
|
273
279
|
* The outcome of attempting to run one harness.
|
|
274
280
|
*/
|
|
275
|
-
export type Status = "ok" | "nonzero" | "timeout" | "spawn-error" | "skipped" | "planned";
|
|
281
|
+
export type Status = "ok" | "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped" | "planned";
|
|
276
282
|
/**
|
|
277
283
|
* One normalized action a harness took, harness-agnostic so a single consumer
|
|
278
284
|
* assertion works across harnesses. Every field is always serialized (null when
|
|
@@ -93,7 +93,7 @@ export type HistoryRecord = {
|
|
|
93
93
|
* run's single prompt).
|
|
94
94
|
*/
|
|
95
95
|
prompt: string;
|
|
96
|
-
schema_version: "1.2" | "1.3";
|
|
96
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
97
97
|
/**
|
|
98
98
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
99
99
|
*/
|
|
@@ -323,7 +323,7 @@ export type HistoryRecord = {
|
|
|
323
323
|
* run's single prompt).
|
|
324
324
|
*/
|
|
325
325
|
prompt: string;
|
|
326
|
-
schema_version: "1.2" | "1.3";
|
|
326
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
327
327
|
/**
|
|
328
328
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
329
329
|
*/
|
|
@@ -415,7 +415,7 @@ export type HistoryRecord = {
|
|
|
415
415
|
* run's single prompt).
|
|
416
416
|
*/
|
|
417
417
|
prompt: string;
|
|
418
|
-
schema_version: "1.2" | "1.3";
|
|
418
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
419
419
|
/**
|
|
420
420
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
421
421
|
*/
|
|
@@ -507,7 +507,7 @@ export type HistoryRecord = {
|
|
|
507
507
|
* run's single prompt).
|
|
508
508
|
*/
|
|
509
509
|
prompt: string;
|
|
510
|
-
schema_version: "1.2" | "1.3";
|
|
510
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
511
511
|
/**
|
|
512
512
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
513
513
|
*/
|
|
@@ -517,7 +517,7 @@ export type HistoryRecord = {
|
|
|
517
517
|
*/
|
|
518
518
|
session_id: string | null;
|
|
519
519
|
started_at?: never | undefined;
|
|
520
|
-
status: "nonzero" | "timeout" | "spawn-error" | "skipped";
|
|
520
|
+
status: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped";
|
|
521
521
|
/**
|
|
522
522
|
* Best-effort final assistant text; `null` when extraction was impossible.
|
|
523
523
|
*/
|
|
@@ -599,7 +599,7 @@ export type HistoryRecord = {
|
|
|
599
599
|
* run's single prompt).
|
|
600
600
|
*/
|
|
601
601
|
prompt: string;
|
|
602
|
-
schema_version: "1.3";
|
|
602
|
+
schema_version: "1.3" | "1.4";
|
|
603
603
|
/**
|
|
604
604
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
605
605
|
*/
|
|
@@ -609,7 +609,7 @@ export type HistoryRecord = {
|
|
|
609
609
|
*/
|
|
610
610
|
session_id: string | null;
|
|
611
611
|
started_at: string;
|
|
612
|
-
status: "nonzero" | "timeout" | "spawn-error" | "skipped";
|
|
612
|
+
status: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped";
|
|
613
613
|
/**
|
|
614
614
|
* Best-effort final assistant text; `null` when extraction was impossible.
|
|
615
615
|
*/
|
|
@@ -704,7 +704,7 @@ export type HistoryRecord = {
|
|
|
704
704
|
*/
|
|
705
705
|
session_id: string | null;
|
|
706
706
|
started_at?: never | undefined;
|
|
707
|
-
status: "nonzero" | "timeout" | "spawn-error" | "skipped";
|
|
707
|
+
status: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped";
|
|
708
708
|
/**
|
|
709
709
|
* Best-effort final assistant text; `null` when extraction was impossible.
|
|
710
710
|
*/
|
|
@@ -954,21 +954,27 @@ export type FailureKind = "auth" | "rate_limit" | "model_not_found" | "quota" |
|
|
|
954
954
|
/**
|
|
955
955
|
* The outcome of attempting to run one harness.
|
|
956
956
|
*/
|
|
957
|
-
export type Status = "ok" | "nonzero" | "timeout" | "spawn-error" | "skipped" | "planned";
|
|
957
|
+
export type Status = "ok" | "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped" | "planned";
|
|
958
958
|
export type HistoryRecords = (({
|
|
959
959
|
error?: null | undefined;
|
|
960
960
|
[k: string]: unknown;
|
|
961
961
|
} | ({
|
|
962
962
|
error: string;
|
|
963
|
-
schema_version?: "1.3" | undefined;
|
|
963
|
+
schema_version?: "1.3" | "1.4" | undefined;
|
|
964
964
|
[k: string]: unknown;
|
|
965
965
|
} & ({
|
|
966
|
-
status?: "nonzero" | "timeout" | "spawn-error" | "skipped" | undefined;
|
|
966
|
+
status?: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped" | undefined;
|
|
967
967
|
[k: string]: unknown;
|
|
968
968
|
} | {
|
|
969
969
|
failure_kind: "tool_deferred";
|
|
970
970
|
[k: string]: unknown;
|
|
971
|
-
}))) &
|
|
971
|
+
}))) & ({
|
|
972
|
+
status?: "ok" | "nonzero" | "timeout" | "spawn-error" | "skipped" | "planned" | undefined;
|
|
973
|
+
[k: string]: unknown;
|
|
974
|
+
} | {
|
|
975
|
+
schema_version?: "1.4" | undefined;
|
|
976
|
+
[k: string]: unknown;
|
|
977
|
+
}) & HistoryRecord)[];
|
|
972
978
|
/**
|
|
973
979
|
* One normalized action a harness took, harness-agnostic so a single consumer
|
|
974
980
|
* assertion works across harnesses. Every field is always serialized (null when
|
|
@@ -14,15 +14,21 @@ export type HistoryStreamEnvelope = {
|
|
|
14
14
|
[k: string]: unknown;
|
|
15
15
|
} | ({
|
|
16
16
|
error: string;
|
|
17
|
-
schema_version?: "1.3" | undefined;
|
|
17
|
+
schema_version?: "1.3" | "1.4" | undefined;
|
|
18
18
|
[k: string]: unknown;
|
|
19
19
|
} & ({
|
|
20
|
-
status?: "nonzero" | "timeout" | "spawn-error" | "skipped" | undefined;
|
|
20
|
+
status?: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped" | undefined;
|
|
21
21
|
[k: string]: unknown;
|
|
22
22
|
} | {
|
|
23
23
|
failure_kind: "tool_deferred";
|
|
24
24
|
[k: string]: unknown;
|
|
25
|
-
}))) &
|
|
25
|
+
}))) & ({
|
|
26
|
+
status?: "ok" | "nonzero" | "timeout" | "spawn-error" | "skipped" | "planned" | undefined;
|
|
27
|
+
[k: string]: unknown;
|
|
28
|
+
} | {
|
|
29
|
+
schema_version?: "1.4" | undefined;
|
|
30
|
+
[k: string]: unknown;
|
|
31
|
+
}) & HistoryRecord;
|
|
26
32
|
type: "record";
|
|
27
33
|
[k: string]: unknown;
|
|
28
34
|
} | {
|
|
@@ -125,7 +131,7 @@ export type HistoryRecord = {
|
|
|
125
131
|
* run's single prompt).
|
|
126
132
|
*/
|
|
127
133
|
prompt: string;
|
|
128
|
-
schema_version: "1.2" | "1.3";
|
|
134
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
129
135
|
/**
|
|
130
136
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
131
137
|
*/
|
|
@@ -355,7 +361,7 @@ export type HistoryRecord = {
|
|
|
355
361
|
* run's single prompt).
|
|
356
362
|
*/
|
|
357
363
|
prompt: string;
|
|
358
|
-
schema_version: "1.2" | "1.3";
|
|
364
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
359
365
|
/**
|
|
360
366
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
361
367
|
*/
|
|
@@ -447,7 +453,7 @@ export type HistoryRecord = {
|
|
|
447
453
|
* run's single prompt).
|
|
448
454
|
*/
|
|
449
455
|
prompt: string;
|
|
450
|
-
schema_version: "1.2" | "1.3";
|
|
456
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
451
457
|
/**
|
|
452
458
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
453
459
|
*/
|
|
@@ -539,7 +545,7 @@ export type HistoryRecord = {
|
|
|
539
545
|
* run's single prompt).
|
|
540
546
|
*/
|
|
541
547
|
prompt: string;
|
|
542
|
-
schema_version: "1.2" | "1.3";
|
|
548
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
543
549
|
/**
|
|
544
550
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
545
551
|
*/
|
|
@@ -549,7 +555,7 @@ export type HistoryRecord = {
|
|
|
549
555
|
*/
|
|
550
556
|
session_id: string | null;
|
|
551
557
|
started_at?: never | undefined;
|
|
552
|
-
status: "nonzero" | "timeout" | "spawn-error" | "skipped";
|
|
558
|
+
status: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped";
|
|
553
559
|
/**
|
|
554
560
|
* Best-effort final assistant text; `null` when extraction was impossible.
|
|
555
561
|
*/
|
|
@@ -631,7 +637,7 @@ export type HistoryRecord = {
|
|
|
631
637
|
* run's single prompt).
|
|
632
638
|
*/
|
|
633
639
|
prompt: string;
|
|
634
|
-
schema_version: "1.3";
|
|
640
|
+
schema_version: "1.3" | "1.4";
|
|
635
641
|
/**
|
|
636
642
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
637
643
|
*/
|
|
@@ -641,7 +647,7 @@ export type HistoryRecord = {
|
|
|
641
647
|
*/
|
|
642
648
|
session_id: string | null;
|
|
643
649
|
started_at: string;
|
|
644
|
-
status: "nonzero" | "timeout" | "spawn-error" | "skipped";
|
|
650
|
+
status: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped";
|
|
645
651
|
/**
|
|
646
652
|
* Best-effort final assistant text; `null` when extraction was impossible.
|
|
647
653
|
*/
|
|
@@ -736,7 +742,7 @@ export type HistoryRecord = {
|
|
|
736
742
|
*/
|
|
737
743
|
session_id: string | null;
|
|
738
744
|
started_at?: never | undefined;
|
|
739
|
-
status: "nonzero" | "timeout" | "spawn-error" | "skipped";
|
|
745
|
+
status: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped";
|
|
740
746
|
/**
|
|
741
747
|
* Best-effort final assistant text; `null` when extraction was impossible.
|
|
742
748
|
*/
|
|
@@ -986,13 +992,13 @@ export type FailureKind = "auth" | "rate_limit" | "model_not_found" | "quota" |
|
|
|
986
992
|
/**
|
|
987
993
|
* The outcome of attempting to run one harness.
|
|
988
994
|
*/
|
|
989
|
-
export type Status = "ok" | "nonzero" | "timeout" | "spawn-error" | "skipped" | "planned";
|
|
995
|
+
export type Status = "ok" | "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped" | "planned";
|
|
990
996
|
export type HistoryEventLine = {
|
|
991
997
|
event: ActionEvent;
|
|
992
998
|
harness: string;
|
|
993
999
|
harness_id?: string | null | undefined;
|
|
994
1000
|
run_id: string;
|
|
995
|
-
schema_version: "1.2" | "1.3";
|
|
1001
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
996
1002
|
variant?: string | null | undefined;
|
|
997
1003
|
[k: string]: unknown;
|
|
998
1004
|
} | {
|
|
@@ -8,15 +8,21 @@ export type HistoryRecord = ({
|
|
|
8
8
|
[k: string]: unknown;
|
|
9
9
|
} | ({
|
|
10
10
|
error: string;
|
|
11
|
-
schema_version?: "1.3" | undefined;
|
|
11
|
+
schema_version?: "1.3" | "1.4" | undefined;
|
|
12
12
|
[k: string]: unknown;
|
|
13
13
|
} & ({
|
|
14
|
-
status?: "nonzero" | "timeout" | "spawn-error" | "skipped" | undefined;
|
|
14
|
+
status?: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped" | undefined;
|
|
15
15
|
[k: string]: unknown;
|
|
16
16
|
} | {
|
|
17
17
|
failure_kind: "tool_deferred";
|
|
18
18
|
[k: string]: unknown;
|
|
19
19
|
}))) & ({
|
|
20
|
+
status?: "ok" | "nonzero" | "timeout" | "spawn-error" | "skipped" | "planned" | undefined;
|
|
21
|
+
[k: string]: unknown;
|
|
22
|
+
} | {
|
|
23
|
+
schema_version?: "1.4" | undefined;
|
|
24
|
+
[k: string]: unknown;
|
|
25
|
+
}) & ({
|
|
20
26
|
duration_ms: number;
|
|
21
27
|
/**
|
|
22
28
|
* Best-effort normalized failure text for a run that did not succeed: the
|
|
@@ -111,7 +117,7 @@ export type HistoryRecord = ({
|
|
|
111
117
|
* run's single prompt).
|
|
112
118
|
*/
|
|
113
119
|
prompt: string;
|
|
114
|
-
schema_version: "1.2" | "1.3";
|
|
120
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
115
121
|
/**
|
|
116
122
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
117
123
|
*/
|
|
@@ -341,7 +347,7 @@ export type HistoryRecord = ({
|
|
|
341
347
|
* run's single prompt).
|
|
342
348
|
*/
|
|
343
349
|
prompt: string;
|
|
344
|
-
schema_version: "1.2" | "1.3";
|
|
350
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
345
351
|
/**
|
|
346
352
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
347
353
|
*/
|
|
@@ -433,7 +439,7 @@ export type HistoryRecord = ({
|
|
|
433
439
|
* run's single prompt).
|
|
434
440
|
*/
|
|
435
441
|
prompt: string;
|
|
436
|
-
schema_version: "1.2" | "1.3";
|
|
442
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
437
443
|
/**
|
|
438
444
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
439
445
|
*/
|
|
@@ -525,7 +531,7 @@ export type HistoryRecord = ({
|
|
|
525
531
|
* run's single prompt).
|
|
526
532
|
*/
|
|
527
533
|
prompt: string;
|
|
528
|
-
schema_version: "1.2" | "1.3";
|
|
534
|
+
schema_version: "1.2" | "1.3" | "1.4";
|
|
529
535
|
/**
|
|
530
536
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
531
537
|
*/
|
|
@@ -535,7 +541,7 @@ export type HistoryRecord = ({
|
|
|
535
541
|
*/
|
|
536
542
|
session_id: string | null;
|
|
537
543
|
started_at?: never | undefined;
|
|
538
|
-
status: "nonzero" | "timeout" | "spawn-error" | "skipped";
|
|
544
|
+
status: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped";
|
|
539
545
|
/**
|
|
540
546
|
* Best-effort final assistant text; `null` when extraction was impossible.
|
|
541
547
|
*/
|
|
@@ -617,7 +623,7 @@ export type HistoryRecord = ({
|
|
|
617
623
|
* run's single prompt).
|
|
618
624
|
*/
|
|
619
625
|
prompt: string;
|
|
620
|
-
schema_version: "1.3";
|
|
626
|
+
schema_version: "1.3" | "1.4";
|
|
621
627
|
/**
|
|
622
628
|
* The oneharness session id this run belongs to (the history file's stem).
|
|
623
629
|
*/
|
|
@@ -627,7 +633,7 @@ export type HistoryRecord = ({
|
|
|
627
633
|
*/
|
|
628
634
|
session_id: string | null;
|
|
629
635
|
started_at: string;
|
|
630
|
-
status: "nonzero" | "timeout" | "spawn-error" | "skipped";
|
|
636
|
+
status: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped";
|
|
631
637
|
/**
|
|
632
638
|
* Best-effort final assistant text; `null` when extraction was impossible.
|
|
633
639
|
*/
|
|
@@ -722,7 +728,7 @@ export type HistoryRecord = ({
|
|
|
722
728
|
*/
|
|
723
729
|
session_id: string | null;
|
|
724
730
|
started_at?: never | undefined;
|
|
725
|
-
status: "nonzero" | "timeout" | "spawn-error" | "skipped";
|
|
731
|
+
status: "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped";
|
|
726
732
|
/**
|
|
727
733
|
* Best-effort final assistant text; `null` when extraction was impossible.
|
|
728
734
|
*/
|
|
@@ -972,7 +978,7 @@ export type FailureKind = "auth" | "rate_limit" | "model_not_found" | "quota" |
|
|
|
972
978
|
/**
|
|
973
979
|
* The outcome of attempting to run one harness.
|
|
974
980
|
*/
|
|
975
|
-
export type Status = "ok" | "nonzero" | "timeout" | "spawn-error" | "skipped" | "planned";
|
|
981
|
+
export type Status = "ok" | "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped" | "planned";
|
|
976
982
|
/**
|
|
977
983
|
* One normalized action a harness took, harness-agnostic so a single consumer
|
|
978
984
|
* assertion works across harnesses. Every field is always serialized (null when
|
|
@@ -42,7 +42,39 @@ export type OutputFormat = "text" | "json" | "stream-json";
|
|
|
42
42
|
/**
|
|
43
43
|
* The outcome of attempting to run one harness.
|
|
44
44
|
*/
|
|
45
|
-
export type Status = "ok" | "nonzero" | "timeout" | "spawn-error" | "skipped" | "planned";
|
|
45
|
+
export type Status = "ok" | "nonzero" | "timeout" | "cancelled" | "spawn-error" | "skipped" | "planned";
|
|
46
|
+
/**
|
|
47
|
+
* Measured execution telemetry for one harness run: when the invocation ran,
|
|
48
|
+
* and — when the harness's transcript supports it — how its wall clock split
|
|
49
|
+
* between provider latency and tool work.
|
|
50
|
+
*
|
|
51
|
+
* Carried from the runner/parser to the history writer *and*, since report
|
|
52
|
+
* schema `0.5`, serialized on [`RunResult::telemetry`]. Exposing it there is
|
|
53
|
+
* what lets a consumer read the numbers off the run it just made instead of
|
|
54
|
+
* re-opening the history file the same run wrote.
|
|
55
|
+
*
|
|
56
|
+
* Internally tagged by `source`, so the variant is a value a consumer switches
|
|
57
|
+
* on rather than a shape it has to sniff. Every variant states only what was
|
|
58
|
+
* actually measured — there is no variant meaning "no telemetry"; that is a
|
|
59
|
+
* `null` field.
|
|
60
|
+
*/
|
|
61
|
+
export type ExecutionTelemetry = {
|
|
62
|
+
finished_at: string | null;
|
|
63
|
+
model_ms: number | null;
|
|
64
|
+
source: "provider_measured";
|
|
65
|
+
started_at: string;
|
|
66
|
+
time_to_first_token_ms: number | null;
|
|
67
|
+
tool_ms: number | null;
|
|
68
|
+
[k: string]: unknown;
|
|
69
|
+
} | {
|
|
70
|
+
source: "partial_invocation";
|
|
71
|
+
started_at: string;
|
|
72
|
+
[k: string]: unknown;
|
|
73
|
+
} | {
|
|
74
|
+
source: "stdout_observed";
|
|
75
|
+
tool_ms: number;
|
|
76
|
+
[k: string]: unknown;
|
|
77
|
+
};
|
|
46
78
|
/**
|
|
47
79
|
* One normalized action a harness took, harness-agnostic so a single consumer
|
|
48
80
|
* assertion works across harnesses. Every field is always serialized (null when
|
|
@@ -395,6 +427,14 @@ export interface RunResult {
|
|
|
395
427
|
* see what the harness produced.
|
|
396
428
|
*/
|
|
397
429
|
structured: unknown;
|
|
430
|
+
/**
|
|
431
|
+
* Measured execution telemetry for this run: the invocation bounds and, when
|
|
432
|
+
* the harness's transcript carried a trace to read them from, the
|
|
433
|
+
* model/tool split. `null` when nothing was measured — never estimated.
|
|
434
|
+
* Added in report schema `0.5`; before that a consumer had to re-read the
|
|
435
|
+
* history file for numbers the run itself already had.
|
|
436
|
+
*/
|
|
437
|
+
telemetry: ExecutionTelemetry | null;
|
|
398
438
|
/**
|
|
399
439
|
* Best-effort final assistant text; `null` when extraction is impossible.
|
|
400
440
|
*/
|