@vibe-agent-toolkit/agent-skills 0.2.0-rc.2 → 0.2.0-rc.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/files-config.d.ts +34 -1
- package/dist/files-config.d.ts.map +1 -1
- package/dist/files-config.js +280 -36
- package/dist/files-config.js.map +1 -1
- package/dist/fs-attribution.d.ts +45 -0
- package/dist/fs-attribution.d.ts.map +1 -0
- package/dist/fs-attribution.js +57 -0
- package/dist/fs-attribution.js.map +1 -0
- package/dist/index.d.ts +5 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +8 -2
- package/dist/index.js.map +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts.map +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts.map +1 -1
- package/dist/inventory/index.d.ts +1 -1
- package/dist/inventory/index.d.ts.map +1 -1
- package/dist/inventory/index.js +1 -1
- package/dist/inventory/index.js.map +1 -1
- package/dist/inventory/serialize.d.ts +4 -4
- package/dist/inventory/serialize.d.ts.map +1 -1
- package/dist/inventory/serialize.js +6 -6
- package/dist/inventory/serialize.js.map +1 -1
- package/dist/post-build-checks.d.ts +1 -1
- package/dist/post-build-checks.d.ts.map +1 -1
- package/dist/post-build-checks.js.map +1 -1
- package/dist/projection/skill-extent.d.ts +247 -0
- package/dist/projection/skill-extent.d.ts.map +1 -0
- package/dist/projection/skill-extent.js +558 -0
- package/dist/projection/skill-extent.js.map +1 -0
- package/dist/schemas/agent-skill-frontmatter.js +1 -1
- package/dist/schemas/agent-skill-frontmatter.js.map +1 -1
- package/dist/skill-packager.d.ts +34 -4
- package/dist/skill-packager.d.ts.map +1 -1
- package/dist/skill-packager.js +113 -23
- package/dist/skill-packager.js.map +1 -1
- package/dist/skill-source/git-clone.d.ts.map +1 -1
- package/dist/skill-source/git-clone.js +9 -8
- package/dist/skill-source/git-clone.js.map +1 -1
- package/dist/skill-test/baseline-delta.d.ts +1575 -0
- package/dist/skill-test/baseline-delta.d.ts.map +1 -0
- package/dist/skill-test/baseline-delta.js +272 -0
- package/dist/skill-test/baseline-delta.js.map +1 -0
- package/dist/skill-test/baseline-integrity.d.ts +944 -0
- package/dist/skill-test/baseline-integrity.d.ts.map +1 -0
- package/dist/skill-test/baseline-integrity.js +3083 -0
- package/dist/skill-test/baseline-integrity.js.map +1 -0
- package/dist/skill-test/eval-executor.d.ts +19 -3
- package/dist/skill-test/eval-executor.d.ts.map +1 -1
- package/dist/skill-test/eval-executor.js +3 -3
- package/dist/skill-test/eval-executor.js.map +1 -1
- package/dist/skill-test/eval-fragment.d.ts +80 -6
- package/dist/skill-test/eval-fragment.d.ts.map +1 -1
- package/dist/skill-test/eval-fragment.js +61 -7
- package/dist/skill-test/eval-fragment.js.map +1 -1
- package/dist/skill-test/eval-grader.d.ts.map +1 -1
- package/dist/skill-test/eval-grader.js +143 -7
- package/dist/skill-test/eval-grader.js.map +1 -1
- package/dist/skill-test/eval-inputs.d.ts +107 -6
- package/dist/skill-test/eval-inputs.d.ts.map +1 -1
- package/dist/skill-test/eval-inputs.js +173 -16
- package/dist/skill-test/eval-inputs.js.map +1 -1
- package/dist/skill-test/eval-lint.d.ts +13 -0
- package/dist/skill-test/eval-lint.d.ts.map +1 -1
- package/dist/skill-test/eval-lint.js +17 -4
- package/dist/skill-test/eval-lint.js.map +1 -1
- package/dist/skill-test/executor-prompt.d.ts +13 -5
- package/dist/skill-test/executor-prompt.d.ts.map +1 -1
- package/dist/skill-test/executor-prompt.js +8 -5
- package/dist/skill-test/executor-prompt.js.map +1 -1
- package/dist/skill-test/exit-codes.d.ts +53 -16
- package/dist/skill-test/exit-codes.d.ts.map +1 -1
- package/dist/skill-test/exit-codes.js +123 -40
- package/dist/skill-test/exit-codes.js.map +1 -1
- package/dist/skill-test/fragment-merge.d.ts +1 -18
- package/dist/skill-test/fragment-merge.d.ts.map +1 -1
- package/dist/skill-test/fragment-merge.js +63 -4
- package/dist/skill-test/fragment-merge.js.map +1 -1
- package/dist/skill-test/grader-prompt.d.ts +27 -7
- package/dist/skill-test/grader-prompt.d.ts.map +1 -1
- package/dist/skill-test/grader-prompt.js +253 -61
- package/dist/skill-test/grader-prompt.js.map +1 -1
- package/dist/skill-test/grader-text.d.ts +145 -0
- package/dist/skill-test/grader-text.d.ts.map +1 -0
- package/dist/skill-test/grader-text.js +410 -0
- package/dist/skill-test/grader-text.js.map +1 -0
- package/dist/skill-test/grading-adapter.d.ts +32 -0
- package/dist/skill-test/grading-adapter.d.ts.map +1 -1
- package/dist/skill-test/grading-adapter.js +50 -3
- package/dist/skill-test/grading-adapter.js.map +1 -1
- package/dist/skill-test/grading-schema.d.ts +160 -5
- package/dist/skill-test/grading-schema.d.ts.map +1 -1
- package/dist/skill-test/grading-schema.js +66 -7
- package/dist/skill-test/grading-schema.js.map +1 -1
- package/dist/skill-test/index.d.ts +1 -1
- package/dist/skill-test/index.d.ts.map +1 -1
- package/dist/skill-test/index.js +1 -1
- package/dist/skill-test/index.js.map +1 -1
- package/dist/skill-test/lock.d.ts +56 -0
- package/dist/skill-test/lock.d.ts.map +1 -1
- package/dist/skill-test/lock.js +75 -2
- package/dist/skill-test/lock.js.map +1 -1
- package/dist/skill-test/pipeline.d.ts +16 -0
- package/dist/skill-test/pipeline.d.ts.map +1 -1
- package/dist/skill-test/pipeline.js +16 -9
- package/dist/skill-test/pipeline.js.map +1 -1
- package/dist/skill-test/preflight.d.ts.map +1 -1
- package/dist/skill-test/preflight.js +58 -14
- package/dist/skill-test/preflight.js.map +1 -1
- package/dist/skill-test/run-harness.d.ts +578 -35
- package/dist/skill-test/run-harness.d.ts.map +1 -1
- package/dist/skill-test/run-harness.js +1278 -147
- package/dist/skill-test/run-harness.js.map +1 -1
- package/dist/skill-test/staging.d.ts +8 -2
- package/dist/skill-test/staging.d.ts.map +1 -1
- package/dist/skill-test/staging.js +8 -4
- package/dist/skill-test/staging.js.map +1 -1
- package/dist/skill-test/tool-eval-schema.d.ts +6 -6
- package/dist/test-input.d.ts +1 -1
- package/dist/test-input.d.ts.map +1 -1
- package/dist/validators/agent-instruction-presence.d.ts +1 -1
- package/dist/validators/agent-instruction-presence.d.ts.map +1 -1
- package/dist/validators/bundled-resource-link-detection.d.ts +1 -1
- package/dist/validators/bundled-resource-link-detection.d.ts.map +1 -1
- package/dist/validators/bundled-resource-link-detection.js +1 -1
- package/dist/validators/bundled-resource-link-detection.js.map +1 -1
- package/dist/validators/compat-detectors.d.ts +1 -1
- package/dist/validators/compat-detectors.d.ts.map +1 -1
- package/dist/validators/compat-detectors.js +1 -1
- package/dist/validators/compat-detectors.js.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.js +1 -1
- package/dist/validators/cross-skill-dependency-detection.js.map +1 -1
- package/dist/validators/description-style-detection.d.ts +1 -1
- package/dist/validators/description-style-detection.d.ts.map +1 -1
- package/dist/validators/description-style-detection.js +1 -1
- package/dist/validators/description-style-detection.js.map +1 -1
- package/dist/validators/frontmatter-validation.d.ts +1 -1
- package/dist/validators/frontmatter-validation.d.ts.map +1 -1
- package/dist/validators/frontmatter-validation.js +1 -1
- package/dist/validators/frontmatter-validation.js.map +1 -1
- package/dist/validators/imperative-body-detection.d.ts +1 -1
- package/dist/validators/imperative-body-detection.d.ts.map +1 -1
- package/dist/validators/imperative-body-detection.js +1 -1
- package/dist/validators/imperative-body-detection.js.map +1 -1
- package/dist/validators/index.d.ts +1 -1
- package/dist/validators/index.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.d.ts +1 -1
- package/dist/validators/kebab-case-detection.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.js +1 -1
- package/dist/validators/kebab-case-detection.js.map +1 -1
- package/dist/validators/marketplace-validator.js +1 -1
- package/dist/validators/marketplace-validator.js.map +1 -1
- package/dist/validators/packaging-validator.d.ts +38 -5
- package/dist/validators/packaging-validator.d.ts.map +1 -1
- package/dist/validators/packaging-validator.js +87 -10
- package/dist/validators/packaging-validator.js.map +1 -1
- package/dist/validators/plugin-hosted-shape.d.ts +1 -1
- package/dist/validators/plugin-hosted-shape.d.ts.map +1 -1
- package/dist/validators/plugin-hosted-shape.js +1 -1
- package/dist/validators/plugin-hosted-shape.js.map +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts.map +1 -1
- package/dist/validators/plugin-recommended-fields.js +1 -1
- package/dist/validators/plugin-recommended-fields.js.map +1 -1
- package/dist/validators/registry-validator.js +1 -1
- package/dist/validators/registry-validator.js.map +1 -1
- package/dist/validators/rule-engine/rule-engine.d.ts +1 -1
- package/dist/validators/rule-engine/rule-engine.d.ts.map +1 -1
- package/dist/validators/rule-engine/rule-engine.js +20 -9
- package/dist/validators/rule-engine/rule-engine.js.map +1 -1
- package/dist/validators/skill-validator.d.ts.map +1 -1
- package/dist/validators/skill-validator.js +21 -3
- package/dist/validators/skill-validator.js.map +1 -1
- package/dist/validators/types.d.ts +1 -1
- package/dist/validators/types.d.ts.map +1 -1
- package/dist/validators/unified-validator.js +1 -1
- package/dist/validators/unified-validator.js.map +1 -1
- package/dist/validators/validation-rules.d.ts +1 -1
- package/dist/validators/validation-rules.d.ts.map +1 -1
- package/dist/validators/walker-to-issues.d.ts +7 -3
- package/dist/validators/walker-to-issues.d.ts.map +1 -1
- package/dist/validators/walker-to-issues.js +37 -8
- package/dist/validators/walker-to-issues.js.map +1 -1
- package/dist/walk-link-graph.d.ts.map +1 -1
- package/dist/walk-link-graph.js +36 -2
- package/dist/walk-link-graph.js.map +1 -1
- package/package.json +7 -7
|
@@ -0,0 +1,1575 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { type ArmEvalGrade, type BaselineArmSkew, type BaselineControlArmFailure } from './baseline-integrity.js';
|
|
3
|
+
export declare const EvalDeltaSchema: z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
4
|
+
evalId: z.ZodString;
|
|
5
|
+
withPassed: z.ZodNumber;
|
|
6
|
+
withTotal: z.ZodNumber;
|
|
7
|
+
withoutPassed: z.ZodNumber;
|
|
8
|
+
withoutTotal: z.ZodNumber;
|
|
9
|
+
/**
|
|
10
|
+
* `withPassed − withoutPassed`, or `null` when THIS eval's two arms were graded to
|
|
11
|
+
* different depths. Signed: a negative delta means the control outscored the
|
|
12
|
+
* treatment, which is a real observed outcome and is never clamped to zero.
|
|
13
|
+
*/
|
|
14
|
+
delta: z.ZodNullable<z.ZodNumber>;
|
|
15
|
+
}, "strict", z.ZodTypeAny, {
|
|
16
|
+
evalId: string;
|
|
17
|
+
withTotal: number;
|
|
18
|
+
withoutTotal: number;
|
|
19
|
+
withPassed: number;
|
|
20
|
+
withoutPassed: number;
|
|
21
|
+
delta: number | null;
|
|
22
|
+
}, {
|
|
23
|
+
evalId: string;
|
|
24
|
+
withTotal: number;
|
|
25
|
+
withoutTotal: number;
|
|
26
|
+
withPassed: number;
|
|
27
|
+
withoutPassed: number;
|
|
28
|
+
delta: number | null;
|
|
29
|
+
}>, {
|
|
30
|
+
evalId: string;
|
|
31
|
+
withTotal: number;
|
|
32
|
+
withoutTotal: number;
|
|
33
|
+
withPassed: number;
|
|
34
|
+
withoutPassed: number;
|
|
35
|
+
delta: number | null;
|
|
36
|
+
}, {
|
|
37
|
+
evalId: string;
|
|
38
|
+
withTotal: number;
|
|
39
|
+
withoutTotal: number;
|
|
40
|
+
withPassed: number;
|
|
41
|
+
withoutPassed: number;
|
|
42
|
+
delta: number | null;
|
|
43
|
+
}>, {
|
|
44
|
+
evalId: string;
|
|
45
|
+
withTotal: number;
|
|
46
|
+
withoutTotal: number;
|
|
47
|
+
withPassed: number;
|
|
48
|
+
withoutPassed: number;
|
|
49
|
+
delta: number | null;
|
|
50
|
+
}, {
|
|
51
|
+
evalId: string;
|
|
52
|
+
withTotal: number;
|
|
53
|
+
withoutTotal: number;
|
|
54
|
+
withPassed: number;
|
|
55
|
+
withoutPassed: number;
|
|
56
|
+
delta: number | null;
|
|
57
|
+
}>;
|
|
58
|
+
export type EvalDelta = z.infer<typeof EvalDeltaSchema>;
|
|
59
|
+
/**
|
|
60
|
+
* The suite the delta does NOT cover, when cost-tiered fail-fast truncated the run.
|
|
61
|
+
*
|
|
62
|
+
* `shouldGateAfterTier` stops the run the moment any WITH-arm eval in a tier does not
|
|
63
|
+
* fully pass, and it stops BOTH arms, so the arithmetic over the tiers that ran stays
|
|
64
|
+
* sound. What is not sound is reading the result as the suite's delta: a tier-0
|
|
65
|
+
* failure in an 8-eval suite printed `Baseline delta: +1 (with skill: 2/2, without
|
|
66
|
+
* skill: 1/2)`, byte-identical to a complete 2-eval suite that measured everything it
|
|
67
|
+
* declared.
|
|
68
|
+
*
|
|
69
|
+
* That is not an exotic path. `--baseline` exists to measure a skill that may not be
|
|
70
|
+
* helping, so a WITH-arm failure is the EXPECTED state of an interesting baseline run
|
|
71
|
+
* — the truncated shape is the common one, not the corner case.
|
|
72
|
+
*/
|
|
73
|
+
export declare const DeltaTruncationSchema: z.ZodObject<{
|
|
74
|
+
/** The (lower) tier whose gating failure stopped the run. */
|
|
75
|
+
gatedByTier: z.ZodNumber;
|
|
76
|
+
/** The lowest tier that never ran. */
|
|
77
|
+
firstSkippedTier: z.ZodNumber;
|
|
78
|
+
/** How many evals were never run on EITHER arm. */
|
|
79
|
+
totalSkipped: z.ZodNumber;
|
|
80
|
+
/** Their ids, flattened across the skipped tiers in ascending tier order. */
|
|
81
|
+
evalIds: z.ZodArray<z.ZodString, "many">;
|
|
82
|
+
}, "strict", z.ZodTypeAny, {
|
|
83
|
+
gatedByTier: number;
|
|
84
|
+
firstSkippedTier: number;
|
|
85
|
+
totalSkipped: number;
|
|
86
|
+
evalIds: string[];
|
|
87
|
+
}, {
|
|
88
|
+
gatedByTier: number;
|
|
89
|
+
firstSkippedTier: number;
|
|
90
|
+
totalSkipped: number;
|
|
91
|
+
evalIds: string[];
|
|
92
|
+
}>;
|
|
93
|
+
export type DeltaTruncation = z.infer<typeof DeltaTruncationSchema>;
|
|
94
|
+
/**
|
|
95
|
+
* The run-level `baselineDelta` block, a SIBLING of `baselineIntegrity` in
|
|
96
|
+
* `baseline.json`. The two are separate on purpose: integrity says whether the
|
|
97
|
+
* comparison is trustworthy, this says what the comparison IS, and a reader who
|
|
98
|
+
* wants only the number should not have to reconstruct it from two summaries.
|
|
99
|
+
*/
|
|
100
|
+
export declare const BaselineDeltaSchema: z.ZodEffects<z.ZodObject<{
|
|
101
|
+
with: z.ZodEffects<z.ZodObject<{
|
|
102
|
+
passed: z.ZodNumber;
|
|
103
|
+
total: z.ZodNumber;
|
|
104
|
+
}, "strict", z.ZodTypeAny, {
|
|
105
|
+
total: number;
|
|
106
|
+
passed: number;
|
|
107
|
+
}, {
|
|
108
|
+
total: number;
|
|
109
|
+
passed: number;
|
|
110
|
+
}>, {
|
|
111
|
+
total: number;
|
|
112
|
+
passed: number;
|
|
113
|
+
}, {
|
|
114
|
+
total: number;
|
|
115
|
+
passed: number;
|
|
116
|
+
}>;
|
|
117
|
+
without: z.ZodEffects<z.ZodObject<{
|
|
118
|
+
passed: z.ZodNumber;
|
|
119
|
+
total: z.ZodNumber;
|
|
120
|
+
}, "strict", z.ZodTypeAny, {
|
|
121
|
+
total: number;
|
|
122
|
+
passed: number;
|
|
123
|
+
}, {
|
|
124
|
+
total: number;
|
|
125
|
+
passed: number;
|
|
126
|
+
}>, {
|
|
127
|
+
total: number;
|
|
128
|
+
passed: number;
|
|
129
|
+
}, {
|
|
130
|
+
total: number;
|
|
131
|
+
passed: number;
|
|
132
|
+
}>;
|
|
133
|
+
/**
|
|
134
|
+
* Run-level lift, or `null` when ANY eval was incomparable.
|
|
135
|
+
*
|
|
136
|
+
* Withheld for the whole run rather than computed over the comparable subset: the
|
|
137
|
+
* arm totals above include every eval, so a partial-subset delta would not be the
|
|
138
|
+
* difference between the two numbers printed beside it — which is precisely the
|
|
139
|
+
* kind of "close enough" number that gets quoted without its caveat.
|
|
140
|
+
*/
|
|
141
|
+
delta: z.ZodNullable<z.ZodNumber>;
|
|
142
|
+
perEval: z.ZodArray<z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
143
|
+
evalId: z.ZodString;
|
|
144
|
+
withPassed: z.ZodNumber;
|
|
145
|
+
withTotal: z.ZodNumber;
|
|
146
|
+
withoutPassed: z.ZodNumber;
|
|
147
|
+
withoutTotal: z.ZodNumber;
|
|
148
|
+
/**
|
|
149
|
+
* `withPassed − withoutPassed`, or `null` when THIS eval's two arms were graded to
|
|
150
|
+
* different depths. Signed: a negative delta means the control outscored the
|
|
151
|
+
* treatment, which is a real observed outcome and is never clamped to zero.
|
|
152
|
+
*/
|
|
153
|
+
delta: z.ZodNullable<z.ZodNumber>;
|
|
154
|
+
}, "strict", z.ZodTypeAny, {
|
|
155
|
+
evalId: string;
|
|
156
|
+
withTotal: number;
|
|
157
|
+
withoutTotal: number;
|
|
158
|
+
withPassed: number;
|
|
159
|
+
withoutPassed: number;
|
|
160
|
+
delta: number | null;
|
|
161
|
+
}, {
|
|
162
|
+
evalId: string;
|
|
163
|
+
withTotal: number;
|
|
164
|
+
withoutTotal: number;
|
|
165
|
+
withPassed: number;
|
|
166
|
+
withoutPassed: number;
|
|
167
|
+
delta: number | null;
|
|
168
|
+
}>, {
|
|
169
|
+
evalId: string;
|
|
170
|
+
withTotal: number;
|
|
171
|
+
withoutTotal: number;
|
|
172
|
+
withPassed: number;
|
|
173
|
+
withoutPassed: number;
|
|
174
|
+
delta: number | null;
|
|
175
|
+
}, {
|
|
176
|
+
evalId: string;
|
|
177
|
+
withTotal: number;
|
|
178
|
+
withoutTotal: number;
|
|
179
|
+
withPassed: number;
|
|
180
|
+
withoutPassed: number;
|
|
181
|
+
delta: number | null;
|
|
182
|
+
}>, {
|
|
183
|
+
evalId: string;
|
|
184
|
+
withTotal: number;
|
|
185
|
+
withoutTotal: number;
|
|
186
|
+
withPassed: number;
|
|
187
|
+
withoutPassed: number;
|
|
188
|
+
delta: number | null;
|
|
189
|
+
}, {
|
|
190
|
+
evalId: string;
|
|
191
|
+
withTotal: number;
|
|
192
|
+
withoutTotal: number;
|
|
193
|
+
withPassed: number;
|
|
194
|
+
withoutPassed: number;
|
|
195
|
+
delta: number | null;
|
|
196
|
+
}>, "many">;
|
|
197
|
+
/**
|
|
198
|
+
* Control-arm evals that produced no grade at all, so there was nothing to
|
|
199
|
+
* subtract for them. Carried HERE as well as on `baselineIntegrity` — it is one
|
|
200
|
+
* derivation handed to two consumers (the same arrangement as `ArmEvalGrade`
|
|
201
|
+
* feeding both the parity check and this block), never two derivations — because
|
|
202
|
+
* a reader who opens `baselineDelta` and finds `delta: null` must be able to learn
|
|
203
|
+
* WHY from the block that withheld it, without first having to know the
|
|
204
|
+
* contamination vocabulary next door.
|
|
205
|
+
*/
|
|
206
|
+
controlArmFailures: z.ZodArray<z.ZodObject<{
|
|
207
|
+
evalId: z.ZodString;
|
|
208
|
+
detail: z.ZodString;
|
|
209
|
+
}, "strict", z.ZodTypeAny, {
|
|
210
|
+
detail: string;
|
|
211
|
+
evalId: string;
|
|
212
|
+
}, {
|
|
213
|
+
detail: string;
|
|
214
|
+
evalId: string;
|
|
215
|
+
}>, "many">;
|
|
216
|
+
/**
|
|
217
|
+
* What the fail-fast gate cut off, or `null` when the whole declared suite ran.
|
|
218
|
+
*
|
|
219
|
+
* `null` rather than an absent key: "this delta covers everything" is a CLAIM, and
|
|
220
|
+
* the field has to be able to make it. An optional field would make a complete run
|
|
221
|
+
* and a run written before this existed indistinguishable — the same argument that
|
|
222
|
+
* makes `baselineIntegrity` unconditional.
|
|
223
|
+
*/
|
|
224
|
+
truncated: z.ZodNullable<z.ZodObject<{
|
|
225
|
+
/** The (lower) tier whose gating failure stopped the run. */
|
|
226
|
+
gatedByTier: z.ZodNumber;
|
|
227
|
+
/** The lowest tier that never ran. */
|
|
228
|
+
firstSkippedTier: z.ZodNumber;
|
|
229
|
+
/** How many evals were never run on EITHER arm. */
|
|
230
|
+
totalSkipped: z.ZodNumber;
|
|
231
|
+
/** Their ids, flattened across the skipped tiers in ascending tier order. */
|
|
232
|
+
evalIds: z.ZodArray<z.ZodString, "many">;
|
|
233
|
+
}, "strict", z.ZodTypeAny, {
|
|
234
|
+
gatedByTier: number;
|
|
235
|
+
firstSkippedTier: number;
|
|
236
|
+
totalSkipped: number;
|
|
237
|
+
evalIds: string[];
|
|
238
|
+
}, {
|
|
239
|
+
gatedByTier: number;
|
|
240
|
+
firstSkippedTier: number;
|
|
241
|
+
totalSkipped: number;
|
|
242
|
+
evalIds: string[];
|
|
243
|
+
}>>;
|
|
244
|
+
}, "strict", z.ZodTypeAny, {
|
|
245
|
+
with: {
|
|
246
|
+
total: number;
|
|
247
|
+
passed: number;
|
|
248
|
+
};
|
|
249
|
+
without: {
|
|
250
|
+
total: number;
|
|
251
|
+
passed: number;
|
|
252
|
+
};
|
|
253
|
+
controlArmFailures: {
|
|
254
|
+
detail: string;
|
|
255
|
+
evalId: string;
|
|
256
|
+
}[];
|
|
257
|
+
delta: number | null;
|
|
258
|
+
perEval: {
|
|
259
|
+
evalId: string;
|
|
260
|
+
withTotal: number;
|
|
261
|
+
withoutTotal: number;
|
|
262
|
+
withPassed: number;
|
|
263
|
+
withoutPassed: number;
|
|
264
|
+
delta: number | null;
|
|
265
|
+
}[];
|
|
266
|
+
truncated: {
|
|
267
|
+
gatedByTier: number;
|
|
268
|
+
firstSkippedTier: number;
|
|
269
|
+
totalSkipped: number;
|
|
270
|
+
evalIds: string[];
|
|
271
|
+
} | null;
|
|
272
|
+
}, {
|
|
273
|
+
with: {
|
|
274
|
+
total: number;
|
|
275
|
+
passed: number;
|
|
276
|
+
};
|
|
277
|
+
without: {
|
|
278
|
+
total: number;
|
|
279
|
+
passed: number;
|
|
280
|
+
};
|
|
281
|
+
controlArmFailures: {
|
|
282
|
+
detail: string;
|
|
283
|
+
evalId: string;
|
|
284
|
+
}[];
|
|
285
|
+
delta: number | null;
|
|
286
|
+
perEval: {
|
|
287
|
+
evalId: string;
|
|
288
|
+
withTotal: number;
|
|
289
|
+
withoutTotal: number;
|
|
290
|
+
withPassed: number;
|
|
291
|
+
withoutPassed: number;
|
|
292
|
+
delta: number | null;
|
|
293
|
+
}[];
|
|
294
|
+
truncated: {
|
|
295
|
+
gatedByTier: number;
|
|
296
|
+
firstSkippedTier: number;
|
|
297
|
+
totalSkipped: number;
|
|
298
|
+
evalIds: string[];
|
|
299
|
+
} | null;
|
|
300
|
+
}>, {
|
|
301
|
+
with: {
|
|
302
|
+
total: number;
|
|
303
|
+
passed: number;
|
|
304
|
+
};
|
|
305
|
+
without: {
|
|
306
|
+
total: number;
|
|
307
|
+
passed: number;
|
|
308
|
+
};
|
|
309
|
+
controlArmFailures: {
|
|
310
|
+
detail: string;
|
|
311
|
+
evalId: string;
|
|
312
|
+
}[];
|
|
313
|
+
delta: number | null;
|
|
314
|
+
perEval: {
|
|
315
|
+
evalId: string;
|
|
316
|
+
withTotal: number;
|
|
317
|
+
withoutTotal: number;
|
|
318
|
+
withPassed: number;
|
|
319
|
+
withoutPassed: number;
|
|
320
|
+
delta: number | null;
|
|
321
|
+
}[];
|
|
322
|
+
truncated: {
|
|
323
|
+
gatedByTier: number;
|
|
324
|
+
firstSkippedTier: number;
|
|
325
|
+
totalSkipped: number;
|
|
326
|
+
evalIds: string[];
|
|
327
|
+
} | null;
|
|
328
|
+
}, {
|
|
329
|
+
with: {
|
|
330
|
+
total: number;
|
|
331
|
+
passed: number;
|
|
332
|
+
};
|
|
333
|
+
without: {
|
|
334
|
+
total: number;
|
|
335
|
+
passed: number;
|
|
336
|
+
};
|
|
337
|
+
controlArmFailures: {
|
|
338
|
+
detail: string;
|
|
339
|
+
evalId: string;
|
|
340
|
+
}[];
|
|
341
|
+
delta: number | null;
|
|
342
|
+
perEval: {
|
|
343
|
+
evalId: string;
|
|
344
|
+
withTotal: number;
|
|
345
|
+
withoutTotal: number;
|
|
346
|
+
withPassed: number;
|
|
347
|
+
withoutPassed: number;
|
|
348
|
+
delta: number | null;
|
|
349
|
+
}[];
|
|
350
|
+
truncated: {
|
|
351
|
+
gatedByTier: number;
|
|
352
|
+
firstSkippedTier: number;
|
|
353
|
+
totalSkipped: number;
|
|
354
|
+
evalIds: string[];
|
|
355
|
+
} | null;
|
|
356
|
+
}>;
|
|
357
|
+
export type BaselineDelta = z.infer<typeof BaselineDeltaSchema>;
|
|
358
|
+
/**
|
|
359
|
+
* `baseline.json` as a whole: a grading report (validated separately by the
|
|
360
|
+
* caller, which owns that schema) carrying vat's two extra blocks.
|
|
361
|
+
*
|
|
362
|
+
* Exists so the post-merge fail-closed gate can validate the file it just wrote.
|
|
363
|
+
* `.passthrough()` because the grading fields ride alongside and are checked by
|
|
364
|
+
* `GradingReportSchema`; the two blocks below are what THIS schema is for, and
|
|
365
|
+
* both are REQUIRED — an absent `baselineIntegrity` is precisely the "written
|
|
366
|
+
* before the check existed" state the block was made unconditional to rule out.
|
|
367
|
+
*/
|
|
368
|
+
export declare const BaselineArtifactSchema: z.ZodObject<{
|
|
369
|
+
baselineIntegrity: z.ZodObject<{
|
|
370
|
+
contaminated: z.ZodBoolean;
|
|
371
|
+
degraded: z.ZodArray<z.ZodObject<{
|
|
372
|
+
reason: z.ZodEnum<["transcript-unparsed", "cwd-unknown", "cwd-untracked", "transcript-malformed", "glob-unexpanded"]>;
|
|
373
|
+
detail: z.ZodString;
|
|
374
|
+
evalId: z.ZodOptional<z.ZodString>;
|
|
375
|
+
}, "strict", z.ZodTypeAny, {
|
|
376
|
+
detail: string;
|
|
377
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
378
|
+
evalId?: string | undefined;
|
|
379
|
+
}, {
|
|
380
|
+
detail: string;
|
|
381
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
382
|
+
evalId?: string | undefined;
|
|
383
|
+
}>, "many">;
|
|
384
|
+
comparable: z.ZodBoolean;
|
|
385
|
+
skew: z.ZodArray<z.ZodObject<{
|
|
386
|
+
evalId: z.ZodString;
|
|
387
|
+
withTotal: z.ZodNumber;
|
|
388
|
+
withoutTotal: z.ZodNumber;
|
|
389
|
+
}, "strict", z.ZodTypeAny, {
|
|
390
|
+
evalId: string;
|
|
391
|
+
withTotal: number;
|
|
392
|
+
withoutTotal: number;
|
|
393
|
+
}, {
|
|
394
|
+
evalId: string;
|
|
395
|
+
withTotal: number;
|
|
396
|
+
withoutTotal: number;
|
|
397
|
+
}>, "many">;
|
|
398
|
+
controlArmFailures: z.ZodArray<z.ZodObject<{
|
|
399
|
+
evalId: z.ZodString;
|
|
400
|
+
detail: z.ZodString;
|
|
401
|
+
}, "strict", z.ZodTypeAny, {
|
|
402
|
+
detail: string;
|
|
403
|
+
evalId: string;
|
|
404
|
+
}, {
|
|
405
|
+
detail: string;
|
|
406
|
+
evalId: string;
|
|
407
|
+
}>, "many">;
|
|
408
|
+
summary: z.ZodString;
|
|
409
|
+
signals: z.ZodArray<z.ZodEnum<["harness-path", "declared-executable", "sibling-arm", "vat-private-dir", "skill-content"]>, "many">;
|
|
410
|
+
findings: z.ZodArray<z.ZodObject<{
|
|
411
|
+
evalId: z.ZodString;
|
|
412
|
+
hits: z.ZodArray<z.ZodObject<{
|
|
413
|
+
kind: z.ZodEnum<["harness-path", "declared-executable", "sibling-arm", "vat-private-dir", "skill-content"]>;
|
|
414
|
+
match: z.ZodString;
|
|
415
|
+
excerpt: z.ZodString;
|
|
416
|
+
}, "strict", z.ZodTypeAny, {
|
|
417
|
+
match: string;
|
|
418
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
419
|
+
excerpt: string;
|
|
420
|
+
}, {
|
|
421
|
+
match: string;
|
|
422
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
423
|
+
excerpt: string;
|
|
424
|
+
}>, "many">;
|
|
425
|
+
}, "strict", z.ZodTypeAny, {
|
|
426
|
+
evalId: string;
|
|
427
|
+
hits: {
|
|
428
|
+
match: string;
|
|
429
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
430
|
+
excerpt: string;
|
|
431
|
+
}[];
|
|
432
|
+
}, {
|
|
433
|
+
evalId: string;
|
|
434
|
+
hits: {
|
|
435
|
+
match: string;
|
|
436
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
437
|
+
excerpt: string;
|
|
438
|
+
}[];
|
|
439
|
+
}>, "many">;
|
|
440
|
+
}, "strict", z.ZodTypeAny, {
|
|
441
|
+
summary: string;
|
|
442
|
+
contaminated: boolean;
|
|
443
|
+
degraded: {
|
|
444
|
+
detail: string;
|
|
445
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
446
|
+
evalId?: string | undefined;
|
|
447
|
+
}[];
|
|
448
|
+
comparable: boolean;
|
|
449
|
+
skew: {
|
|
450
|
+
evalId: string;
|
|
451
|
+
withTotal: number;
|
|
452
|
+
withoutTotal: number;
|
|
453
|
+
}[];
|
|
454
|
+
controlArmFailures: {
|
|
455
|
+
detail: string;
|
|
456
|
+
evalId: string;
|
|
457
|
+
}[];
|
|
458
|
+
signals: ("harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content")[];
|
|
459
|
+
findings: {
|
|
460
|
+
evalId: string;
|
|
461
|
+
hits: {
|
|
462
|
+
match: string;
|
|
463
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
464
|
+
excerpt: string;
|
|
465
|
+
}[];
|
|
466
|
+
}[];
|
|
467
|
+
}, {
|
|
468
|
+
summary: string;
|
|
469
|
+
contaminated: boolean;
|
|
470
|
+
degraded: {
|
|
471
|
+
detail: string;
|
|
472
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
473
|
+
evalId?: string | undefined;
|
|
474
|
+
}[];
|
|
475
|
+
comparable: boolean;
|
|
476
|
+
skew: {
|
|
477
|
+
evalId: string;
|
|
478
|
+
withTotal: number;
|
|
479
|
+
withoutTotal: number;
|
|
480
|
+
}[];
|
|
481
|
+
controlArmFailures: {
|
|
482
|
+
detail: string;
|
|
483
|
+
evalId: string;
|
|
484
|
+
}[];
|
|
485
|
+
signals: ("harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content")[];
|
|
486
|
+
findings: {
|
|
487
|
+
evalId: string;
|
|
488
|
+
hits: {
|
|
489
|
+
match: string;
|
|
490
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
491
|
+
excerpt: string;
|
|
492
|
+
}[];
|
|
493
|
+
}[];
|
|
494
|
+
}>;
|
|
495
|
+
baselineDelta: z.ZodEffects<z.ZodObject<{
|
|
496
|
+
with: z.ZodEffects<z.ZodObject<{
|
|
497
|
+
passed: z.ZodNumber;
|
|
498
|
+
total: z.ZodNumber;
|
|
499
|
+
}, "strict", z.ZodTypeAny, {
|
|
500
|
+
total: number;
|
|
501
|
+
passed: number;
|
|
502
|
+
}, {
|
|
503
|
+
total: number;
|
|
504
|
+
passed: number;
|
|
505
|
+
}>, {
|
|
506
|
+
total: number;
|
|
507
|
+
passed: number;
|
|
508
|
+
}, {
|
|
509
|
+
total: number;
|
|
510
|
+
passed: number;
|
|
511
|
+
}>;
|
|
512
|
+
without: z.ZodEffects<z.ZodObject<{
|
|
513
|
+
passed: z.ZodNumber;
|
|
514
|
+
total: z.ZodNumber;
|
|
515
|
+
}, "strict", z.ZodTypeAny, {
|
|
516
|
+
total: number;
|
|
517
|
+
passed: number;
|
|
518
|
+
}, {
|
|
519
|
+
total: number;
|
|
520
|
+
passed: number;
|
|
521
|
+
}>, {
|
|
522
|
+
total: number;
|
|
523
|
+
passed: number;
|
|
524
|
+
}, {
|
|
525
|
+
total: number;
|
|
526
|
+
passed: number;
|
|
527
|
+
}>;
|
|
528
|
+
/**
|
|
529
|
+
* Run-level lift, or `null` when ANY eval was incomparable.
|
|
530
|
+
*
|
|
531
|
+
* Withheld for the whole run rather than computed over the comparable subset: the
|
|
532
|
+
* arm totals above include every eval, so a partial-subset delta would not be the
|
|
533
|
+
* difference between the two numbers printed beside it — which is precisely the
|
|
534
|
+
* kind of "close enough" number that gets quoted without its caveat.
|
|
535
|
+
*/
|
|
536
|
+
delta: z.ZodNullable<z.ZodNumber>;
|
|
537
|
+
perEval: z.ZodArray<z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
538
|
+
evalId: z.ZodString;
|
|
539
|
+
withPassed: z.ZodNumber;
|
|
540
|
+
withTotal: z.ZodNumber;
|
|
541
|
+
withoutPassed: z.ZodNumber;
|
|
542
|
+
withoutTotal: z.ZodNumber;
|
|
543
|
+
/**
|
|
544
|
+
* `withPassed − withoutPassed`, or `null` when THIS eval's two arms were graded to
|
|
545
|
+
* different depths. Signed: a negative delta means the control outscored the
|
|
546
|
+
* treatment, which is a real observed outcome and is never clamped to zero.
|
|
547
|
+
*/
|
|
548
|
+
delta: z.ZodNullable<z.ZodNumber>;
|
|
549
|
+
}, "strict", z.ZodTypeAny, {
|
|
550
|
+
evalId: string;
|
|
551
|
+
withTotal: number;
|
|
552
|
+
withoutTotal: number;
|
|
553
|
+
withPassed: number;
|
|
554
|
+
withoutPassed: number;
|
|
555
|
+
delta: number | null;
|
|
556
|
+
}, {
|
|
557
|
+
evalId: string;
|
|
558
|
+
withTotal: number;
|
|
559
|
+
withoutTotal: number;
|
|
560
|
+
withPassed: number;
|
|
561
|
+
withoutPassed: number;
|
|
562
|
+
delta: number | null;
|
|
563
|
+
}>, {
|
|
564
|
+
evalId: string;
|
|
565
|
+
withTotal: number;
|
|
566
|
+
withoutTotal: number;
|
|
567
|
+
withPassed: number;
|
|
568
|
+
withoutPassed: number;
|
|
569
|
+
delta: number | null;
|
|
570
|
+
}, {
|
|
571
|
+
evalId: string;
|
|
572
|
+
withTotal: number;
|
|
573
|
+
withoutTotal: number;
|
|
574
|
+
withPassed: number;
|
|
575
|
+
withoutPassed: number;
|
|
576
|
+
delta: number | null;
|
|
577
|
+
}>, {
|
|
578
|
+
evalId: string;
|
|
579
|
+
withTotal: number;
|
|
580
|
+
withoutTotal: number;
|
|
581
|
+
withPassed: number;
|
|
582
|
+
withoutPassed: number;
|
|
583
|
+
delta: number | null;
|
|
584
|
+
}, {
|
|
585
|
+
evalId: string;
|
|
586
|
+
withTotal: number;
|
|
587
|
+
withoutTotal: number;
|
|
588
|
+
withPassed: number;
|
|
589
|
+
withoutPassed: number;
|
|
590
|
+
delta: number | null;
|
|
591
|
+
}>, "many">;
|
|
592
|
+
/**
|
|
593
|
+
* Control-arm evals that produced no grade at all, so there was nothing to
|
|
594
|
+
* subtract for them. Carried HERE as well as on `baselineIntegrity` — it is one
|
|
595
|
+
* derivation handed to two consumers (the same arrangement as `ArmEvalGrade`
|
|
596
|
+
* feeding both the parity check and this block), never two derivations — because
|
|
597
|
+
* a reader who opens `baselineDelta` and finds `delta: null` must be able to learn
|
|
598
|
+
* WHY from the block that withheld it, without first having to know the
|
|
599
|
+
* contamination vocabulary next door.
|
|
600
|
+
*/
|
|
601
|
+
controlArmFailures: z.ZodArray<z.ZodObject<{
|
|
602
|
+
evalId: z.ZodString;
|
|
603
|
+
detail: z.ZodString;
|
|
604
|
+
}, "strict", z.ZodTypeAny, {
|
|
605
|
+
detail: string;
|
|
606
|
+
evalId: string;
|
|
607
|
+
}, {
|
|
608
|
+
detail: string;
|
|
609
|
+
evalId: string;
|
|
610
|
+
}>, "many">;
|
|
611
|
+
/**
|
|
612
|
+
* What the fail-fast gate cut off, or `null` when the whole declared suite ran.
|
|
613
|
+
*
|
|
614
|
+
* `null` rather than an absent key: "this delta covers everything" is a CLAIM, and
|
|
615
|
+
* the field has to be able to make it. An optional field would make a complete run
|
|
616
|
+
* and a run written before this existed indistinguishable — the same argument that
|
|
617
|
+
* makes `baselineIntegrity` unconditional.
|
|
618
|
+
*/
|
|
619
|
+
truncated: z.ZodNullable<z.ZodObject<{
|
|
620
|
+
/** The (lower) tier whose gating failure stopped the run. */
|
|
621
|
+
gatedByTier: z.ZodNumber;
|
|
622
|
+
/** The lowest tier that never ran. */
|
|
623
|
+
firstSkippedTier: z.ZodNumber;
|
|
624
|
+
/** How many evals were never run on EITHER arm. */
|
|
625
|
+
totalSkipped: z.ZodNumber;
|
|
626
|
+
/** Their ids, flattened across the skipped tiers in ascending tier order. */
|
|
627
|
+
evalIds: z.ZodArray<z.ZodString, "many">;
|
|
628
|
+
}, "strict", z.ZodTypeAny, {
|
|
629
|
+
gatedByTier: number;
|
|
630
|
+
firstSkippedTier: number;
|
|
631
|
+
totalSkipped: number;
|
|
632
|
+
evalIds: string[];
|
|
633
|
+
}, {
|
|
634
|
+
gatedByTier: number;
|
|
635
|
+
firstSkippedTier: number;
|
|
636
|
+
totalSkipped: number;
|
|
637
|
+
evalIds: string[];
|
|
638
|
+
}>>;
|
|
639
|
+
}, "strict", z.ZodTypeAny, {
|
|
640
|
+
with: {
|
|
641
|
+
total: number;
|
|
642
|
+
passed: number;
|
|
643
|
+
};
|
|
644
|
+
without: {
|
|
645
|
+
total: number;
|
|
646
|
+
passed: number;
|
|
647
|
+
};
|
|
648
|
+
controlArmFailures: {
|
|
649
|
+
detail: string;
|
|
650
|
+
evalId: string;
|
|
651
|
+
}[];
|
|
652
|
+
delta: number | null;
|
|
653
|
+
perEval: {
|
|
654
|
+
evalId: string;
|
|
655
|
+
withTotal: number;
|
|
656
|
+
withoutTotal: number;
|
|
657
|
+
withPassed: number;
|
|
658
|
+
withoutPassed: number;
|
|
659
|
+
delta: number | null;
|
|
660
|
+
}[];
|
|
661
|
+
truncated: {
|
|
662
|
+
gatedByTier: number;
|
|
663
|
+
firstSkippedTier: number;
|
|
664
|
+
totalSkipped: number;
|
|
665
|
+
evalIds: string[];
|
|
666
|
+
} | null;
|
|
667
|
+
}, {
|
|
668
|
+
with: {
|
|
669
|
+
total: number;
|
|
670
|
+
passed: number;
|
|
671
|
+
};
|
|
672
|
+
without: {
|
|
673
|
+
total: number;
|
|
674
|
+
passed: number;
|
|
675
|
+
};
|
|
676
|
+
controlArmFailures: {
|
|
677
|
+
detail: string;
|
|
678
|
+
evalId: string;
|
|
679
|
+
}[];
|
|
680
|
+
delta: number | null;
|
|
681
|
+
perEval: {
|
|
682
|
+
evalId: string;
|
|
683
|
+
withTotal: number;
|
|
684
|
+
withoutTotal: number;
|
|
685
|
+
withPassed: number;
|
|
686
|
+
withoutPassed: number;
|
|
687
|
+
delta: number | null;
|
|
688
|
+
}[];
|
|
689
|
+
truncated: {
|
|
690
|
+
gatedByTier: number;
|
|
691
|
+
firstSkippedTier: number;
|
|
692
|
+
totalSkipped: number;
|
|
693
|
+
evalIds: string[];
|
|
694
|
+
} | null;
|
|
695
|
+
}>, {
|
|
696
|
+
with: {
|
|
697
|
+
total: number;
|
|
698
|
+
passed: number;
|
|
699
|
+
};
|
|
700
|
+
without: {
|
|
701
|
+
total: number;
|
|
702
|
+
passed: number;
|
|
703
|
+
};
|
|
704
|
+
controlArmFailures: {
|
|
705
|
+
detail: string;
|
|
706
|
+
evalId: string;
|
|
707
|
+
}[];
|
|
708
|
+
delta: number | null;
|
|
709
|
+
perEval: {
|
|
710
|
+
evalId: string;
|
|
711
|
+
withTotal: number;
|
|
712
|
+
withoutTotal: number;
|
|
713
|
+
withPassed: number;
|
|
714
|
+
withoutPassed: number;
|
|
715
|
+
delta: number | null;
|
|
716
|
+
}[];
|
|
717
|
+
truncated: {
|
|
718
|
+
gatedByTier: number;
|
|
719
|
+
firstSkippedTier: number;
|
|
720
|
+
totalSkipped: number;
|
|
721
|
+
evalIds: string[];
|
|
722
|
+
} | null;
|
|
723
|
+
}, {
|
|
724
|
+
with: {
|
|
725
|
+
total: number;
|
|
726
|
+
passed: number;
|
|
727
|
+
};
|
|
728
|
+
without: {
|
|
729
|
+
total: number;
|
|
730
|
+
passed: number;
|
|
731
|
+
};
|
|
732
|
+
controlArmFailures: {
|
|
733
|
+
detail: string;
|
|
734
|
+
evalId: string;
|
|
735
|
+
}[];
|
|
736
|
+
delta: number | null;
|
|
737
|
+
perEval: {
|
|
738
|
+
evalId: string;
|
|
739
|
+
withTotal: number;
|
|
740
|
+
withoutTotal: number;
|
|
741
|
+
withPassed: number;
|
|
742
|
+
withoutPassed: number;
|
|
743
|
+
delta: number | null;
|
|
744
|
+
}[];
|
|
745
|
+
truncated: {
|
|
746
|
+
gatedByTier: number;
|
|
747
|
+
firstSkippedTier: number;
|
|
748
|
+
totalSkipped: number;
|
|
749
|
+
evalIds: string[];
|
|
750
|
+
} | null;
|
|
751
|
+
}>;
|
|
752
|
+
}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
|
|
753
|
+
baselineIntegrity: z.ZodObject<{
|
|
754
|
+
contaminated: z.ZodBoolean;
|
|
755
|
+
degraded: z.ZodArray<z.ZodObject<{
|
|
756
|
+
reason: z.ZodEnum<["transcript-unparsed", "cwd-unknown", "cwd-untracked", "transcript-malformed", "glob-unexpanded"]>;
|
|
757
|
+
detail: z.ZodString;
|
|
758
|
+
evalId: z.ZodOptional<z.ZodString>;
|
|
759
|
+
}, "strict", z.ZodTypeAny, {
|
|
760
|
+
detail: string;
|
|
761
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
762
|
+
evalId?: string | undefined;
|
|
763
|
+
}, {
|
|
764
|
+
detail: string;
|
|
765
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
766
|
+
evalId?: string | undefined;
|
|
767
|
+
}>, "many">;
|
|
768
|
+
comparable: z.ZodBoolean;
|
|
769
|
+
skew: z.ZodArray<z.ZodObject<{
|
|
770
|
+
evalId: z.ZodString;
|
|
771
|
+
withTotal: z.ZodNumber;
|
|
772
|
+
withoutTotal: z.ZodNumber;
|
|
773
|
+
}, "strict", z.ZodTypeAny, {
|
|
774
|
+
evalId: string;
|
|
775
|
+
withTotal: number;
|
|
776
|
+
withoutTotal: number;
|
|
777
|
+
}, {
|
|
778
|
+
evalId: string;
|
|
779
|
+
withTotal: number;
|
|
780
|
+
withoutTotal: number;
|
|
781
|
+
}>, "many">;
|
|
782
|
+
controlArmFailures: z.ZodArray<z.ZodObject<{
|
|
783
|
+
evalId: z.ZodString;
|
|
784
|
+
detail: z.ZodString;
|
|
785
|
+
}, "strict", z.ZodTypeAny, {
|
|
786
|
+
detail: string;
|
|
787
|
+
evalId: string;
|
|
788
|
+
}, {
|
|
789
|
+
detail: string;
|
|
790
|
+
evalId: string;
|
|
791
|
+
}>, "many">;
|
|
792
|
+
summary: z.ZodString;
|
|
793
|
+
signals: z.ZodArray<z.ZodEnum<["harness-path", "declared-executable", "sibling-arm", "vat-private-dir", "skill-content"]>, "many">;
|
|
794
|
+
findings: z.ZodArray<z.ZodObject<{
|
|
795
|
+
evalId: z.ZodString;
|
|
796
|
+
hits: z.ZodArray<z.ZodObject<{
|
|
797
|
+
kind: z.ZodEnum<["harness-path", "declared-executable", "sibling-arm", "vat-private-dir", "skill-content"]>;
|
|
798
|
+
match: z.ZodString;
|
|
799
|
+
excerpt: z.ZodString;
|
|
800
|
+
}, "strict", z.ZodTypeAny, {
|
|
801
|
+
match: string;
|
|
802
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
803
|
+
excerpt: string;
|
|
804
|
+
}, {
|
|
805
|
+
match: string;
|
|
806
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
807
|
+
excerpt: string;
|
|
808
|
+
}>, "many">;
|
|
809
|
+
}, "strict", z.ZodTypeAny, {
|
|
810
|
+
evalId: string;
|
|
811
|
+
hits: {
|
|
812
|
+
match: string;
|
|
813
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
814
|
+
excerpt: string;
|
|
815
|
+
}[];
|
|
816
|
+
}, {
|
|
817
|
+
evalId: string;
|
|
818
|
+
hits: {
|
|
819
|
+
match: string;
|
|
820
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
821
|
+
excerpt: string;
|
|
822
|
+
}[];
|
|
823
|
+
}>, "many">;
|
|
824
|
+
}, "strict", z.ZodTypeAny, {
|
|
825
|
+
summary: string;
|
|
826
|
+
contaminated: boolean;
|
|
827
|
+
degraded: {
|
|
828
|
+
detail: string;
|
|
829
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
830
|
+
evalId?: string | undefined;
|
|
831
|
+
}[];
|
|
832
|
+
comparable: boolean;
|
|
833
|
+
skew: {
|
|
834
|
+
evalId: string;
|
|
835
|
+
withTotal: number;
|
|
836
|
+
withoutTotal: number;
|
|
837
|
+
}[];
|
|
838
|
+
controlArmFailures: {
|
|
839
|
+
detail: string;
|
|
840
|
+
evalId: string;
|
|
841
|
+
}[];
|
|
842
|
+
signals: ("harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content")[];
|
|
843
|
+
findings: {
|
|
844
|
+
evalId: string;
|
|
845
|
+
hits: {
|
|
846
|
+
match: string;
|
|
847
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
848
|
+
excerpt: string;
|
|
849
|
+
}[];
|
|
850
|
+
}[];
|
|
851
|
+
}, {
|
|
852
|
+
summary: string;
|
|
853
|
+
contaminated: boolean;
|
|
854
|
+
degraded: {
|
|
855
|
+
detail: string;
|
|
856
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
857
|
+
evalId?: string | undefined;
|
|
858
|
+
}[];
|
|
859
|
+
comparable: boolean;
|
|
860
|
+
skew: {
|
|
861
|
+
evalId: string;
|
|
862
|
+
withTotal: number;
|
|
863
|
+
withoutTotal: number;
|
|
864
|
+
}[];
|
|
865
|
+
controlArmFailures: {
|
|
866
|
+
detail: string;
|
|
867
|
+
evalId: string;
|
|
868
|
+
}[];
|
|
869
|
+
signals: ("harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content")[];
|
|
870
|
+
findings: {
|
|
871
|
+
evalId: string;
|
|
872
|
+
hits: {
|
|
873
|
+
match: string;
|
|
874
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
875
|
+
excerpt: string;
|
|
876
|
+
}[];
|
|
877
|
+
}[];
|
|
878
|
+
}>;
|
|
879
|
+
baselineDelta: z.ZodEffects<z.ZodObject<{
|
|
880
|
+
with: z.ZodEffects<z.ZodObject<{
|
|
881
|
+
passed: z.ZodNumber;
|
|
882
|
+
total: z.ZodNumber;
|
|
883
|
+
}, "strict", z.ZodTypeAny, {
|
|
884
|
+
total: number;
|
|
885
|
+
passed: number;
|
|
886
|
+
}, {
|
|
887
|
+
total: number;
|
|
888
|
+
passed: number;
|
|
889
|
+
}>, {
|
|
890
|
+
total: number;
|
|
891
|
+
passed: number;
|
|
892
|
+
}, {
|
|
893
|
+
total: number;
|
|
894
|
+
passed: number;
|
|
895
|
+
}>;
|
|
896
|
+
without: z.ZodEffects<z.ZodObject<{
|
|
897
|
+
passed: z.ZodNumber;
|
|
898
|
+
total: z.ZodNumber;
|
|
899
|
+
}, "strict", z.ZodTypeAny, {
|
|
900
|
+
total: number;
|
|
901
|
+
passed: number;
|
|
902
|
+
}, {
|
|
903
|
+
total: number;
|
|
904
|
+
passed: number;
|
|
905
|
+
}>, {
|
|
906
|
+
total: number;
|
|
907
|
+
passed: number;
|
|
908
|
+
}, {
|
|
909
|
+
total: number;
|
|
910
|
+
passed: number;
|
|
911
|
+
}>;
|
|
912
|
+
/**
|
|
913
|
+
* Run-level lift, or `null` when ANY eval was incomparable.
|
|
914
|
+
*
|
|
915
|
+
* Withheld for the whole run rather than computed over the comparable subset: the
|
|
916
|
+
* arm totals above include every eval, so a partial-subset delta would not be the
|
|
917
|
+
* difference between the two numbers printed beside it — which is precisely the
|
|
918
|
+
* kind of "close enough" number that gets quoted without its caveat.
|
|
919
|
+
*/
|
|
920
|
+
delta: z.ZodNullable<z.ZodNumber>;
|
|
921
|
+
perEval: z.ZodArray<z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
922
|
+
evalId: z.ZodString;
|
|
923
|
+
withPassed: z.ZodNumber;
|
|
924
|
+
withTotal: z.ZodNumber;
|
|
925
|
+
withoutPassed: z.ZodNumber;
|
|
926
|
+
withoutTotal: z.ZodNumber;
|
|
927
|
+
/**
|
|
928
|
+
* `withPassed − withoutPassed`, or `null` when THIS eval's two arms were graded to
|
|
929
|
+
* different depths. Signed: a negative delta means the control outscored the
|
|
930
|
+
* treatment, which is a real observed outcome and is never clamped to zero.
|
|
931
|
+
*/
|
|
932
|
+
delta: z.ZodNullable<z.ZodNumber>;
|
|
933
|
+
}, "strict", z.ZodTypeAny, {
|
|
934
|
+
evalId: string;
|
|
935
|
+
withTotal: number;
|
|
936
|
+
withoutTotal: number;
|
|
937
|
+
withPassed: number;
|
|
938
|
+
withoutPassed: number;
|
|
939
|
+
delta: number | null;
|
|
940
|
+
}, {
|
|
941
|
+
evalId: string;
|
|
942
|
+
withTotal: number;
|
|
943
|
+
withoutTotal: number;
|
|
944
|
+
withPassed: number;
|
|
945
|
+
withoutPassed: number;
|
|
946
|
+
delta: number | null;
|
|
947
|
+
}>, {
|
|
948
|
+
evalId: string;
|
|
949
|
+
withTotal: number;
|
|
950
|
+
withoutTotal: number;
|
|
951
|
+
withPassed: number;
|
|
952
|
+
withoutPassed: number;
|
|
953
|
+
delta: number | null;
|
|
954
|
+
}, {
|
|
955
|
+
evalId: string;
|
|
956
|
+
withTotal: number;
|
|
957
|
+
withoutTotal: number;
|
|
958
|
+
withPassed: number;
|
|
959
|
+
withoutPassed: number;
|
|
960
|
+
delta: number | null;
|
|
961
|
+
}>, {
|
|
962
|
+
evalId: string;
|
|
963
|
+
withTotal: number;
|
|
964
|
+
withoutTotal: number;
|
|
965
|
+
withPassed: number;
|
|
966
|
+
withoutPassed: number;
|
|
967
|
+
delta: number | null;
|
|
968
|
+
}, {
|
|
969
|
+
evalId: string;
|
|
970
|
+
withTotal: number;
|
|
971
|
+
withoutTotal: number;
|
|
972
|
+
withPassed: number;
|
|
973
|
+
withoutPassed: number;
|
|
974
|
+
delta: number | null;
|
|
975
|
+
}>, "many">;
|
|
976
|
+
/**
|
|
977
|
+
* Control-arm evals that produced no grade at all, so there was nothing to
|
|
978
|
+
* subtract for them. Carried HERE as well as on `baselineIntegrity` — it is one
|
|
979
|
+
* derivation handed to two consumers (the same arrangement as `ArmEvalGrade`
|
|
980
|
+
* feeding both the parity check and this block), never two derivations — because
|
|
981
|
+
* a reader who opens `baselineDelta` and finds `delta: null` must be able to learn
|
|
982
|
+
* WHY from the block that withheld it, without first having to know the
|
|
983
|
+
* contamination vocabulary next door.
|
|
984
|
+
*/
|
|
985
|
+
controlArmFailures: z.ZodArray<z.ZodObject<{
|
|
986
|
+
evalId: z.ZodString;
|
|
987
|
+
detail: z.ZodString;
|
|
988
|
+
}, "strict", z.ZodTypeAny, {
|
|
989
|
+
detail: string;
|
|
990
|
+
evalId: string;
|
|
991
|
+
}, {
|
|
992
|
+
detail: string;
|
|
993
|
+
evalId: string;
|
|
994
|
+
}>, "many">;
|
|
995
|
+
/**
|
|
996
|
+
* What the fail-fast gate cut off, or `null` when the whole declared suite ran.
|
|
997
|
+
*
|
|
998
|
+
* `null` rather than an absent key: "this delta covers everything" is a CLAIM, and
|
|
999
|
+
* the field has to be able to make it. An optional field would make a complete run
|
|
1000
|
+
* and a run written before this existed indistinguishable — the same argument that
|
|
1001
|
+
* makes `baselineIntegrity` unconditional.
|
|
1002
|
+
*/
|
|
1003
|
+
truncated: z.ZodNullable<z.ZodObject<{
|
|
1004
|
+
/** The (lower) tier whose gating failure stopped the run. */
|
|
1005
|
+
gatedByTier: z.ZodNumber;
|
|
1006
|
+
/** The lowest tier that never ran. */
|
|
1007
|
+
firstSkippedTier: z.ZodNumber;
|
|
1008
|
+
/** How many evals were never run on EITHER arm. */
|
|
1009
|
+
totalSkipped: z.ZodNumber;
|
|
1010
|
+
/** Their ids, flattened across the skipped tiers in ascending tier order. */
|
|
1011
|
+
evalIds: z.ZodArray<z.ZodString, "many">;
|
|
1012
|
+
}, "strict", z.ZodTypeAny, {
|
|
1013
|
+
gatedByTier: number;
|
|
1014
|
+
firstSkippedTier: number;
|
|
1015
|
+
totalSkipped: number;
|
|
1016
|
+
evalIds: string[];
|
|
1017
|
+
}, {
|
|
1018
|
+
gatedByTier: number;
|
|
1019
|
+
firstSkippedTier: number;
|
|
1020
|
+
totalSkipped: number;
|
|
1021
|
+
evalIds: string[];
|
|
1022
|
+
}>>;
|
|
1023
|
+
}, "strict", z.ZodTypeAny, {
|
|
1024
|
+
with: {
|
|
1025
|
+
total: number;
|
|
1026
|
+
passed: number;
|
|
1027
|
+
};
|
|
1028
|
+
without: {
|
|
1029
|
+
total: number;
|
|
1030
|
+
passed: number;
|
|
1031
|
+
};
|
|
1032
|
+
controlArmFailures: {
|
|
1033
|
+
detail: string;
|
|
1034
|
+
evalId: string;
|
|
1035
|
+
}[];
|
|
1036
|
+
delta: number | null;
|
|
1037
|
+
perEval: {
|
|
1038
|
+
evalId: string;
|
|
1039
|
+
withTotal: number;
|
|
1040
|
+
withoutTotal: number;
|
|
1041
|
+
withPassed: number;
|
|
1042
|
+
withoutPassed: number;
|
|
1043
|
+
delta: number | null;
|
|
1044
|
+
}[];
|
|
1045
|
+
truncated: {
|
|
1046
|
+
gatedByTier: number;
|
|
1047
|
+
firstSkippedTier: number;
|
|
1048
|
+
totalSkipped: number;
|
|
1049
|
+
evalIds: string[];
|
|
1050
|
+
} | null;
|
|
1051
|
+
}, {
|
|
1052
|
+
with: {
|
|
1053
|
+
total: number;
|
|
1054
|
+
passed: number;
|
|
1055
|
+
};
|
|
1056
|
+
without: {
|
|
1057
|
+
total: number;
|
|
1058
|
+
passed: number;
|
|
1059
|
+
};
|
|
1060
|
+
controlArmFailures: {
|
|
1061
|
+
detail: string;
|
|
1062
|
+
evalId: string;
|
|
1063
|
+
}[];
|
|
1064
|
+
delta: number | null;
|
|
1065
|
+
perEval: {
|
|
1066
|
+
evalId: string;
|
|
1067
|
+
withTotal: number;
|
|
1068
|
+
withoutTotal: number;
|
|
1069
|
+
withPassed: number;
|
|
1070
|
+
withoutPassed: number;
|
|
1071
|
+
delta: number | null;
|
|
1072
|
+
}[];
|
|
1073
|
+
truncated: {
|
|
1074
|
+
gatedByTier: number;
|
|
1075
|
+
firstSkippedTier: number;
|
|
1076
|
+
totalSkipped: number;
|
|
1077
|
+
evalIds: string[];
|
|
1078
|
+
} | null;
|
|
1079
|
+
}>, {
|
|
1080
|
+
with: {
|
|
1081
|
+
total: number;
|
|
1082
|
+
passed: number;
|
|
1083
|
+
};
|
|
1084
|
+
without: {
|
|
1085
|
+
total: number;
|
|
1086
|
+
passed: number;
|
|
1087
|
+
};
|
|
1088
|
+
controlArmFailures: {
|
|
1089
|
+
detail: string;
|
|
1090
|
+
evalId: string;
|
|
1091
|
+
}[];
|
|
1092
|
+
delta: number | null;
|
|
1093
|
+
perEval: {
|
|
1094
|
+
evalId: string;
|
|
1095
|
+
withTotal: number;
|
|
1096
|
+
withoutTotal: number;
|
|
1097
|
+
withPassed: number;
|
|
1098
|
+
withoutPassed: number;
|
|
1099
|
+
delta: number | null;
|
|
1100
|
+
}[];
|
|
1101
|
+
truncated: {
|
|
1102
|
+
gatedByTier: number;
|
|
1103
|
+
firstSkippedTier: number;
|
|
1104
|
+
totalSkipped: number;
|
|
1105
|
+
evalIds: string[];
|
|
1106
|
+
} | null;
|
|
1107
|
+
}, {
|
|
1108
|
+
with: {
|
|
1109
|
+
total: number;
|
|
1110
|
+
passed: number;
|
|
1111
|
+
};
|
|
1112
|
+
without: {
|
|
1113
|
+
total: number;
|
|
1114
|
+
passed: number;
|
|
1115
|
+
};
|
|
1116
|
+
controlArmFailures: {
|
|
1117
|
+
detail: string;
|
|
1118
|
+
evalId: string;
|
|
1119
|
+
}[];
|
|
1120
|
+
delta: number | null;
|
|
1121
|
+
perEval: {
|
|
1122
|
+
evalId: string;
|
|
1123
|
+
withTotal: number;
|
|
1124
|
+
withoutTotal: number;
|
|
1125
|
+
withPassed: number;
|
|
1126
|
+
withoutPassed: number;
|
|
1127
|
+
delta: number | null;
|
|
1128
|
+
}[];
|
|
1129
|
+
truncated: {
|
|
1130
|
+
gatedByTier: number;
|
|
1131
|
+
firstSkippedTier: number;
|
|
1132
|
+
totalSkipped: number;
|
|
1133
|
+
evalIds: string[];
|
|
1134
|
+
} | null;
|
|
1135
|
+
}>;
|
|
1136
|
+
}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
|
|
1137
|
+
baselineIntegrity: z.ZodObject<{
|
|
1138
|
+
contaminated: z.ZodBoolean;
|
|
1139
|
+
degraded: z.ZodArray<z.ZodObject<{
|
|
1140
|
+
reason: z.ZodEnum<["transcript-unparsed", "cwd-unknown", "cwd-untracked", "transcript-malformed", "glob-unexpanded"]>;
|
|
1141
|
+
detail: z.ZodString;
|
|
1142
|
+
evalId: z.ZodOptional<z.ZodString>;
|
|
1143
|
+
}, "strict", z.ZodTypeAny, {
|
|
1144
|
+
detail: string;
|
|
1145
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
1146
|
+
evalId?: string | undefined;
|
|
1147
|
+
}, {
|
|
1148
|
+
detail: string;
|
|
1149
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
1150
|
+
evalId?: string | undefined;
|
|
1151
|
+
}>, "many">;
|
|
1152
|
+
comparable: z.ZodBoolean;
|
|
1153
|
+
skew: z.ZodArray<z.ZodObject<{
|
|
1154
|
+
evalId: z.ZodString;
|
|
1155
|
+
withTotal: z.ZodNumber;
|
|
1156
|
+
withoutTotal: z.ZodNumber;
|
|
1157
|
+
}, "strict", z.ZodTypeAny, {
|
|
1158
|
+
evalId: string;
|
|
1159
|
+
withTotal: number;
|
|
1160
|
+
withoutTotal: number;
|
|
1161
|
+
}, {
|
|
1162
|
+
evalId: string;
|
|
1163
|
+
withTotal: number;
|
|
1164
|
+
withoutTotal: number;
|
|
1165
|
+
}>, "many">;
|
|
1166
|
+
controlArmFailures: z.ZodArray<z.ZodObject<{
|
|
1167
|
+
evalId: z.ZodString;
|
|
1168
|
+
detail: z.ZodString;
|
|
1169
|
+
}, "strict", z.ZodTypeAny, {
|
|
1170
|
+
detail: string;
|
|
1171
|
+
evalId: string;
|
|
1172
|
+
}, {
|
|
1173
|
+
detail: string;
|
|
1174
|
+
evalId: string;
|
|
1175
|
+
}>, "many">;
|
|
1176
|
+
summary: z.ZodString;
|
|
1177
|
+
signals: z.ZodArray<z.ZodEnum<["harness-path", "declared-executable", "sibling-arm", "vat-private-dir", "skill-content"]>, "many">;
|
|
1178
|
+
findings: z.ZodArray<z.ZodObject<{
|
|
1179
|
+
evalId: z.ZodString;
|
|
1180
|
+
hits: z.ZodArray<z.ZodObject<{
|
|
1181
|
+
kind: z.ZodEnum<["harness-path", "declared-executable", "sibling-arm", "vat-private-dir", "skill-content"]>;
|
|
1182
|
+
match: z.ZodString;
|
|
1183
|
+
excerpt: z.ZodString;
|
|
1184
|
+
}, "strict", z.ZodTypeAny, {
|
|
1185
|
+
match: string;
|
|
1186
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
1187
|
+
excerpt: string;
|
|
1188
|
+
}, {
|
|
1189
|
+
match: string;
|
|
1190
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
1191
|
+
excerpt: string;
|
|
1192
|
+
}>, "many">;
|
|
1193
|
+
}, "strict", z.ZodTypeAny, {
|
|
1194
|
+
evalId: string;
|
|
1195
|
+
hits: {
|
|
1196
|
+
match: string;
|
|
1197
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
1198
|
+
excerpt: string;
|
|
1199
|
+
}[];
|
|
1200
|
+
}, {
|
|
1201
|
+
evalId: string;
|
|
1202
|
+
hits: {
|
|
1203
|
+
match: string;
|
|
1204
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
1205
|
+
excerpt: string;
|
|
1206
|
+
}[];
|
|
1207
|
+
}>, "many">;
|
|
1208
|
+
}, "strict", z.ZodTypeAny, {
|
|
1209
|
+
summary: string;
|
|
1210
|
+
contaminated: boolean;
|
|
1211
|
+
degraded: {
|
|
1212
|
+
detail: string;
|
|
1213
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
1214
|
+
evalId?: string | undefined;
|
|
1215
|
+
}[];
|
|
1216
|
+
comparable: boolean;
|
|
1217
|
+
skew: {
|
|
1218
|
+
evalId: string;
|
|
1219
|
+
withTotal: number;
|
|
1220
|
+
withoutTotal: number;
|
|
1221
|
+
}[];
|
|
1222
|
+
controlArmFailures: {
|
|
1223
|
+
detail: string;
|
|
1224
|
+
evalId: string;
|
|
1225
|
+
}[];
|
|
1226
|
+
signals: ("harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content")[];
|
|
1227
|
+
findings: {
|
|
1228
|
+
evalId: string;
|
|
1229
|
+
hits: {
|
|
1230
|
+
match: string;
|
|
1231
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
1232
|
+
excerpt: string;
|
|
1233
|
+
}[];
|
|
1234
|
+
}[];
|
|
1235
|
+
}, {
|
|
1236
|
+
summary: string;
|
|
1237
|
+
contaminated: boolean;
|
|
1238
|
+
degraded: {
|
|
1239
|
+
detail: string;
|
|
1240
|
+
reason: "transcript-unparsed" | "cwd-unknown" | "cwd-untracked" | "transcript-malformed" | "glob-unexpanded";
|
|
1241
|
+
evalId?: string | undefined;
|
|
1242
|
+
}[];
|
|
1243
|
+
comparable: boolean;
|
|
1244
|
+
skew: {
|
|
1245
|
+
evalId: string;
|
|
1246
|
+
withTotal: number;
|
|
1247
|
+
withoutTotal: number;
|
|
1248
|
+
}[];
|
|
1249
|
+
controlArmFailures: {
|
|
1250
|
+
detail: string;
|
|
1251
|
+
evalId: string;
|
|
1252
|
+
}[];
|
|
1253
|
+
signals: ("harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content")[];
|
|
1254
|
+
findings: {
|
|
1255
|
+
evalId: string;
|
|
1256
|
+
hits: {
|
|
1257
|
+
match: string;
|
|
1258
|
+
kind: "harness-path" | "declared-executable" | "sibling-arm" | "vat-private-dir" | "skill-content";
|
|
1259
|
+
excerpt: string;
|
|
1260
|
+
}[];
|
|
1261
|
+
}[];
|
|
1262
|
+
}>;
|
|
1263
|
+
baselineDelta: z.ZodEffects<z.ZodObject<{
|
|
1264
|
+
with: z.ZodEffects<z.ZodObject<{
|
|
1265
|
+
passed: z.ZodNumber;
|
|
1266
|
+
total: z.ZodNumber;
|
|
1267
|
+
}, "strict", z.ZodTypeAny, {
|
|
1268
|
+
total: number;
|
|
1269
|
+
passed: number;
|
|
1270
|
+
}, {
|
|
1271
|
+
total: number;
|
|
1272
|
+
passed: number;
|
|
1273
|
+
}>, {
|
|
1274
|
+
total: number;
|
|
1275
|
+
passed: number;
|
|
1276
|
+
}, {
|
|
1277
|
+
total: number;
|
|
1278
|
+
passed: number;
|
|
1279
|
+
}>;
|
|
1280
|
+
without: z.ZodEffects<z.ZodObject<{
|
|
1281
|
+
passed: z.ZodNumber;
|
|
1282
|
+
total: z.ZodNumber;
|
|
1283
|
+
}, "strict", z.ZodTypeAny, {
|
|
1284
|
+
total: number;
|
|
1285
|
+
passed: number;
|
|
1286
|
+
}, {
|
|
1287
|
+
total: number;
|
|
1288
|
+
passed: number;
|
|
1289
|
+
}>, {
|
|
1290
|
+
total: number;
|
|
1291
|
+
passed: number;
|
|
1292
|
+
}, {
|
|
1293
|
+
total: number;
|
|
1294
|
+
passed: number;
|
|
1295
|
+
}>;
|
|
1296
|
+
/**
|
|
1297
|
+
* Run-level lift, or `null` when ANY eval was incomparable.
|
|
1298
|
+
*
|
|
1299
|
+
* Withheld for the whole run rather than computed over the comparable subset: the
|
|
1300
|
+
* arm totals above include every eval, so a partial-subset delta would not be the
|
|
1301
|
+
* difference between the two numbers printed beside it — which is precisely the
|
|
1302
|
+
* kind of "close enough" number that gets quoted without its caveat.
|
|
1303
|
+
*/
|
|
1304
|
+
delta: z.ZodNullable<z.ZodNumber>;
|
|
1305
|
+
perEval: z.ZodArray<z.ZodEffects<z.ZodEffects<z.ZodObject<{
|
|
1306
|
+
evalId: z.ZodString;
|
|
1307
|
+
withPassed: z.ZodNumber;
|
|
1308
|
+
withTotal: z.ZodNumber;
|
|
1309
|
+
withoutPassed: z.ZodNumber;
|
|
1310
|
+
withoutTotal: z.ZodNumber;
|
|
1311
|
+
/**
|
|
1312
|
+
* `withPassed − withoutPassed`, or `null` when THIS eval's two arms were graded to
|
|
1313
|
+
* different depths. Signed: a negative delta means the control outscored the
|
|
1314
|
+
* treatment, which is a real observed outcome and is never clamped to zero.
|
|
1315
|
+
*/
|
|
1316
|
+
delta: z.ZodNullable<z.ZodNumber>;
|
|
1317
|
+
}, "strict", z.ZodTypeAny, {
|
|
1318
|
+
evalId: string;
|
|
1319
|
+
withTotal: number;
|
|
1320
|
+
withoutTotal: number;
|
|
1321
|
+
withPassed: number;
|
|
1322
|
+
withoutPassed: number;
|
|
1323
|
+
delta: number | null;
|
|
1324
|
+
}, {
|
|
1325
|
+
evalId: string;
|
|
1326
|
+
withTotal: number;
|
|
1327
|
+
withoutTotal: number;
|
|
1328
|
+
withPassed: number;
|
|
1329
|
+
withoutPassed: number;
|
|
1330
|
+
delta: number | null;
|
|
1331
|
+
}>, {
|
|
1332
|
+
evalId: string;
|
|
1333
|
+
withTotal: number;
|
|
1334
|
+
withoutTotal: number;
|
|
1335
|
+
withPassed: number;
|
|
1336
|
+
withoutPassed: number;
|
|
1337
|
+
delta: number | null;
|
|
1338
|
+
}, {
|
|
1339
|
+
evalId: string;
|
|
1340
|
+
withTotal: number;
|
|
1341
|
+
withoutTotal: number;
|
|
1342
|
+
withPassed: number;
|
|
1343
|
+
withoutPassed: number;
|
|
1344
|
+
delta: number | null;
|
|
1345
|
+
}>, {
|
|
1346
|
+
evalId: string;
|
|
1347
|
+
withTotal: number;
|
|
1348
|
+
withoutTotal: number;
|
|
1349
|
+
withPassed: number;
|
|
1350
|
+
withoutPassed: number;
|
|
1351
|
+
delta: number | null;
|
|
1352
|
+
}, {
|
|
1353
|
+
evalId: string;
|
|
1354
|
+
withTotal: number;
|
|
1355
|
+
withoutTotal: number;
|
|
1356
|
+
withPassed: number;
|
|
1357
|
+
withoutPassed: number;
|
|
1358
|
+
delta: number | null;
|
|
1359
|
+
}>, "many">;
|
|
1360
|
+
/**
|
|
1361
|
+
* Control-arm evals that produced no grade at all, so there was nothing to
|
|
1362
|
+
* subtract for them. Carried HERE as well as on `baselineIntegrity` — it is one
|
|
1363
|
+
* derivation handed to two consumers (the same arrangement as `ArmEvalGrade`
|
|
1364
|
+
* feeding both the parity check and this block), never two derivations — because
|
|
1365
|
+
* a reader who opens `baselineDelta` and finds `delta: null` must be able to learn
|
|
1366
|
+
* WHY from the block that withheld it, without first having to know the
|
|
1367
|
+
* contamination vocabulary next door.
|
|
1368
|
+
*/
|
|
1369
|
+
controlArmFailures: z.ZodArray<z.ZodObject<{
|
|
1370
|
+
evalId: z.ZodString;
|
|
1371
|
+
detail: z.ZodString;
|
|
1372
|
+
}, "strict", z.ZodTypeAny, {
|
|
1373
|
+
detail: string;
|
|
1374
|
+
evalId: string;
|
|
1375
|
+
}, {
|
|
1376
|
+
detail: string;
|
|
1377
|
+
evalId: string;
|
|
1378
|
+
}>, "many">;
|
|
1379
|
+
/**
|
|
1380
|
+
* What the fail-fast gate cut off, or `null` when the whole declared suite ran.
|
|
1381
|
+
*
|
|
1382
|
+
* `null` rather than an absent key: "this delta covers everything" is a CLAIM, and
|
|
1383
|
+
* the field has to be able to make it. An optional field would make a complete run
|
|
1384
|
+
* and a run written before this existed indistinguishable — the same argument that
|
|
1385
|
+
* makes `baselineIntegrity` unconditional.
|
|
1386
|
+
*/
|
|
1387
|
+
truncated: z.ZodNullable<z.ZodObject<{
|
|
1388
|
+
/** The (lower) tier whose gating failure stopped the run. */
|
|
1389
|
+
gatedByTier: z.ZodNumber;
|
|
1390
|
+
/** The lowest tier that never ran. */
|
|
1391
|
+
firstSkippedTier: z.ZodNumber;
|
|
1392
|
+
/** How many evals were never run on EITHER arm. */
|
|
1393
|
+
totalSkipped: z.ZodNumber;
|
|
1394
|
+
/** Their ids, flattened across the skipped tiers in ascending tier order. */
|
|
1395
|
+
evalIds: z.ZodArray<z.ZodString, "many">;
|
|
1396
|
+
}, "strict", z.ZodTypeAny, {
|
|
1397
|
+
gatedByTier: number;
|
|
1398
|
+
firstSkippedTier: number;
|
|
1399
|
+
totalSkipped: number;
|
|
1400
|
+
evalIds: string[];
|
|
1401
|
+
}, {
|
|
1402
|
+
gatedByTier: number;
|
|
1403
|
+
firstSkippedTier: number;
|
|
1404
|
+
totalSkipped: number;
|
|
1405
|
+
evalIds: string[];
|
|
1406
|
+
}>>;
|
|
1407
|
+
}, "strict", z.ZodTypeAny, {
|
|
1408
|
+
with: {
|
|
1409
|
+
total: number;
|
|
1410
|
+
passed: number;
|
|
1411
|
+
};
|
|
1412
|
+
without: {
|
|
1413
|
+
total: number;
|
|
1414
|
+
passed: number;
|
|
1415
|
+
};
|
|
1416
|
+
controlArmFailures: {
|
|
1417
|
+
detail: string;
|
|
1418
|
+
evalId: string;
|
|
1419
|
+
}[];
|
|
1420
|
+
delta: number | null;
|
|
1421
|
+
perEval: {
|
|
1422
|
+
evalId: string;
|
|
1423
|
+
withTotal: number;
|
|
1424
|
+
withoutTotal: number;
|
|
1425
|
+
withPassed: number;
|
|
1426
|
+
withoutPassed: number;
|
|
1427
|
+
delta: number | null;
|
|
1428
|
+
}[];
|
|
1429
|
+
truncated: {
|
|
1430
|
+
gatedByTier: number;
|
|
1431
|
+
firstSkippedTier: number;
|
|
1432
|
+
totalSkipped: number;
|
|
1433
|
+
evalIds: string[];
|
|
1434
|
+
} | null;
|
|
1435
|
+
}, {
|
|
1436
|
+
with: {
|
|
1437
|
+
total: number;
|
|
1438
|
+
passed: number;
|
|
1439
|
+
};
|
|
1440
|
+
without: {
|
|
1441
|
+
total: number;
|
|
1442
|
+
passed: number;
|
|
1443
|
+
};
|
|
1444
|
+
controlArmFailures: {
|
|
1445
|
+
detail: string;
|
|
1446
|
+
evalId: string;
|
|
1447
|
+
}[];
|
|
1448
|
+
delta: number | null;
|
|
1449
|
+
perEval: {
|
|
1450
|
+
evalId: string;
|
|
1451
|
+
withTotal: number;
|
|
1452
|
+
withoutTotal: number;
|
|
1453
|
+
withPassed: number;
|
|
1454
|
+
withoutPassed: number;
|
|
1455
|
+
delta: number | null;
|
|
1456
|
+
}[];
|
|
1457
|
+
truncated: {
|
|
1458
|
+
gatedByTier: number;
|
|
1459
|
+
firstSkippedTier: number;
|
|
1460
|
+
totalSkipped: number;
|
|
1461
|
+
evalIds: string[];
|
|
1462
|
+
} | null;
|
|
1463
|
+
}>, {
|
|
1464
|
+
with: {
|
|
1465
|
+
total: number;
|
|
1466
|
+
passed: number;
|
|
1467
|
+
};
|
|
1468
|
+
without: {
|
|
1469
|
+
total: number;
|
|
1470
|
+
passed: number;
|
|
1471
|
+
};
|
|
1472
|
+
controlArmFailures: {
|
|
1473
|
+
detail: string;
|
|
1474
|
+
evalId: string;
|
|
1475
|
+
}[];
|
|
1476
|
+
delta: number | null;
|
|
1477
|
+
perEval: {
|
|
1478
|
+
evalId: string;
|
|
1479
|
+
withTotal: number;
|
|
1480
|
+
withoutTotal: number;
|
|
1481
|
+
withPassed: number;
|
|
1482
|
+
withoutPassed: number;
|
|
1483
|
+
delta: number | null;
|
|
1484
|
+
}[];
|
|
1485
|
+
truncated: {
|
|
1486
|
+
gatedByTier: number;
|
|
1487
|
+
firstSkippedTier: number;
|
|
1488
|
+
totalSkipped: number;
|
|
1489
|
+
evalIds: string[];
|
|
1490
|
+
} | null;
|
|
1491
|
+
}, {
|
|
1492
|
+
with: {
|
|
1493
|
+
total: number;
|
|
1494
|
+
passed: number;
|
|
1495
|
+
};
|
|
1496
|
+
without: {
|
|
1497
|
+
total: number;
|
|
1498
|
+
passed: number;
|
|
1499
|
+
};
|
|
1500
|
+
controlArmFailures: {
|
|
1501
|
+
detail: string;
|
|
1502
|
+
evalId: string;
|
|
1503
|
+
}[];
|
|
1504
|
+
delta: number | null;
|
|
1505
|
+
perEval: {
|
|
1506
|
+
evalId: string;
|
|
1507
|
+
withTotal: number;
|
|
1508
|
+
withoutTotal: number;
|
|
1509
|
+
withPassed: number;
|
|
1510
|
+
withoutPassed: number;
|
|
1511
|
+
delta: number | null;
|
|
1512
|
+
}[];
|
|
1513
|
+
truncated: {
|
|
1514
|
+
gatedByTier: number;
|
|
1515
|
+
firstSkippedTier: number;
|
|
1516
|
+
totalSkipped: number;
|
|
1517
|
+
evalIds: string[];
|
|
1518
|
+
} | null;
|
|
1519
|
+
}>;
|
|
1520
|
+
}, z.ZodTypeAny, "passthrough">>;
|
|
1521
|
+
/** Everything the run-level delta block is computed from. */
|
|
1522
|
+
export interface ComputeBaselineDeltaInput {
|
|
1523
|
+
/** The treatment arm's per-eval grades. */
|
|
1524
|
+
withArm: readonly ArmEvalGrade[];
|
|
1525
|
+
/** The control arm's per-eval grades — evals whose control arm died are simply absent. */
|
|
1526
|
+
withoutArm: readonly ArmEvalGrade[];
|
|
1527
|
+
/** The SINGLE authority on which evals are incomparable. See below. */
|
|
1528
|
+
skew: readonly BaselineArmSkew[];
|
|
1529
|
+
/** Control-arm evals that never graded, for the block and the printed reason. */
|
|
1530
|
+
controlArmFailures: readonly BaselineControlArmFailure[];
|
|
1531
|
+
/** What fail-fast cut off, or `null` when the whole suite ran. */
|
|
1532
|
+
truncated: DeltaTruncation | null;
|
|
1533
|
+
}
|
|
1534
|
+
/**
|
|
1535
|
+
* Compute the A/B delta from the two arms' per-eval grades.
|
|
1536
|
+
*
|
|
1537
|
+
* `skew` is the AUTHORITY on which evals are incomparable — it is not re-derived
|
|
1538
|
+
* from the counts here, and `controlArmFailures` does NOT get a second vote. Two
|
|
1539
|
+
* derivations of the same judgement is how the delta block and
|
|
1540
|
+
* `baselineIntegrity.skew` would end up disagreeing about one run, and a reader has
|
|
1541
|
+
* no way to tell which of two contradicting blocks to believe. A dead control arm
|
|
1542
|
+
* needs no second vote in any case: it leaves that eval graded on one arm only,
|
|
1543
|
+
* which {@link armExpectationSkew} already reports as skew. Callers pass the output
|
|
1544
|
+
* of {@link armExpectationSkew} over the SAME `ArmEvalGrade[]` values they pass here.
|
|
1545
|
+
*
|
|
1546
|
+
* `perEval` ordering matches {@link armExpectationSkew}: the treatment arm in its own
|
|
1547
|
+
* order, then the evals only the control arm graded. An eval present on one arm only
|
|
1548
|
+
* appears with the missing side reported as 0 and `delta: null`, exactly as the skew
|
|
1549
|
+
* check reports it — a silently absent eval skews the run total the same way a
|
|
1550
|
+
* short-graded one does, so it is surfaced rather than dropped.
|
|
1551
|
+
*
|
|
1552
|
+
* `truncated` does NOT withhold the delta. The tiers that ran, ran on both arms, so
|
|
1553
|
+
* their subtraction is legal; what truncation changes is the SCOPE of the claim, and
|
|
1554
|
+
* a scope caveat is reported (here and in the printed line), not a `null`. Spending
|
|
1555
|
+
* `null` — which means "these arms cannot be subtracted" — on a run where they can be
|
|
1556
|
+
* would collapse two failures with different remedies into one value.
|
|
1557
|
+
*
|
|
1558
|
+
* Pure.
|
|
1559
|
+
*/
|
|
1560
|
+
export declare function computeBaselineDelta(input: ComputeBaselineDeltaInput): BaselineDelta;
|
|
1561
|
+
/**
|
|
1562
|
+
* The one line an operator actually reads about the delta.
|
|
1563
|
+
*
|
|
1564
|
+
* `baseline.json` holds the per-eval detail, and nobody opens it unprompted — the
|
|
1565
|
+
* two rounds of integrity work before this one both ended up shipping findings that
|
|
1566
|
+
* existed only inside an artifact. So this line has to be self-sufficient in BOTH
|
|
1567
|
+
* states: it names each arm's passed/total (so the lift can be sanity-checked
|
|
1568
|
+
* against the numbers it came from), it says what the numbers do NOT cover when the
|
|
1569
|
+
* run was truncated, and when the delta is withheld it says so, says WHICH ARM broke
|
|
1570
|
+
* and how, and points at the block holding the evidence.
|
|
1571
|
+
*
|
|
1572
|
+
* Pure, and deliberately not wired here — the caller decides where it goes.
|
|
1573
|
+
*/
|
|
1574
|
+
export declare function formatBaselineDeltaLine(delta: BaselineDelta): string;
|
|
1575
|
+
//# sourceMappingURL=baseline-delta.d.ts.map
|