eval-quality 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +142 -67
- package/corpus/dev/README.md +17 -11
- package/corpus/dev/compile-seal-example/brief.json +1 -1
- package/corpus/dev/compile-seal-example/contract.json +1 -1
- package/corpus/dev/contracts/absent-collection-locations.json +1 -1
- package/corpus/dev/contracts/absent-sibling-groups.json +1 -1
- package/corpus/dev/contracts/absent-success-indicator.json +1 -1
- package/corpus/dev/contracts/empty-channel-roles.json +1 -1
- package/corpus/dev/contracts/empty-collection-locations.json +1 -1
- package/corpus/dev/contracts/empty-request-shapes.json +1 -1
- package/corpus/dev/contracts/empty-sibling-groups.json +1 -1
- package/corpus/dev/contracts/no-collection-quantifier.json +1 -1
- package/corpus/dev/contracts/no-operation-inventory.json +1 -1
- package/corpus/dev/contracts/no-read-back-relation.json +1 -1
- package/corpus/dev/contracts/no-state-change-marker.json +1 -1
- package/corpus/dev/contracts/no-type-violating-step.json +1 -1
- package/corpus/dev/contracts/per-key-split-oracles.json +1 -1
- package/corpus/dev/contracts/satisfied-declarations.json +1 -1
- package/corpus/dev/contracts/single-required-response-key.json +1 -1
- package/corpus/dev/contracts/split-indicator-oracle.json +1 -1
- package/corpus/dev/contracts/unaddressed-parameter-sibling.json +1 -1
- package/corpus/dev/contracts/unnamed-reference-set.json +1 -1
- package/corpus/dev/contracts/wrong-cardinality-form.json +1 -1
- package/corpus/dev/index.json +1 -1
- package/dist/application/index.d.ts +2 -0
- package/dist/application/index.js +1 -0
- package/dist/application/score.d.ts +48 -0
- package/dist/application/score.js +186 -0
- package/dist/cli/arguments.d.ts +7 -3
- package/dist/cli/arguments.js +73 -5
- package/dist/cli/exit-codes.d.ts +6 -2
- package/dist/cli/exit-codes.js +8 -9
- package/dist/cli/main.js +9 -0
- package/dist/cli/render.d.ts +1 -1
- package/dist/cli/render.js +6 -4
- package/dist/cli/run.d.ts +13 -3
- package/dist/cli/run.js +136 -16
- package/dist/core/canonical/canonicalize.d.ts +12 -0
- package/dist/core/canonical/canonicalize.js +18 -4
- package/dist/core/compile/bindings.d.ts +70 -0
- package/dist/core/compile/bindings.js +317 -0
- package/dist/core/compile/compile.d.ts +5 -0
- package/dist/core/compile/compile.js +9 -0
- package/dist/core/compile/expression-legality.d.ts +49 -0
- package/dist/core/compile/expression-legality.js +105 -31
- package/dist/core/compile/forbidden-inputs.d.ts +8 -1
- package/dist/core/compile/forbidden-inputs.js +16 -3
- package/dist/core/compile/interface-inventory.d.ts +23 -1
- package/dist/core/compile/interface-inventory.js +32 -3
- package/dist/core/compile/reachability.d.ts +18 -1
- package/dist/core/compile/reachability.js +46 -1
- package/dist/core/compile/sensitivity-witness.d.ts +1 -1
- package/dist/core/compile/sensitivity-witness.js +1 -1
- package/dist/core/emit/emit.d.ts +9 -0
- package/dist/core/emit/emit.js +214 -0
- package/dist/core/emit/private-artifact-digest.d.ts +11 -0
- package/dist/core/emit/private-artifact-digest.js +31 -0
- package/dist/core/evaluate/evidence-resolution.d.ts +11 -1
- package/dist/core/evaluate/evidence-resolution.js +4 -1
- package/dist/core/failure-codes.d.ts +2 -2
- package/dist/core/failure-codes.js +4 -2
- package/dist/core/ingest/conditions.d.ts +201 -0
- package/dist/core/ingest/conditions.js +48 -0
- package/dist/core/ingest/index.d.ts +13 -0
- package/dist/core/ingest/index.js +11 -0
- package/dist/core/ingest/ingest.d.ts +68 -0
- package/dist/core/ingest/ingest.js +310 -0
- package/dist/core/lineage/stage-table.d.ts +21 -4
- package/dist/core/lineage/stage-table.js +35 -3
- package/dist/core/preflight/witness-evidence.js +10 -0
- package/dist/core/schemas/artifact.d.ts +385 -14
- package/dist/core/schemas/constraint-ledger.js +40 -0
- package/dist/core/schemas/defect-signature.d.ts +273 -0
- package/dist/core/schemas/defect-signature.js +127 -0
- package/dist/core/schemas/eval-contract.d.ts +33 -0
- package/dist/core/schemas/eval-contract.js +14 -0
- package/dist/core/schemas/evidence-artifact.d.ts +99 -19
- package/dist/core/schemas/evidence-artifact.js +64 -8
- package/dist/core/schemas/plan.d.ts +112 -0
- package/dist/core/schemas/plan.js +67 -2
- package/dist/core/schemas/probe-qualification.d.ts +168 -0
- package/dist/core/schemas/probe-qualification.js +106 -0
- package/dist/core/schemas/probe.d.ts +312 -0
- package/dist/core/schemas/probe.js +6 -2
- package/dist/core/schemas/scoring-policy.d.ts +1 -0
- package/dist/core/schemas/scoring-policy.js +6 -1
- package/dist/core/schemas/sealed-evaluator-brief.d.ts +1 -0
- package/dist/core/schemas/sealed-evaluator-brief.js +4 -1
- package/dist/core/schemas/sealed-run-record.d.ts +30 -3
- package/dist/core/schemas/sealed-run-record.js +37 -5
- package/dist/core/score/binding-order.d.ts +31 -0
- package/dist/core/score/binding-order.js +88 -0
- package/dist/core/score/bindings.d.ts +93 -0
- package/dist/core/score/bindings.js +297 -0
- package/dist/core/score/ladder-table.d.ts +23 -0
- package/dist/core/score/ladder-table.js +109 -0
- package/dist/core/score/ladder.d.ts +229 -0
- package/dist/core/score/ladder.js +464 -0
- package/dist/core/score/mode-agreement.d.ts +30 -0
- package/dist/core/score/mode-agreement.js +16 -0
- package/dist/core/score/outcome-table.d.ts +17 -0
- package/dist/core/score/outcome-table.js +172 -0
- package/dist/core/score/outcome.d.ts +466 -0
- package/dist/core/score/outcome.js +562 -0
- package/dist/core/score/qualification.d.ts +83 -0
- package/dist/core/score/qualification.js +615 -0
- package/dist/core/score/quotation.d.ts +79 -0
- package/dist/core/score/quotation.js +201 -0
- package/dist/core/score/reduce-trials.d.ts +80 -0
- package/dist/core/score/reduce-trials.js +90 -0
- package/dist/core/score/score.d.ts +72 -0
- package/dist/core/score/score.js +581 -0
- package/dist/core/score/selection.d.ts +80 -0
- package/dist/core/score/selection.js +98 -0
- package/dist/core/score/strength.d.ts +49 -0
- package/dist/core/score/strength.js +193 -0
- package/dist/core/score/witness.d.ts +117 -0
- package/dist/core/score/witness.js +315 -0
- package/dist/core/seal/derived-reference.d.ts +1 -1
- package/dist/core/seal/derived-reference.js +236 -28
- package/dist/core/seal/seal.js +8 -3
- package/dist/core/stage-contracts.d.ts +62 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/testing/index.d.ts +2 -0
- package/dist/testing/index.js +1 -0
- package/package.json +12 -3
- package/schemas/eval-contract.schema.json +100 -2
- package/schemas/evidence-artifact.schema.json +96 -9
- package/schemas/probe.schema.json +508 -3
- package/schemas/scoring-policy.schema.json +8 -1
- package/schemas/sealed-evaluator-brief.schema.json +11 -1
- package/schemas/sealed-run-record.schema.json +19 -2
|
@@ -72,6 +72,177 @@
|
|
|
72
72
|
"type": "string",
|
|
73
73
|
"minLength": 1
|
|
74
74
|
},
|
|
75
|
+
"qualification": {
|
|
76
|
+
"oneOf": [
|
|
77
|
+
{
|
|
78
|
+
"type": "object",
|
|
79
|
+
"properties": {
|
|
80
|
+
"route": {
|
|
81
|
+
"type": "string",
|
|
82
|
+
"const": "historical"
|
|
83
|
+
},
|
|
84
|
+
"failBeforeEvidence": {
|
|
85
|
+
"description": "The recorded failure at the parent revision. AD-9 exists because 2 of 18 mined fix commits had a test that already passed at the parent, so this is the half that cannot be assumed.",
|
|
86
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
87
|
+
},
|
|
88
|
+
"passAfterEvidence": {
|
|
89
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
90
|
+
},
|
|
91
|
+
"fixCommitDigest": {
|
|
92
|
+
"type": "string",
|
|
93
|
+
"pattern": "^sha256:[0-9a-f]{64}$",
|
|
94
|
+
"description": "The causally isolated fix boundary AD-9 requires the two evidence records to straddle."
|
|
95
|
+
},
|
|
96
|
+
"oracleStableAcrossRevisions": {
|
|
97
|
+
"type": "boolean",
|
|
98
|
+
"description": "AD-9 requires the oracle to be stable across both revisions. `false` parses and fails the gate: a route that cannot record its own unmet precondition cannot be audited, and the schema is not where AD-9's admission decision is made."
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
"required": [
|
|
102
|
+
"route",
|
|
103
|
+
"failBeforeEvidence",
|
|
104
|
+
"passAfterEvidence",
|
|
105
|
+
"fixCommitDigest",
|
|
106
|
+
"oracleStableAcrossRevisions"
|
|
107
|
+
],
|
|
108
|
+
"additionalProperties": false,
|
|
109
|
+
"description": "A historical probe: a defect mined from a real fix boundary, seeded by reverting it."
|
|
110
|
+
},
|
|
111
|
+
{
|
|
112
|
+
"type": "object",
|
|
113
|
+
"properties": {
|
|
114
|
+
"route": {
|
|
115
|
+
"type": "string",
|
|
116
|
+
"const": "controlled-mutation"
|
|
117
|
+
},
|
|
118
|
+
"mutationSource": {
|
|
119
|
+
"type": "string",
|
|
120
|
+
"minLength": 1,
|
|
121
|
+
"description": "Where the mutation came from, in the corpus author's own terms. An opaque caller string, unrelated to `Defect.source`'s two-member enum, which records whether the defect is natural or introduced."
|
|
122
|
+
},
|
|
123
|
+
"mutationOperator": {
|
|
124
|
+
"type": "string",
|
|
125
|
+
"minLength": 1
|
|
126
|
+
},
|
|
127
|
+
"targetArtifact": {
|
|
128
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
129
|
+
},
|
|
130
|
+
"expectedObservableFailure": {
|
|
131
|
+
"type": "string",
|
|
132
|
+
"minLength": 1
|
|
133
|
+
},
|
|
134
|
+
"baselinePassEvidence": {
|
|
135
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
136
|
+
},
|
|
137
|
+
"mutatedFailEvidence": {
|
|
138
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
139
|
+
},
|
|
140
|
+
"rollbackVerified": {
|
|
141
|
+
"type": "boolean",
|
|
142
|
+
"description": "AD-9's verified rollback or cleanup. `false` parses and fails the gate, for the same reason `oracleStableAcrossRevisions` does."
|
|
143
|
+
}
|
|
144
|
+
},
|
|
145
|
+
"required": [
|
|
146
|
+
"route",
|
|
147
|
+
"mutationSource",
|
|
148
|
+
"mutationOperator",
|
|
149
|
+
"targetArtifact",
|
|
150
|
+
"expectedObservableFailure",
|
|
151
|
+
"baselinePassEvidence",
|
|
152
|
+
"mutatedFailEvidence",
|
|
153
|
+
"rollbackVerified"
|
|
154
|
+
],
|
|
155
|
+
"additionalProperties": false,
|
|
156
|
+
"description": "A controlled mutation: a defect introduced by a named operator against a named artifact."
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
"type": "object",
|
|
160
|
+
"properties": {
|
|
161
|
+
"route": {
|
|
162
|
+
"type": "string",
|
|
163
|
+
"const": "gameability"
|
|
164
|
+
},
|
|
165
|
+
"degenerateResponse": {
|
|
166
|
+
"type": "string",
|
|
167
|
+
"minLength": 1,
|
|
168
|
+
"description": "The compliant-looking degenerate response, described rather than embedded: AD-8 forbids sealed-case content from entering any artifact."
|
|
169
|
+
},
|
|
170
|
+
"naiveOracleSatisfiedEvidence": {
|
|
171
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
172
|
+
},
|
|
173
|
+
"disciplinedOracleRejectedEvidence": {
|
|
174
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
"required": [
|
|
178
|
+
"route",
|
|
179
|
+
"degenerateResponse",
|
|
180
|
+
"naiveOracleSatisfiedEvidence",
|
|
181
|
+
"disciplinedOracleRejectedEvidence"
|
|
182
|
+
],
|
|
183
|
+
"additionalProperties": false,
|
|
184
|
+
"description": "A gameability probe: the demonstration that a degenerate response satisfies a naive oracle and is rejected by a disciplined one."
|
|
185
|
+
},
|
|
186
|
+
{
|
|
187
|
+
"type": "object",
|
|
188
|
+
"properties": {
|
|
189
|
+
"route": {
|
|
190
|
+
"type": "string",
|
|
191
|
+
"const": "canary"
|
|
192
|
+
},
|
|
193
|
+
"indicts": {
|
|
194
|
+
"type": "string",
|
|
195
|
+
"enum": [
|
|
196
|
+
"corpus",
|
|
197
|
+
"fixture"
|
|
198
|
+
],
|
|
199
|
+
"description": "AD-9's canary qualifies by demonstrating that non-detection indicts one of these two rather than the contract. Which one it indicts is the demonstration's own result and is recorded rather than left to the reader."
|
|
200
|
+
},
|
|
201
|
+
"nonDetectionEvidence": {
|
|
202
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
203
|
+
}
|
|
204
|
+
},
|
|
205
|
+
"required": [
|
|
206
|
+
"route",
|
|
207
|
+
"indicts",
|
|
208
|
+
"nonDetectionEvidence"
|
|
209
|
+
],
|
|
210
|
+
"additionalProperties": false,
|
|
211
|
+
"description": "A canary: a probe whose non-detection is evidence about the instrument, never about the contract."
|
|
212
|
+
},
|
|
213
|
+
{
|
|
214
|
+
"type": "object",
|
|
215
|
+
"properties": {
|
|
216
|
+
"route": {
|
|
217
|
+
"type": "string",
|
|
218
|
+
"const": "clean-control"
|
|
219
|
+
},
|
|
220
|
+
"baselinePassEvidence": {
|
|
221
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
222
|
+
},
|
|
223
|
+
"revisionCommitDigest": {
|
|
224
|
+
"type": "string",
|
|
225
|
+
"pattern": "^sha256:[0-9a-f]{64}$",
|
|
226
|
+
"description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
|
|
227
|
+
},
|
|
228
|
+
"noKnownDefectStatement": {
|
|
229
|
+
"type": "string",
|
|
230
|
+
"minLength": 1,
|
|
231
|
+
"description": "AD-9 qualifies a clean control at \"a revision with no known defect in the probed interface\". No field can prove an absence, so the statement is recorded as an attestation under AD-32 and read as trust rather than as a computed fact."
|
|
232
|
+
}
|
|
233
|
+
},
|
|
234
|
+
"required": [
|
|
235
|
+
"route",
|
|
236
|
+
"baselinePassEvidence",
|
|
237
|
+
"revisionCommitDigest",
|
|
238
|
+
"noKnownDefectStatement"
|
|
239
|
+
],
|
|
240
|
+
"additionalProperties": false,
|
|
241
|
+
"description": "A known-clean control: recorded baseline-pass evidence at a revision with no known defect in the probed interface."
|
|
242
|
+
}
|
|
243
|
+
],
|
|
244
|
+
"description": "AD-9's qualification record: which of the five routes earned this probe its ground truth, and the evidence that route demands. Required on every branch and on every class, canaries included, because AD-9 closes with \"an unqualified probe cannot enter a sealed set\" and spells a route for all five kinds. That the route is compatible with this probe's class and `expectedClean` flag is a cross-field rule the export cannot carry; the corpus qualification gate enforces it and returns a reason code. Required, not optional, which with `defectSignature` below makes the probe's `schemaVersion` 1 -> 2 BREAKING bump under AD-11, whose rule is that \"adding an optional field is a `schemaVersion` bump recorded in the field's own description; removing or retyping is breaking\". This field is on both branches, so it alone is what stops every version-1 probe from parsing."
|
|
245
|
+
},
|
|
75
246
|
"expectedClean": {
|
|
76
247
|
"type": "boolean",
|
|
77
248
|
"const": true
|
|
@@ -185,6 +356,7 @@
|
|
|
185
356
|
"artifactDigest",
|
|
186
357
|
"commitDigest",
|
|
187
358
|
"rationale",
|
|
359
|
+
"qualification",
|
|
188
360
|
"expectedClean",
|
|
189
361
|
"defects"
|
|
190
362
|
],
|
|
@@ -261,6 +433,177 @@
|
|
|
261
433
|
"type": "string",
|
|
262
434
|
"minLength": 1
|
|
263
435
|
},
|
|
436
|
+
"qualification": {
|
|
437
|
+
"oneOf": [
|
|
438
|
+
{
|
|
439
|
+
"type": "object",
|
|
440
|
+
"properties": {
|
|
441
|
+
"route": {
|
|
442
|
+
"type": "string",
|
|
443
|
+
"const": "historical"
|
|
444
|
+
},
|
|
445
|
+
"failBeforeEvidence": {
|
|
446
|
+
"description": "The recorded failure at the parent revision. AD-9 exists because 2 of 18 mined fix commits had a test that already passed at the parent, so this is the half that cannot be assumed.",
|
|
447
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
448
|
+
},
|
|
449
|
+
"passAfterEvidence": {
|
|
450
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
451
|
+
},
|
|
452
|
+
"fixCommitDigest": {
|
|
453
|
+
"type": "string",
|
|
454
|
+
"pattern": "^sha256:[0-9a-f]{64}$",
|
|
455
|
+
"description": "The causally isolated fix boundary AD-9 requires the two evidence records to straddle."
|
|
456
|
+
},
|
|
457
|
+
"oracleStableAcrossRevisions": {
|
|
458
|
+
"type": "boolean",
|
|
459
|
+
"description": "AD-9 requires the oracle to be stable across both revisions. `false` parses and fails the gate: a route that cannot record its own unmet precondition cannot be audited, and the schema is not where AD-9's admission decision is made."
|
|
460
|
+
}
|
|
461
|
+
},
|
|
462
|
+
"required": [
|
|
463
|
+
"route",
|
|
464
|
+
"failBeforeEvidence",
|
|
465
|
+
"passAfterEvidence",
|
|
466
|
+
"fixCommitDigest",
|
|
467
|
+
"oracleStableAcrossRevisions"
|
|
468
|
+
],
|
|
469
|
+
"additionalProperties": false,
|
|
470
|
+
"description": "A historical probe: a defect mined from a real fix boundary, seeded by reverting it."
|
|
471
|
+
},
|
|
472
|
+
{
|
|
473
|
+
"type": "object",
|
|
474
|
+
"properties": {
|
|
475
|
+
"route": {
|
|
476
|
+
"type": "string",
|
|
477
|
+
"const": "controlled-mutation"
|
|
478
|
+
},
|
|
479
|
+
"mutationSource": {
|
|
480
|
+
"type": "string",
|
|
481
|
+
"minLength": 1,
|
|
482
|
+
"description": "Where the mutation came from, in the corpus author's own terms. An opaque caller string, unrelated to `Defect.source`'s two-member enum, which records whether the defect is natural or introduced."
|
|
483
|
+
},
|
|
484
|
+
"mutationOperator": {
|
|
485
|
+
"type": "string",
|
|
486
|
+
"minLength": 1
|
|
487
|
+
},
|
|
488
|
+
"targetArtifact": {
|
|
489
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
490
|
+
},
|
|
491
|
+
"expectedObservableFailure": {
|
|
492
|
+
"type": "string",
|
|
493
|
+
"minLength": 1
|
|
494
|
+
},
|
|
495
|
+
"baselinePassEvidence": {
|
|
496
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
497
|
+
},
|
|
498
|
+
"mutatedFailEvidence": {
|
|
499
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
500
|
+
},
|
|
501
|
+
"rollbackVerified": {
|
|
502
|
+
"type": "boolean",
|
|
503
|
+
"description": "AD-9's verified rollback or cleanup. `false` parses and fails the gate, for the same reason `oracleStableAcrossRevisions` does."
|
|
504
|
+
}
|
|
505
|
+
},
|
|
506
|
+
"required": [
|
|
507
|
+
"route",
|
|
508
|
+
"mutationSource",
|
|
509
|
+
"mutationOperator",
|
|
510
|
+
"targetArtifact",
|
|
511
|
+
"expectedObservableFailure",
|
|
512
|
+
"baselinePassEvidence",
|
|
513
|
+
"mutatedFailEvidence",
|
|
514
|
+
"rollbackVerified"
|
|
515
|
+
],
|
|
516
|
+
"additionalProperties": false,
|
|
517
|
+
"description": "A controlled mutation: a defect introduced by a named operator against a named artifact."
|
|
518
|
+
},
|
|
519
|
+
{
|
|
520
|
+
"type": "object",
|
|
521
|
+
"properties": {
|
|
522
|
+
"route": {
|
|
523
|
+
"type": "string",
|
|
524
|
+
"const": "gameability"
|
|
525
|
+
},
|
|
526
|
+
"degenerateResponse": {
|
|
527
|
+
"type": "string",
|
|
528
|
+
"minLength": 1,
|
|
529
|
+
"description": "The compliant-looking degenerate response, described rather than embedded: AD-8 forbids sealed-case content from entering any artifact."
|
|
530
|
+
},
|
|
531
|
+
"naiveOracleSatisfiedEvidence": {
|
|
532
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
533
|
+
},
|
|
534
|
+
"disciplinedOracleRejectedEvidence": {
|
|
535
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
536
|
+
}
|
|
537
|
+
},
|
|
538
|
+
"required": [
|
|
539
|
+
"route",
|
|
540
|
+
"degenerateResponse",
|
|
541
|
+
"naiveOracleSatisfiedEvidence",
|
|
542
|
+
"disciplinedOracleRejectedEvidence"
|
|
543
|
+
],
|
|
544
|
+
"additionalProperties": false,
|
|
545
|
+
"description": "A gameability probe: the demonstration that a degenerate response satisfies a naive oracle and is rejected by a disciplined one."
|
|
546
|
+
},
|
|
547
|
+
{
|
|
548
|
+
"type": "object",
|
|
549
|
+
"properties": {
|
|
550
|
+
"route": {
|
|
551
|
+
"type": "string",
|
|
552
|
+
"const": "canary"
|
|
553
|
+
},
|
|
554
|
+
"indicts": {
|
|
555
|
+
"type": "string",
|
|
556
|
+
"enum": [
|
|
557
|
+
"corpus",
|
|
558
|
+
"fixture"
|
|
559
|
+
],
|
|
560
|
+
"description": "AD-9's canary qualifies by demonstrating that non-detection indicts one of these two rather than the contract. Which one it indicts is the demonstration's own result and is recorded rather than left to the reader."
|
|
561
|
+
},
|
|
562
|
+
"nonDetectionEvidence": {
|
|
563
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
564
|
+
}
|
|
565
|
+
},
|
|
566
|
+
"required": [
|
|
567
|
+
"route",
|
|
568
|
+
"indicts",
|
|
569
|
+
"nonDetectionEvidence"
|
|
570
|
+
],
|
|
571
|
+
"additionalProperties": false,
|
|
572
|
+
"description": "A canary: a probe whose non-detection is evidence about the instrument, never about the contract."
|
|
573
|
+
},
|
|
574
|
+
{
|
|
575
|
+
"type": "object",
|
|
576
|
+
"properties": {
|
|
577
|
+
"route": {
|
|
578
|
+
"type": "string",
|
|
579
|
+
"const": "clean-control"
|
|
580
|
+
},
|
|
581
|
+
"baselinePassEvidence": {
|
|
582
|
+
"$ref": "#/$defs/ArtifactReference"
|
|
583
|
+
},
|
|
584
|
+
"revisionCommitDigest": {
|
|
585
|
+
"type": "string",
|
|
586
|
+
"pattern": "^sha256:[0-9a-f]{64}$",
|
|
587
|
+
"description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
|
|
588
|
+
},
|
|
589
|
+
"noKnownDefectStatement": {
|
|
590
|
+
"type": "string",
|
|
591
|
+
"minLength": 1,
|
|
592
|
+
"description": "AD-9 qualifies a clean control at \"a revision with no known defect in the probed interface\". No field can prove an absence, so the statement is recorded as an attestation under AD-32 and read as trust rather than as a computed fact."
|
|
593
|
+
}
|
|
594
|
+
},
|
|
595
|
+
"required": [
|
|
596
|
+
"route",
|
|
597
|
+
"baselinePassEvidence",
|
|
598
|
+
"revisionCommitDigest",
|
|
599
|
+
"noKnownDefectStatement"
|
|
600
|
+
],
|
|
601
|
+
"additionalProperties": false,
|
|
602
|
+
"description": "A known-clean control: recorded baseline-pass evidence at a revision with no known defect in the probed interface."
|
|
603
|
+
}
|
|
604
|
+
],
|
|
605
|
+
"description": "AD-9's qualification record: which of the five routes earned this probe its ground truth, and the evidence that route demands. Required on every branch and on every class, canaries included, because AD-9 closes with \"an unqualified probe cannot enter a sealed set\" and spells a route for all five kinds. That the route is compatible with this probe's class and `expectedClean` flag is a cross-field rule the export cannot carry; the corpus qualification gate enforces it and returns a reason code. Required, not optional, which with `defectSignature` below makes the probe's `schemaVersion` 1 -> 2 BREAKING bump under AD-11, whose rule is that \"adding an optional field is a `schemaVersion` bump recorded in the field's own description; removing or retyping is breaking\". This field is on both branches, so it alone is what stops every version-1 probe from parsing."
|
|
606
|
+
},
|
|
264
607
|
"expectedClean": {
|
|
265
608
|
"type": "boolean",
|
|
266
609
|
"const": false
|
|
@@ -359,6 +702,105 @@
|
|
|
359
702
|
"additionalProperties": false
|
|
360
703
|
},
|
|
361
704
|
"description": "No minimum. AD-9 states none, and a minimum would make a canary unrepresentable, since a canary indicts the fixture rather than seeding a defect."
|
|
705
|
+
},
|
|
706
|
+
"defectSignature": {
|
|
707
|
+
"anyOf": [
|
|
708
|
+
{
|
|
709
|
+
"type": "object",
|
|
710
|
+
"properties": {
|
|
711
|
+
"interfaceKind": {
|
|
712
|
+
"type": "string",
|
|
713
|
+
"enum": [
|
|
714
|
+
"api",
|
|
715
|
+
"web",
|
|
716
|
+
"cli",
|
|
717
|
+
"mcp"
|
|
718
|
+
],
|
|
719
|
+
"description": "The interface the seeded defect lives behind. All four kinds parse so `unsupported-interface-kind` stays fireable contract-side, and the qualification gate rejects every kind but `api`: `Operation` carries no interface kind and requires a method and a path template with no per-kind variation, so a `cli` or `mcp` signature would declare a meaningless `POST /path`."
|
|
720
|
+
},
|
|
721
|
+
"method": {
|
|
722
|
+
"type": "string",
|
|
723
|
+
"enum": [
|
|
724
|
+
"GET",
|
|
725
|
+
"HEAD",
|
|
726
|
+
"POST",
|
|
727
|
+
"PUT",
|
|
728
|
+
"PATCH",
|
|
729
|
+
"DELETE",
|
|
730
|
+
"OPTIONS"
|
|
731
|
+
]
|
|
732
|
+
},
|
|
733
|
+
"pathTemplate": {
|
|
734
|
+
"type": "string",
|
|
735
|
+
"pattern": "^(?:\\/(?:[^/{}:]|\\{[A-Za-z0-9_-]+\\})*)+$",
|
|
736
|
+
"description": "A path template whose parameters are spelled `{name}` in braces. The `:name` spelling is rejected: AD-40 resolves a defect signature by comparing method and path template, and that comparison is not implementable against an unstated syntax."
|
|
737
|
+
},
|
|
738
|
+
"observableChannel": {
|
|
739
|
+
"description": "AD-26's channel the seeded defect manifests in. The qualification gate reads it: a condition passes only if its pointers name this channel, or name two channels with at least one response-side member. That rule exists to reject a condition collapsing to \"the evidence contains the string I sent\", which is satisfied by a finding that merely echoes its own input.",
|
|
740
|
+
"$ref": "#/$defs/EvidenceChannel"
|
|
741
|
+
},
|
|
742
|
+
"condition": {
|
|
743
|
+
"type": "object",
|
|
744
|
+
"properties": {
|
|
745
|
+
"selector": {
|
|
746
|
+
"type": "object",
|
|
747
|
+
"properties": {
|
|
748
|
+
"inputBinding": {
|
|
749
|
+
"type": "object",
|
|
750
|
+
"properties": {
|
|
751
|
+
"path": {
|
|
752
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
753
|
+
},
|
|
754
|
+
"query": {
|
|
755
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
756
|
+
},
|
|
757
|
+
"header": {
|
|
758
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
759
|
+
},
|
|
760
|
+
"body": {
|
|
761
|
+
"$ref": "#/$defs/ProbeInputBindingChannel"
|
|
762
|
+
}
|
|
763
|
+
},
|
|
764
|
+
"required": [
|
|
765
|
+
"path",
|
|
766
|
+
"query",
|
|
767
|
+
"header",
|
|
768
|
+
"body"
|
|
769
|
+
],
|
|
770
|
+
"additionalProperties": false
|
|
771
|
+
}
|
|
772
|
+
},
|
|
773
|
+
"required": [
|
|
774
|
+
"inputBinding"
|
|
775
|
+
],
|
|
776
|
+
"additionalProperties": false
|
|
777
|
+
},
|
|
778
|
+
"predicate": {
|
|
779
|
+
"description": "An AD-4 expression over AD-26 response channels, every pointer rooted at the reserved step identifier `observed`. Legality is the corpus qualification gate's: an unchecked predicate admits `existence` over a literal, which is true of every observation and makes the catch rate 1.00 by construction.",
|
|
780
|
+
"$ref": "#/$defs/Expression"
|
|
781
|
+
}
|
|
782
|
+
},
|
|
783
|
+
"required": [
|
|
784
|
+
"selector",
|
|
785
|
+
"predicate"
|
|
786
|
+
],
|
|
787
|
+
"additionalProperties": false
|
|
788
|
+
}
|
|
789
|
+
},
|
|
790
|
+
"required": [
|
|
791
|
+
"interfaceKind",
|
|
792
|
+
"method",
|
|
793
|
+
"pathTemplate",
|
|
794
|
+
"observableChannel",
|
|
795
|
+
"condition"
|
|
796
|
+
],
|
|
797
|
+
"additionalProperties": false
|
|
798
|
+
},
|
|
799
|
+
{
|
|
800
|
+
"type": "null"
|
|
801
|
+
}
|
|
802
|
+
],
|
|
803
|
+
"description": "AD-40's machine-readable defect signature, declared on this branch alone: a clean control seeds nothing, so its branch has no signature to carry. Nullable rather than plain, because the one class that legitimately carries none is `canary`, and a union-level refinement expressing that is dropped from the published export, which would leave Zod rejecting a probe ajv accepts. The corpus qualification gate is the enforcement point and the constraint ledger records the gap; the shipped precedent is `compile/interface-inventory.ts`, which put the principal check in the compiler for the same reason. The other half of the probe's `schemaVersion` 1 -> 2 BREAKING bump under AD-11, recorded here as that rule requires: the key is required on this branch, and `null` is the legal value a canary carries."
|
|
362
804
|
}
|
|
363
805
|
},
|
|
364
806
|
"required": [
|
|
@@ -373,14 +815,16 @@
|
|
|
373
815
|
"artifactDigest",
|
|
374
816
|
"commitDigest",
|
|
375
817
|
"rationale",
|
|
818
|
+
"qualification",
|
|
376
819
|
"expectedClean",
|
|
377
|
-
"defects"
|
|
820
|
+
"defects",
|
|
821
|
+
"defectSignature"
|
|
378
822
|
],
|
|
379
823
|
"additionalProperties": false,
|
|
380
|
-
"description": "A probe that is not a known-clean control."
|
|
824
|
+
"description": "A probe that is not a known-clean control. The one branch AD-40 gives a defect signature."
|
|
381
825
|
}
|
|
382
826
|
],
|
|
383
|
-
"description": "One corpus probe. Succeeds the prior-art `h0-ground-truth` schema per AD-24, carrying its system identifier, implementation digest, `expectedClean` flag, seeded defects, and rationale, and adding AD-9's probe class and AD-9's per-probe artifact and commit digests. Divergences: `implementationSha` becomes `implementationDigest`, `taskId` does not survive because the probe pins what it describes by digest, and `expectedGate` does not survive because AD-40 makes detection a signature match rather than a verdict comparison and AD-7 keeps comparisons inside the dominance vector, so carrying an expected gate would invite a comparison the architecture forbids. Two constructions
|
|
827
|
+
"description": "One corpus probe. Succeeds the prior-art `h0-ground-truth` schema per AD-24, carrying its system identifier, implementation digest, `expectedClean` flag, seeded defects, and rationale, and adding AD-9's probe class and AD-9's per-probe artifact and commit digests. Divergences: `implementationSha` becomes `implementationDigest`, `taskId` does not survive because the probe pins what it describes by digest, and `expectedGate` does not survive because AD-40 makes detection a signature match rather than a verdict comparison and AD-7 keeps comparisons inside the dominance vector, so carrying an expected gate would invite a comparison the architecture forbids. Two constructions landed here together under one BREAKING `schemaVersion` bump: AD-9's per-class QUALIFICATION record, as a five-route tagged union required on every branch, and AD-40's machine-readable DEFECT SIGNATURE on the `expectedClean: false` branch, carrying the interface kind, the home operation as a method and a path template, the observable channel, and the discriminating condition. Both are required rather than optional, which is what makes the bump breaking under AD-11: every corpus written against version 1 fails to parse. What the schema still does not decide is stated rather than hidden. AD-9's \"an unqualified probe cannot enter a sealed set\" is a corpus-construction invariant enforced by the qualification gate in `core/score/qualification.ts`, not by this schema: all eight class-and-`expectedClean` pairings parse, a route incompatible with the pair parses, and a signature-less non-canary parses, each so the gate can return a reason code carrying an artifact path instead of an anonymous parse failure. The constraint ledger carries both gaps.",
|
|
384
828
|
"$defs": {
|
|
385
829
|
"ArtifactReference": {
|
|
386
830
|
"oneOf": [
|
|
@@ -1146,6 +1590,67 @@
|
|
|
1146
1590
|
}
|
|
1147
1591
|
],
|
|
1148
1592
|
"description": "One of the three operand forms, each a single-keyed object. Named so the published schema carries it once as a shared definition rather than inlining it at seventeen use sites. Which forms are legal in which position is declared on each operator's own `operands`, never narrowed here."
|
|
1593
|
+
},
|
|
1594
|
+
"EvidenceChannel": {
|
|
1595
|
+
"type": "string",
|
|
1596
|
+
"enum": [
|
|
1597
|
+
"response-body",
|
|
1598
|
+
"response-headers",
|
|
1599
|
+
"response-status",
|
|
1600
|
+
"call-inputs",
|
|
1601
|
+
"stdout",
|
|
1602
|
+
"stderr",
|
|
1603
|
+
"exit-code"
|
|
1604
|
+
],
|
|
1605
|
+
"description": "AD-26's closed evidence channel vocabulary. The same seven the interaction-rooted pointer addresses; a channel outside this set is a syntax error rather than an unreachable-evidence finding."
|
|
1606
|
+
},
|
|
1607
|
+
"ProbeInputBindingChannel": {
|
|
1608
|
+
"anyOf": [
|
|
1609
|
+
{
|
|
1610
|
+
"type": "object",
|
|
1611
|
+
"propertyNames": {
|
|
1612
|
+
"type": "string",
|
|
1613
|
+
"minLength": 1
|
|
1614
|
+
},
|
|
1615
|
+
"additionalProperties": {
|
|
1616
|
+
"anyOf": [
|
|
1617
|
+
{
|
|
1618
|
+
"type": "object",
|
|
1619
|
+
"properties": {
|
|
1620
|
+
"literal": {
|
|
1621
|
+
"$ref": "#/$defs/JsonValue"
|
|
1622
|
+
}
|
|
1623
|
+
},
|
|
1624
|
+
"required": [
|
|
1625
|
+
"literal"
|
|
1626
|
+
],
|
|
1627
|
+
"additionalProperties": false
|
|
1628
|
+
},
|
|
1629
|
+
{
|
|
1630
|
+
"type": "object",
|
|
1631
|
+
"properties": {
|
|
1632
|
+
"matcher": {
|
|
1633
|
+
"type": "string",
|
|
1634
|
+
"enum": [
|
|
1635
|
+
"any",
|
|
1636
|
+
"type-violating"
|
|
1637
|
+
]
|
|
1638
|
+
}
|
|
1639
|
+
},
|
|
1640
|
+
"required": [
|
|
1641
|
+
"matcher"
|
|
1642
|
+
],
|
|
1643
|
+
"additionalProperties": false
|
|
1644
|
+
}
|
|
1645
|
+
]
|
|
1646
|
+
}
|
|
1647
|
+
},
|
|
1648
|
+
{
|
|
1649
|
+
"type": "null"
|
|
1650
|
+
}
|
|
1651
|
+
],
|
|
1652
|
+
"description": "A parameter-name-to-binding-value map for one transport channel of a defect signature's selector, or `null` for a channel the selector binds nothing in. An empty map is rejected: `null` is the only spelling for unbound. Admits `{ literal }` and `{ matcher }` only; `{ captured }` and `{ principal }` are contract-local vocabulary and a corpus signature cannot carry either.",
|
|
1653
|
+
"minProperties": 1
|
|
1149
1654
|
}
|
|
1150
1655
|
}
|
|
1151
1656
|
}
|
|
@@ -48,6 +48,12 @@
|
|
|
48
48
|
"maximum": 1,
|
|
49
49
|
"description": "The same closed unit interval a finding's `confidence` uses. AD-21's \"a finding whose confidence falls below the policy threshold\" compares the two, and two different scales would make that comparison meaningless."
|
|
50
50
|
},
|
|
51
|
+
"catchThreshold": {
|
|
52
|
+
"type": "number",
|
|
53
|
+
"minimum": 0,
|
|
54
|
+
"maximum": 1,
|
|
55
|
+
"description": "AD-7's trial-set reducer: a probe counts as caught only when its caught-trial count is strictly greater than this fraction of its valid-trial count, so an exact tie never counts as caught. The published default artifact carries 0.5, the pre-registered \"at least two catches in three valid repetitions\" read at a valid count of three. No `.default()`, on the same reasoning `confidenceThreshold` gives. Required, not optional, which makes this the scoring policy's `schemaVersion` 1 -> 2 BREAKING bump under AD-11, whose rule is that \"adding an optional field is a `schemaVersion` bump recorded in the field's own description; removing or retyping is breaking\". With no default to fill it in, every version-1 policy document fails to parse."
|
|
56
|
+
},
|
|
51
57
|
"minimumTrialCount": {
|
|
52
58
|
"type": "integer",
|
|
53
59
|
"minimum": 1,
|
|
@@ -80,11 +86,12 @@
|
|
|
80
86
|
"policyId",
|
|
81
87
|
"severityFloor",
|
|
82
88
|
"confidenceThreshold",
|
|
89
|
+
"catchThreshold",
|
|
83
90
|
"minimumTrialCount",
|
|
84
91
|
"reExecutionCap",
|
|
85
92
|
"remediationCap",
|
|
86
93
|
"regexMatchStepBudget"
|
|
87
94
|
],
|
|
88
95
|
"additionalProperties": false,
|
|
89
|
-
"description": "The scoring policy, with no prior art. It carries the severity floor, confidence threshold, minimum trial count, re-execution cap, remediation cap, and a regex match-step budget that AD-6, AD-12, AD-4, and AD-21 read, and its digest is one of the
|
|
96
|
+
"description": "The scoring policy, with no prior art. It carries the severity floor, confidence threshold, catch threshold, minimum trial count, re-execution cap, remediation cap, and a regex match-step budget that AD-6, AD-7, AD-12, AD-4, and AD-21 read, and its digest is one of the six named inputs to AD-11's scoring version. It is a published artifact rather than a set of constants so that \"the default\" has an identity a result can name by digest."
|
|
90
97
|
}
|
|
@@ -203,6 +203,15 @@
|
|
|
203
203
|
},
|
|
204
204
|
"description": "AD-16's scoped resource references. Not nullable here, unlike on the contract: absent-versus-explicitly-empty is an AD-31 grading of a CALLER declaration, and this artifact is generated by `seal`, which always knows the answer. No scoped reference may resolve to a forbidden input; that resolution is `scoped-reference-resolves-forbidden` at compile time and is not a schema check."
|
|
205
205
|
},
|
|
206
|
+
"principals": {
|
|
207
|
+
"type": "array",
|
|
208
|
+
"items": {
|
|
209
|
+
"type": "string",
|
|
210
|
+
"pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$",
|
|
211
|
+
"description": "A kebab-case slug. Excludes \"/\" and \"~\" so an identifier can be embedded in an interaction-rooted pointer without escaping."
|
|
212
|
+
},
|
|
213
|
+
"description": "The names declared in the contract's `testData.principals`, sorted. Not nullable, on `scopedResources`'s reasoning above: `seal` always knows the answer, and empty is the legal spelling for a contract declaring none. Names only, no `kind`: `kind` is the author's own opaque vocabulary with no value space any AD declares, so shipping it would put uninterpretable text on the one artifact AD-16 keeps minimal. A name is an opaque label (AD-18), so it discloses no account identifier or credential, and it carries no step identifier (AD-16). Required, not optional, which makes this the sealed brief's `schemaVersion` 1 -> 2 BREAKING bump under AD-11, whose rule is that \"adding an optional field is a `schemaVersion` bump recorded in the field's own description; removing or retyping is breaking\". `[]` is the empty spelling and the key is mandatory, so no version-1 brief parses."
|
|
214
|
+
},
|
|
206
215
|
"budgets": {
|
|
207
216
|
"type": "object",
|
|
208
217
|
"properties": {
|
|
@@ -259,10 +268,11 @@
|
|
|
259
268
|
"directions",
|
|
260
269
|
"permittedInterfaces",
|
|
261
270
|
"scopedResources",
|
|
271
|
+
"principals",
|
|
262
272
|
"budgets",
|
|
263
273
|
"safetyLimits",
|
|
264
274
|
"probeStepBound"
|
|
265
275
|
],
|
|
266
276
|
"additionalProperties": false,
|
|
267
|
-
"description": "The sealed evaluator brief, with no prior art. AD-16 fixes what it carries: the contract's behaviours, the generated evaluator-facing directions of AD-3, permitted interfaces,
|
|
277
|
+
"description": "The sealed evaluator brief, with no prior art. AD-16 fixes what it carries: the contract's behaviours, the generated evaluator-facing directions of AD-3, permitted interfaces, scoped resource references, and the declared test-data principal names, plus budgets and safety limits. It also fixes what the brief never carries: author commentary, the interaction plan, or the plan's step identifiers, and never a prescribed action sequence. Those exclusions are structural rather than conventional: this object is strict, so a brief carrying `interactionPlan`, `commentary`, or a `stepId` fails with `unrecognized_keys`, and each of the three has its own reject fixture rather than resting on strictness alone. It carries no identity field of its own, because every artifact that refers to it does so by digest."
|
|
268
278
|
}
|