eval-quality 0.1.0 → 0.3.0

package/schemas/evidence-artifact.schema.json

@@ -37,7 +37,7 @@
         "scoringVersion": {
           "type": "string",
           "pattern": "^sha256:[0-9a-f]{64}$",
-          "description": "AD-11: computed by the scorer over the five named inputs below and never caller-supplied."
+          "description": "AD-11: computed by the scorer over the six named inputs below and never caller-supplied."
         },
         "scoringVersionInputs": {
           "type": "object",
@@ -66,6 +66,11 @@
               "type": "string",
               "pattern": "^sha256:[0-9a-f]{64}$",
               "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+            },
+            "mode": {
+              "type": "string",
+              "const": "production",
+              "description": "Owed item 4's identity clause: mode enters AD-11's scoring version, superseding AD-11's five-field sentence. Read from the sealed run record, never re-derived, so two runs scored under one mode never collide with a version computed for the other. Required, not optional, which makes this the evidence artifact's `schemaVersion` 1 -> 2 BREAKING bump under AD-11, whose rule is that \"adding an optional field is a `schemaVersion` bump recorded in the field's own description; removing or retyping is breaking\". A version-1 artifact carries five scoring-version inputs where this one carries six, so it does not parse, and no version computed before the bump compares with one computed after it."
             }
           },
           "required": [
@@ -73,7 +78,8 @@
             "corpusDigest",
             "fixtureDigest",
             "evaluatorConfigurationDigest",
-            "scoringPolicyDigest"
+            "scoringPolicyDigest",
+            "mode"
           ],
           "additionalProperties": false
         },
@@ -113,10 +119,11 @@
               "corpusDigest",
               "fixtureDigest",
               "evaluatorConfigurationDigest",
-              "scoringPolicyDigest"
+              "scoringPolicyDigest",
+              "mode"
             ]
           },
-          "description": "Which of the five scoring-version inputs were caller-attested rather than computed by this package. An enum over the five key names rather than free strings, because AD-32 requires the artifact to state *which* inputs were attested and an unconstrained string cannot be checked against anything. AD-11 names three of the five as caller-attested; the enum admits any subset so a stricter or looser integration stays representable."
+          "description": "Which of the six scoring-version inputs were caller-attested rather than computed by this package. An enum over the six key names rather than free strings, because AD-32 requires the artifact to state *which* inputs were attested and an unconstrained string cannot be checked against anything. AD-11 names three of the six as caller-attested; the enum admits any subset so a stricter or looser integration stays representable. `mode` is the one member that is not a choice: `ScoringVersionInputs.mode` is read from the sealed run record and never re-derived, so a list omitting it is a misdeclaration rather than a stricter integration. The schema admits that shape and the constraint ledger records why; `core/emit` always names it."
         },
         "trials": {
           "type": "object",
@@ -610,7 +617,7 @@
         "scoringVersion": {
           "type": "string",
           "pattern": "^sha256:[0-9a-f]{64}$",
-          "description": "AD-11: computed by the scorer over the five named inputs below and never caller-supplied."
+          "description": "AD-11: computed by the scorer over the six named inputs below and never caller-supplied."
         },
         "scoringVersionInputs": {
           "type": "object",
@@ -639,6 +646,11 @@
               "type": "string",
               "pattern": "^sha256:[0-9a-f]{64}$",
               "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+            },
+            "mode": {
+              "type": "string",
+              "const": "contract-scoring",
+              "description": "Owed item 4's identity clause: mode enters AD-11's scoring version, superseding AD-11's five-field sentence. Read from the sealed run record, never re-derived, so two runs scored under one mode never collide with a version computed for the other. Required, not optional, which makes this the evidence artifact's `schemaVersion` 1 -> 2 BREAKING bump under AD-11, whose rule is that \"adding an optional field is a `schemaVersion` bump recorded in the field's own description; removing or retyping is breaking\". A version-1 artifact carries five scoring-version inputs where this one carries six, so it does not parse, and no version computed before the bump compares with one computed after it."
             }
           },
           "required": [
@@ -646,7 +658,8 @@
             "corpusDigest",
             "fixtureDigest",
             "evaluatorConfigurationDigest",
-            "scoringPolicyDigest"
+            "scoringPolicyDigest",
+            "mode"
           ],
           "additionalProperties": false
         },
@@ -686,10 +699,11 @@
               "corpusDigest",
               "fixtureDigest",
               "evaluatorConfigurationDigest",
-              "scoringPolicyDigest"
+              "scoringPolicyDigest",
+              "mode"
             ]
           },
-          "description": "Which of the five scoring-version inputs were caller-attested rather than computed by this package. An enum over the five key names rather than free strings, because AD-32 requires the artifact to state *which* inputs were attested and an unconstrained string cannot be checked against anything. AD-11 names three of the five as caller-attested; the enum admits any subset so a stricter or looser integration stays representable."
+          "description": "Which of the six scoring-version inputs were caller-attested rather than computed by this package. An enum over the six key names rather than free strings, because AD-32 requires the artifact to state *which* inputs were attested and an unconstrained string cannot be checked against anything. AD-11 names three of the six as caller-attested; the enum admits any subset so a stricter or looser integration stays representable. `mode` is the one member that is not a choice: `ScoringVersionInputs.mode` is read from the sealed run record and never re-derived, so a list omitting it is a misdeclaration rather than a stricter integration. The schema admits that shape and the constraint ledger records why; `core/emit` always names it."
         },
         "trials": {
           "type": "object",
@@ -1123,6 +1137,108 @@
         "contractVerdict": {
           "$ref": "#/$defs/Verdict"
         },
+        "uncitedFindingGaps": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "findingId": {
+                "type": "string",
+                "pattern": "^F-[0-9]{3,}$"
+              },
+              "observationIds": {
+                "minItems": 1,
+                "type": "array",
+                "items": {
+                  "type": "string",
+                  "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$",
+                  "description": "A kebab-case slug. Excludes \"/\" and \"~\" so an identifier can be embedded in an interaction-rooted pointer without escaping."
+                }
+              },
+              "quotedEvidence": {
+                "minItems": 1,
+                "type": "array",
+                "items": {
+                  "anyOf": [
+                    {
+                      "type": "object",
+                      "properties": {
+                        "quote": {
+                          "type": "string",
+                          "minLength": 1,
+                          "description": "The evaluator's verbatim quotation, per AD-23. Non-empty: an empty quotation quotes nothing, no AD-5 code names the condition, and under the admit-rule's second clause the schema is therefore the enforcement point. That this text appears in at least one of the finding's cited observations is NOT checked here; it is an AD-32 declared-versus-observed inconsistency that invalidates at ingest, and ADR-009 Decision 2 settles the precedence: \"cited identifiers govern the witness match; quotation audits it.\""
+                        },
+                        "channel": {
+                          "type": "string",
+                          "const": "artifact"
+                        },
+                        "artifactId": {
+                          "type": "string",
+                          "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$",
+                          "description": "Which written file the quotation came from. A channel alone does not identify one: an operation may write several, and an audit that searched all of them at once accepted a quotation lifted from a file the finding never cited, which is the opposite of what AD-23's verbatim requirement is for."
+                        }
+                      },
+                      "required": [
+                        "quote",
+                        "channel",
+                        "artifactId"
+                      ],
+                      "additionalProperties": false
+                    },
+                    {
+                      "type": "object",
+                      "properties": {
+                        "quote": {
+                          "type": "string",
+                          "minLength": 1,
+                          "description": "The evaluator's verbatim quotation, per AD-23. Non-empty: an empty quotation quotes nothing, no AD-5 code names the condition, and under the admit-rule's second clause the schema is therefore the enforcement point. That this text appears in at least one of the finding's cited observations is NOT checked here; it is an AD-32 declared-versus-observed inconsistency that invalidates at ingest, and ADR-009 Decision 2 settles the precedence: \"cited identifiers govern the witness match; quotation audits it.\""
+                        },
+                        "channel": {
+                          "type": "string",
+                          "enum": [
+                            "response-body",
+                            "response-headers",
+                            "response-status",
+                            "call-inputs",
+                            "stdout",
+                            "stderr",
+                            "exit-code"
+                          ]
+                        },
+                        "artifactId": {
+                          "type": "null",
+                          "description": "Null on every channel but `artifact`, and required rather than optional so a record cannot omit the question. Only a written file needs naming; every other channel is one thing per observation."
+                        }
+                      },
+                      "required": [
+                        "quote",
+                        "channel",
+                        "artifactId"
+                      ],
+                      "additionalProperties": false
+                    }
+                  ]
+                }
+              },
+              "severity": {
+                "type": "string",
+                "enum": [
+                  "low",
+                  "material",
+                  "critical"
+                ]
+              }
+            },
+            "required": [
+              "findingId",
+              "observationIds",
+              "quotedEvidence",
+              "severity"
+            ],
+            "additionalProperties": false
+          },
+          "description": "Owed item 5's rung record, contract-scoring only: one entry per uncited `defect` finding, per `core/score/outcome.ts`'s `uncitedDefectFindingGaps`. Required, not optional, which makes this the `schemaVersion` 2 -> 3 BREAKING bump under AD-11: an empty array is legal and is what a run with no uncited defect finding carries, but the key itself must be present. Never on the production branch, where the ladder signal alone is enough and this key would fail `strictObject`'s `unrecognized_keys` rule."
+        },
         "systemRecommendationRecorded": {
           "description": "The evaluator's recommendation about the SYSTEM, recorded and never promoted. On a knowingly defective probe a system-directed FAIL is an input rather than a signal about the contract.",
           "$ref": "#/$defs/EvaluatorRecommendation"
@@ -1158,6 +1274,7 @@
         "remediation",
         "mode",
         "contractVerdict",
+        "uncitedFindingGaps",
         "systemRecommendationRecorded",
         "systemRecommendationNote"
       ],
@@ -1165,7 +1282,7 @@
       "description": "Contract-scoring mode: the verdict is about the contract, and the evaluator's recommendation about the system is recorded beside it rather than promoted."
     }
   ],
-  "description": "The scored output, owned by `emit` per AD-24, with no prior art; `score` produces the outcome and verdict values it serializes. The two modes are separate branches because AD-21 requires that the production verdict and the contract verdict never share a field. This expresses that settled sentence and no more: Owed item 4's remaining half stays open, namely mode being fixed before ingest and entering AD-11's identity inputs rather than appearing first here; and the Sealed Run Record still carries no mode.",
+  "description": "The scored output, owned by `emit` per AD-24, with no prior art; `score` produces the outcome and verdict values it serializes. The two modes are separate branches because AD-21 requires that the production verdict and the contract verdict never share a field. Mode is no longer first stated here: the Sealed Run Record now carries a required `mode`, where AD-21's \"fixed before ingest\" puts it, so this artifact restates a mode the sealed record already fixed and is never the source. Owed item 4 is closed as of this schemaVersion: `mode` is `ScoringVersionInputs`'s sixth field, `core/score/ladder.ts` carries `ProductionAssessment`/`ContractAssessment` and their own total ladders, and `core/score/mode-agreement.ts` rejects a `production` record paired with a `contract-scoring` artifact in both directions as an AD-32 cross-artifact disagreement. That check is a hand-written function at the assembly boundary rather than a schema refinement, since no single schema sees both artifacts; `emit` is its one caller, immediately before returning the artifact it just built. Owed item 5 is closed as of this schemaVersion too: the contract-scoring branch carries a required `uncitedFindingGaps`, and both ladders in `core/score/ladder.ts` gain a shared `uncited-defect-finding` CONCERNS row over `AssessmentCommon.uncitedDefectFindings`, so an ingested defect finding citing no oracle now resolves at least CONCERNS instead of routing nowhere.",
   "$defs": {
     "CheckResolution": {
       "description": "One node of an oracle's `check` tree as it resolved. AD-4's resolution is three-valued rather than two, and the introduction vocabulary has exactly one member because AD-4 closes the introduction set at one: an empty collection. `null` where the resolution is not `insufficient-evidence`. A leaf carries an empty `children` array.",

package/schemas/isolation-manifest.schema.json

@@ -35,7 +35,7 @@
     "contractId": {
       "type": "string",
       "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$",
-      "description": "The prior art's `taskId`, renamed. \"Task\" is experiment vocabulary with no product meaning, and the manifest keeps an identifier because it is the artifact `core/ingest` matches against a run; the run record and the probe pin what they describe by digest instead."
+      "description": "The prior art's `taskId`, renamed. \"Task\" is experiment vocabulary with no product meaning, and the manifest keeps an identifier so a reader can say which contract the run was isolated for. It is a label and not a second operand for any check: `core/ingest` matches this artifact to a run on `runId`, `contractDigest`, and `evaluatorConfigurationDigest`, which is the whole of the agreement the two artifacts make. Nothing in the pipeline carries a contract identifier to compare this against, since the sealed run record pins its contract by digest and the eval contract is not among ingest's inputs."
     },
     "conditionArm": {
       "type": "string",
@@ -68,38 +68,44 @@
     "allowedMounts": {
       "type": "array",
       "items": {
-        "type": "string"
+        "type": "string",
+        "minLength": 1
       }
     },
     "observedMounts": {
       "type": "array",
       "items": {
-        "type": "string"
+        "type": "string",
+        "minLength": 1
       },
-      "description": "Observed mounts, network targets, and tool calls exceeding their allowlist is a violation AD-16 has `core/ingest` record; it is a cross-field rule with no AD-5 code and the schema admits it."
+      "description": "Observed mounts, network targets, and tool calls exceeding their allowlist is a violation AD-16 has `core/ingest` record; it is a cross-field rule with no AD-5 code and the schema admits it. Each entry is non-empty: an empty string is not a mount, and it renders as a basis line naming nothing after its colon."
     },
     "networkAllowlist": {
       "type": "array",
       "items": {
-        "type": "string"
+        "type": "string",
+        "minLength": 1
       }
     },
     "observedNetworkTargets": {
       "type": "array",
       "items": {
-        "type": "string"
+        "type": "string",
+        "minLength": 1
       }
     },
     "toolAllowlist": {
       "type": "array",
       "items": {
-        "type": "string"
+        "type": "string",
+        "minLength": 1
       }
     },
     "observedToolCalls": {
       "type": "array",
       "items": {
-        "type": "string"
+        "type": "string",
+        "minLength": 1
       }
     },
     "resourceCeilings": {
@@ -357,13 +363,14 @@
     "violation": {
       "anyOf": [
         {
-          "type": "string"
+          "type": "string",
+          "minLength": 1
         },
         {
           "type": "null"
         }
       ],
-      "description": "The sixteenth of the prior art's sixteen required members, and the one that is not a description of the run: a non-null violation invalidates it. AD-16 says the schema is \"seeded from the prior art's fifteen\"; the prior art's `required` array has sixteen. Settled by construction: the fifteen are the fields describing the run and this is the invalidation outcome. All sixteen are carried."
+      "description": "The sixteenth of the prior art's sixteen required members, and the one that is not a description of the run: a non-null violation invalidates it. Non-empty when present, on `QuotedEvidence.quote`'s precedent: `null` is the spelling for no violation, so an empty string is a violation that names nothing while still invalidating the run and rendering a blank basis entry. AD-16 says the schema is \"seeded from the prior art's fifteen\"; the prior art's `required` array has sixteen. Settled by construction: the fifteen are the fields describing the run and this is the invalidation outcome. All sixteen are carried."
     }
   },
   "required": [