eval-quality 0.1.0

package/schemas/rubric.schema.json

@@ -0,0 +1,148 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:eval-quality:schema:rubric",
+  "type": "object",
+  "properties": {
+    "id": {
+      "type": "string",
+      "pattern": "^R-[0-9]{3,}$"
+    },
+    "scaleLevels": {
+      "anyOf": [
+        {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "level": {
+                "type": "integer",
+                "minimum": -9007199254740991,
+                "maximum": 9007199254740991,
+                "description": "The ordinal this level sits at. A repeated ordinal is the one half the compiler closes, failing under `rubric-unanchored`, since two levels at one ordinal make the ordinal address two anchors. Magnitude, sign, ordering, and contiguity stay free everywhere, because a scale running -2 to 2 or 1, 3, 5 is an ordinary authoring choice."
+              },
+              "anchor": {
+                "type": "string",
+                "description": "The observable condition that anchors this level. AD-22 calls an unanchored scale a compile-time failure under `rubric-unanchored`, so the anchor is a declared field rather than a convention."
+              }
+            },
+            "required": [
+              "level",
+              "anchor"
+            ],
+            "additionalProperties": false
+          }
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "`null` and `[]` are both the unanchored shape `rubric-unanchored` fires on, and both must parse."
+    },
+    "failureModePenalties": {
+      "anyOf": [
+        {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "name": {
+                "type": "string",
+                "minLength": 1
+              },
+              "description": {
+                "type": "string",
+                "description": "AD-22 requires named failure-mode penalties and states no magnitude. None is invented here: a penalty weight would be a scoring semantic no module below `score` has authority to mint, and AD-7 keeps weighted composites out of the reported result. AD-22's compile checks ship without one, so a penalty carries a name and a description and nothing else."
+              }
+            },
+            "required": [
+              "name",
+              "description"
+            ],
+            "additionalProperties": false
+          }
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "Missing named penalties is `rubric-unanchored`, not a parse failure."
+    },
+    "maxLength": {
+      "anyOf": [
+        {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 9007199254740991
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "An unbounded length is `null`, which is one of the shapes `rubric-unanchored` fires on."
+    },
+    "criteria": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string",
+            "pattern": "^RC-[0-9]{3,}$"
+          },
+          "text": {
+            "type": "string",
+            "description": "A criterion whose wording matches the compiler's closed stated-reasoning vocabulary fails compilation under `rubric-scores-reasoning-prose`, so such a criterion must stay representable here. A blank text parses for the same reason and fails under `rubric-unanchored`."
+          },
+          "evidence": {
+            "type": "string",
+            "pattern": "^\\/interactions\\/[a-z0-9]+(?:-[a-z0-9]+)*\\/(?:(?:response-body|response-headers|stdout|stderr)(?:\\/(?:[^/~]|~[01])*)*|(?:response-status|exit-code)|call-inputs\\/(?:path|query|header|body)(?:\\/(?:[^/~]|~[01])*)*)$",
+            "description": "Where the criterion is answerable from. A pointer that resolves nowhere is `rubric-evidence-unreachable`, a compile-time code, and is deliberately not a schema rejection."
+          }
+        },
+        "required": [
+          "id",
+          "text",
+          "evidence"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "schemaVersion": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 9007199254740991,
+      "description": "AD-11 requires an integer under this exact name. Deliberately not `z.literal(1)`: the literal exports as `{\"type\":\"number\",\"const\":1}`, losing `integer` for a non-TypeScript consumer, and it would turn a version-2 artifact into an anonymous schema-parse failure instead of AD-28's dedicated `schema-version-mismatch` fault. Version equality belongs to the reader that throws that fault."
+    },
+    "parentDigest": {
+      "anyOf": [
+        {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$",
+          "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "AD-29 lineage. `null` if and only if `revisionCount` is 0. That biconditional is stated here rather than refined: a refinement is silently dropped from the published schema, so a non-TypeScript consumer would never see it, and the constraint ledger records it as not expressible."
+    },
+    "revisionCount": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991,
+      "description": "AD-29: one greater than the parent artifact's."
+    }
+  },
+  "required": [
+    "id",
+    "scaleLevels",
+    "failureModePenalties",
+    "maxLength",
+    "criteria",
+    "schemaVersion",
+    "parentDigest",
+    "revisionCount"
+  ],
+  "additionalProperties": false,
+  "description": "A published rubric, with no prior art: the experiments' evaluator-result and trace-label schemas belong to the deferred semantic layer and their names are not reused. AD-22's compile checks cover anchored scale levels, named failure-mode penalties, a bounded length, criteria that state a question, addressable rubric and criterion identifiers, and wording that matches the compiler's closed stated-reasoning vocabulary. All of them fire under `rubric-unanchored`, `rubric-evidence-unreachable`, and `rubric-scores-reasoning-prose` at compile time, so every shape those codes fire on parses here."
+}

package/schemas/scoring-policy.schema.json

@@ -0,0 +1,90 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:eval-quality:schema:scoring-policy",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 9007199254740991,
+      "description": "AD-11 requires an integer under this exact name. Deliberately not `z.literal(1)`: the literal exports as `{\"type\":\"number\",\"const\":1}`, losing `integer` for a non-TypeScript consumer, and it would turn a version-2 artifact into an anonymous schema-parse failure instead of AD-28's dedicated `schema-version-mismatch` fault. Version equality belongs to the reader that throws that fault."
+    },
+    "parentDigest": {
+      "anyOf": [
+        {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$",
+          "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "AD-29 lineage. `null` if and only if `revisionCount` is 0. That biconditional is stated here rather than refined: a refinement is silently dropped from the published schema, so a non-TypeScript consumer would never see it, and the constraint ledger records it as not expressible."
+    },
+    "revisionCount": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991,
+      "description": "AD-29: one greater than the parent artifact's."
+    },
+    "policyId": {
+      "type": "string",
+      "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$",
+      "description": "A kebab-case slug. Excludes \"/\" and \"~\" so an identifier can be embedded in an interaction-rooted pointer without escaping."
+    },
+    "severityFloor": {
+      "type": "string",
+      "enum": [
+        "low",
+        "material",
+        "critical"
+      ],
+      "description": "AD-21 routes on this: a behavioural failure at or above the floor is FAIL and one below it is CONCERNS, and AD-7 makes it the one severity that constrains the dominance relation."
+    },
+    "confidenceThreshold": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1,
+      "description": "The same closed unit interval a finding's `confidence` uses. AD-21's \"a finding whose confidence falls below the policy threshold\" compares the two, and two different scales would make that comparison meaningless."
+    },
+    "minimumTrialCount": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 9007199254740991,
+      "description": "AD-6: repeated runs are trials, never retries. The published default artifact carries 3, \"because that is what the instrument behind the measured effect used\"; that number is recorded here and is deliberately not a schema default."
+    },
+    "reExecutionCap": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991,
+      "description": "AD-6: exceeding the cap resolves under AD-21 as a statement that the harness is unfit, never that the contract is weak. The published default artifact carries 2."
+    },
+    "remediationCap": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991,
+      "description": "AD-12's remediation cap. The published default artifact carries 3. AD-12 states the package validates rather than enforces it, which is why the Evidence Artifact records the cap's source as caller-attested."
+    },
+    "regexMatchStepBudget": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 9007199254740991,
+      "description": "AD-4: \"a match-step budget from the scoring policy whose breach is a fault, not an outcome state.\" The budget bounds a pure, static, pre-execution complexity estimate over the declared pattern and the candidate string length, never a literal engine-internal step count, and that length is whatever the system under test returned, which the contract author does not control. The structural nested-quantifier rejection stays unconditional and independent of this value, so a generous ceiling here does not weaken that real backstop. The published default artifact carries 1000000."
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "parentDigest",
+    "revisionCount",
+    "policyId",
+    "severityFloor",
+    "confidenceThreshold",
+    "minimumTrialCount",
+    "reExecutionCap",
+    "remediationCap",
+    "regexMatchStepBudget"
+  ],
+  "additionalProperties": false,
+  "description": "The scoring policy, with no prior art. It carries the severity floor, confidence threshold, minimum trial count, re-execution cap, remediation cap, and a regex match-step budget that AD-6, AD-12, AD-4, and AD-21 read, and its digest is one of the five named inputs to AD-11's scoring version. It is a published artifact rather than a set of constants so that \"the default\" has an identity a result can name by digest."
+}

package/schemas/sealed-evaluator-brief.schema.json

@@ -0,0 +1,268 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:eval-quality:schema:sealed-evaluator-brief",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 9007199254740991,
+      "description": "AD-11 requires an integer under this exact name. Deliberately not `z.literal(1)`: the literal exports as `{\"type\":\"number\",\"const\":1}`, losing `integer` for a non-TypeScript consumer, and it would turn a version-2 artifact into an anonymous schema-parse failure instead of AD-28's dedicated `schema-version-mismatch` fault. Version equality belongs to the reader that throws that fault."
+    },
+    "parentDigest": {
+      "anyOf": [
+        {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$",
+          "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "AD-29 lineage. `null` if and only if `revisionCount` is 0. That biconditional is stated here rather than refined: a refinement is silently dropped from the published schema, so a non-TypeScript consumer would never see it, and the constraint ledger records it as not expressible."
+    },
+    "revisionCount": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991,
+      "description": "AD-29: one greater than the parent artifact's."
+    },
+    "contractDigest": {
+      "type": "string",
+      "pattern": "^sha256:[0-9a-f]{64}$",
+      "description": "Binds this brief to the contract it seals. The brief carries no contract identifier of its own and no identity field at all: every artifact that refers to a brief does so by `sealedBriefDigest`, so an identifier would be a second name nothing reads."
+    },
+    "behaviors": {
+      "minItems": 1,
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string",
+            "pattern": "^B-[0-9]{3,}$"
+          },
+          "description": {
+            "type": "string"
+          },
+          "severity": {
+            "type": "string",
+            "enum": [
+              "low",
+              "material",
+              "critical"
+            ]
+          },
+          "observableSuccessCriterion": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "description": "AD-19: what observable condition means this behaviour was met, which is not the nominated success indicator of any one operation. `null` is the shape `no-observable-success-criterion` fires on, and that code fires on exactly this condition and never on an empty oracle list."
+          },
+          "requirementLinks": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "scheme": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "id": {
+                  "type": "string",
+                  "minLength": 1
+                }
+              },
+              "required": [
+                "scheme",
+                "id"
+              ],
+              "additionalProperties": false
+            },
+            "description": "Coin flip (c), settled: requirement and risk linkage lives per behaviour rather than at contract level, because `missing-requirement-linkage` fires when \"a behaviour declares no requirement or risk identifier\" and a contract-level array cannot make that predicate decidable per behaviour. Empty is legal, and both link arrays empty is exactly what the code fires on; a schema demanding non-empty makes it fire never."
+          },
+          "riskLinks": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "scheme": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "id": {
+                  "type": "string",
+                  "minLength": 1
+                }
+              },
+              "required": [
+                "scheme",
+                "id"
+              ],
+              "additionalProperties": false
+            }
+          },
+          "oracles": {
+            "type": "array",
+            "items": {
+              "type": "string",
+              "pattern": "^O-[0-9]{3,}$"
+            },
+            "description": "Identifiers of the oracles this behaviour is discharged by. Deliberately not refined against the contract's declared oracles: no AD-5 code names a behaviour citing an undeclared oracle, so it joins the cross-field rules with no code, unenforced in v0 rather than silently pushed into the schema."
+          }
+        },
+        "required": [
+          "id",
+          "description",
+          "severity",
+          "observableSuccessCriterion",
+          "requirementLinks",
+          "riskLinks",
+          "oracles"
+        ],
+        "additionalProperties": false
+      },
+      "description": "The contract's behaviours, carried through unchanged. At least one, on the contract's own reasoning: a brief with no behaviour directs nothing."
+    },
+    "directions": {
+      "minItems": 1,
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "oracleId": {
+            "type": "string",
+            "pattern": "^O-[0-9]{3,}$"
+          },
+          "text": {
+            "type": "string",
+            "minLength": 1,
+            "description": "The generated prose. `seal` renders each evidence target as a descriptive reference derived from the step's operation and selection predicate, such as \"the response you obtained when you sent an invalid identifier\", never as the step identifier itself; the mapping from step to phrase is one-way. It never carries a prescribed action sequence."
+          }
+        },
+        "required": [
+          "oracleId",
+          "text"
+        ],
+        "additionalProperties": false
+      },
+      "description": "AD-3's generated evaluator-facing directions, one per discharged oracle. At least one, on the same reasoning as `behaviors` above: a brief with no direction verifies nothing."
+    },
+    "permittedInterfaces": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "logicalId": {
+            "type": "string",
+            "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$",
+            "description": "AD-35 binds the brief as it binds the contract: a logical interface identifier only, never a URL, host, or port. The brief is the artifact that actually reaches the executing caller, so a URL leaking onto it defeats the mapping AD-35 keeps outside the package."
+          },
+          "kind": {
+            "type": "string",
+            "enum": [
+              "api",
+              "web",
+              "cli",
+              "mcp"
+            ]
+          }
+        },
+        "required": [
+          "logicalId",
+          "kind"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "scopedResources": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "reference": {
+            "type": "string",
+            "minLength": 1
+          },
+          "kind": {
+            "type": "string",
+            "minLength": 1
+          }
+        },
+        "required": [
+          "reference",
+          "kind"
+        ],
+        "additionalProperties": false
+      },
+      "description": "AD-16's scoped resource references. Not nullable here, unlike on the contract: absent-versus-explicitly-empty is an AD-31 grading of a CALLER declaration, and this artifact is generated by `seal`, which always knows the answer. No scoped reference may resolve to a forbidden input; that resolution is `scoped-reference-resolves-forbidden` at compile time and is not a schema check."
+    },
+    "budgets": {
+      "type": "object",
+      "properties": {
+        "maxToolCalls": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "maxWallClockMinutes": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "maxCostUsd": {
+          "type": "string",
+          "pattern": "^(0|[1-9][0-9]*)(\\.[0-9]+)?$",
+          "description": "Money, so AD-36 carries it as a string in a declared format rather than as a JSON number. Constrained to the unsigned form: a negative ceiling is not a ceiling."
+        }
+      },
+      "required": [
+        "maxToolCalls",
+        "maxWallClockMinutes",
+        "maxCostUsd"
+      ],
+      "additionalProperties": false,
+      "description": "The contract's declared ceilings, carried because AD-16 says the caller who executes needs the ceilings the isolation manifest reports against. The manifest's own `resourceCeilings` is a different shape and a different concern: this is a declaration, that is a record of what the harness enforced."
+    },
+    "safetyLimits": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "probeStepBound": {
+      "anyOf": [
+        {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "AD-16's declared bound on enumerated probe steps, carried through from the contract. The brief-side scripting audit reads it after generation and fires `brief-exceeds-scripting-bound` on any single direction whose rendered text carries more markers than the bound; counts are never summed across directions."
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "parentDigest",
+    "revisionCount",
+    "contractDigest",
+    "behaviors",
+    "directions",
+    "permittedInterfaces",
+    "scopedResources",
+    "budgets",
+    "safetyLimits",
+    "probeStepBound"
+  ],
+  "additionalProperties": false,
+  "description": "The sealed evaluator brief, with no prior art. AD-16 fixes what it carries: the contract's behaviours, the generated evaluator-facing directions of AD-3, permitted interfaces, and scoped resource references, plus budgets and safety limits. It also fixes what the brief never carries: author commentary, the interaction plan, or the plan's step identifiers, and never a prescribed action sequence. Those exclusions are structural rather than conventional: this object is strict, so a brief carrying `interactionPlan`, `commentary`, or a `stepId` fails with `unrecognized_keys`, and each of the three has its own reject fixture rather than resting on strictness alone. It carries no identity field of its own, because every artifact that refers to it does so by digest."
+}