eval-quality 0.1.0

package/schemas/isolation-manifest.schema.json

@@ -0,0 +1,394 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:eval-quality:schema:isolation-manifest",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 9007199254740991,
+      "description": "AD-11 requires an integer under this exact name. Deliberately not `z.literal(1)`: the literal exports as `{\"type\":\"number\",\"const\":1}`, losing `integer` for a non-TypeScript consumer, and it would turn a version-2 artifact into an anonymous schema-parse failure instead of AD-28's dedicated `schema-version-mismatch` fault. Version equality belongs to the reader that throws that fault."
+    },
+    "parentDigest": {
+      "anyOf": [
+        {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$",
+          "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "AD-29 lineage. `null` if and only if `revisionCount` is 0. That biconditional is stated here rather than refined: a refinement is silently dropped from the published schema, so a non-TypeScript consumer would never see it, and the constraint ledger records it as not expressible."
+    },
+    "revisionCount": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991,
+      "description": "AD-29: one greater than the parent artifact's."
+    },
+    "runId": {
+      "type": "string",
+      "minLength": 1
+    },
+    "contractId": {
+      "type": "string",
+      "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$",
+      "description": "The prior art's `taskId`, renamed. \"Task\" is experiment vocabulary with no product meaning, and the manifest keeps an identifier because it is the artifact `core/ingest` matches against a run; the run record and the probe pin what they describe by digest instead."
+    },
+    "conditionArm": {
+      "type": "string",
+      "minLength": 1,
+      "description": "An opaque caller label with no product semantics, per AD-24. The prior art's three-member enum does not survive, on the same reasoning as the run record's."
+    },
+    "modelSnapshot": {
+      "type": "string",
+      "minLength": 1
+    },
+    "systemPromptDigest": {
+      "type": "string",
+      "pattern": "^sha256:[0-9a-f]{64}$",
+      "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+    },
+    "contractDigest": {
+      "type": "string",
+      "pattern": "^sha256:[0-9a-f]{64}$",
+      "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+    },
+    "evaluatorConfigurationDigest": {
+      "type": "string",
+      "pattern": "^sha256:[0-9a-f]{64}$",
+      "description": "Added here and absent from the prior art. AD-32 requires this digest to agree between the manifest and the run record, which was previously an agreement rule asserted over two fields that did not exist. Required and non-nullable on both, which is what makes the substitution cost two contradictions rather than one omission. The agreement itself is a cross-artifact rule no schema can see."
+    },
+    "workspaceIdentity": {
+      "type": "string",
+      "minLength": 1
+    },
+    "allowedMounts": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "observedMounts": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "description": "Observed mounts, network targets, and tool calls exceeding their allowlist is a violation AD-16 has `core/ingest` record; it is a cross-field rule with no AD-5 code and the schema admits it."
+    },
+    "networkAllowlist": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "observedNetworkTargets": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "toolAllowlist": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "observedToolCalls": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "resourceCeilings": {
+      "type": "object",
+      "properties": {
+        "maxToolCalls": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 9007199254740991
+        },
+        "maxInputTokens": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 9007199254740991
+        },
+        "maxOutputTokens": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 9007199254740991
+        },
+        "maxWallClockMinutes": {
+          "type": "number",
+          "exclusiveMinimum": 0,
+          "description": "The prior art's `exclusiveMinimum: 0`, kept. A ceiling of zero minutes is not a ceiling, it is a run that may not happen, and the manifest records what the harness actually enforced."
+        },
+        "maxCostUsd": {
+          "type": "string",
+          "pattern": "^([1-9][0-9]*(\\.[0-9]+)?|0\\.[0-9]*[1-9][0-9]*)$",
+          "description": "Money, so AD-36 carries it as a string. The prior art's number does not survive; an AD-24 divergence, and the reason AD-36's exact-decimal rule reaches the manifest as well as the run record. The retyping keeps the prior art's `exclusiveMinimum: 0` rather than dropping it: `UnsignedDecimalString` would admit \"0\", which is the bound the prior art spent a keyword excluding."
+        }
+      },
+      "required": [
+        "maxToolCalls",
+        "maxInputTokens",
+        "maxOutputTokens",
+        "maxWallClockMinutes",
+        "maxCostUsd"
+      ],
+      "additionalProperties": false
+    },
+    "actualResourceUse": {
+      "type": "object",
+      "properties": {
+        "toolCalls": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "inputTokens": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "outputTokens": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        "wallClockSeconds": {
+          "type": "number",
+          "minimum": 0
+        },
+        "costUsd": {
+          "type": "string",
+          "pattern": "^(0|[1-9][0-9]*)(\\.[0-9]+)?$",
+          "description": "The decimal-string format restricted to non-negative values, for quantities where a negative has no meaning."
+        }
+      },
+      "required": [
+        "toolCalls",
+        "inputTokens",
+        "outputTokens",
+        "wallClockSeconds",
+        "costUsd"
+      ],
+      "additionalProperties": false
+    },
+    "forbiddenInputAccounting": {
+      "type": "object",
+      "properties": {
+        "original-spec": {
+          "type": "object",
+          "properties": {
+            "withheld": {
+              "type": "boolean"
+            },
+            "note": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            }
+          },
+          "required": [
+            "withheld",
+            "note"
+          ],
+          "additionalProperties": false
+        },
+        "source-code": {
+          "type": "object",
+          "properties": {
+            "withheld": {
+              "type": "boolean"
+            },
+            "note": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            }
+          },
+          "required": [
+            "withheld",
+            "note"
+          ],
+          "additionalProperties": false
+        },
+        "repository": {
+          "type": "object",
+          "properties": {
+            "withheld": {
+              "type": "boolean"
+            },
+            "note": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            }
+          },
+          "required": [
+            "withheld",
+            "note"
+          ],
+          "additionalProperties": false
+        },
+        "builder-transcript": {
+          "type": "object",
+          "properties": {
+            "withheld": {
+              "type": "boolean"
+            },
+            "note": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            }
+          },
+          "required": [
+            "withheld",
+            "note"
+          ],
+          "additionalProperties": false
+        },
+        "implementation-logs": {
+          "type": "object",
+          "properties": {
+            "withheld": {
+              "type": "boolean"
+            },
+            "note": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            }
+          },
+          "required": [
+            "withheld",
+            "note"
+          ],
+          "additionalProperties": false
+        },
+        "comparator-results": {
+          "type": "object",
+          "properties": {
+            "withheld": {
+              "type": "boolean"
+            },
+            "note": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            }
+          },
+          "required": [
+            "withheld",
+            "note"
+          ],
+          "additionalProperties": false
+        },
+        "human-labels": {
+          "type": "object",
+          "properties": {
+            "withheld": {
+              "type": "boolean"
+            },
+            "note": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            }
+          },
+          "required": [
+            "withheld",
+            "note"
+          ],
+          "additionalProperties": false
+        }
+      },
+      "required": [
+        "original-spec",
+        "source-code",
+        "repository",
+        "builder-transcript",
+        "implementation-logs",
+        "comparator-results",
+        "human-labels"
+      ],
+      "additionalProperties": false,
+      "description": "The field AD-16 adds and the prior art lacks: \"the isolation manifest's required fields \u2026 account for each forbidden input by name\". Seven keys, generated from `FORBIDDEN_INPUT_FLOOR` so the floor has one home."
+    },
+    "violation": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "The sixteenth of the prior art's sixteen required members, and the one that is not a description of the run: a non-null violation invalidates it. AD-16 says the schema is \"seeded from the prior art's fifteen\"; the prior art's `required` array has sixteen. Settled by construction: the fifteen are the fields describing the run and this is the invalidation outcome. All sixteen are carried."
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "parentDigest",
+    "revisionCount",
+    "runId",
+    "contractId",
+    "conditionArm",
+    "modelSnapshot",
+    "systemPromptDigest",
+    "contractDigest",
+    "evaluatorConfigurationDigest",
+    "workspaceIdentity",
+    "allowedMounts",
+    "observedMounts",
+    "networkAllowlist",
+    "observedNetworkTargets",
+    "toolAllowlist",
+    "observedToolCalls",
+    "resourceCeilings",
+    "actualResourceUse",
+    "forbiddenInputAccounting",
+    "violation"
+  ],
+  "additionalProperties": false,
+  "description": "The per-run isolation audit. Succeeds the prior-art `isolation-manifest` schema per AD-24, carrying all sixteen of its required members plus `evaluatorConfigurationDigest`, `forbiddenInputAccounting`, and AD-29's lineage; `taskId` becomes `contractId`, `condition` is demoted to an opaque `conditionArm`, and money in both resource shapes becomes a decimal string. An absent, unparseable, incomplete, or violating manifest invalidates the run under AD-16 and never downgrades a verdict; a schema rejection is the correct expression of unparseable and incomplete, so nothing is admitted for their sake, and the violating case is `core/ingest`'s."
+}

package/schemas/preflight-verdict.schema.json

@@ -0,0 +1,114 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:eval-quality:schema:preflight-verdict",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 9007199254740991,
+      "description": "AD-11 requires an integer under this exact name. Deliberately not `z.literal(1)`: the literal exports as `{\"type\":\"number\",\"const\":1}`, losing `integer` for a non-TypeScript consumer, and it would turn a version-2 artifact into an anonymous schema-parse failure instead of AD-28's dedicated `schema-version-mismatch` fault. Version equality belongs to the reader that throws that fault."
+    },
+    "parentDigest": {
+      "anyOf": [
+        {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$",
+          "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "AD-29 lineage. `null` if and only if `revisionCount` is 0. That biconditional is stated here rather than refined: a refinement is silently dropped from the published schema, so a non-TypeScript consumer would never see it, and the constraint ledger records it as not expressible."
+    },
+    "revisionCount": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991,
+      "description": "AD-29: one greater than the parent artifact's."
+    },
+    "runId": {
+      "type": "string",
+      "minLength": 1,
+      "description": "An opaque caller label."
+    },
+    "fixtureDigest": {
+      "type": "string",
+      "pattern": "^sha256:[0-9a-f]{64}$",
+      "description": "Required, not nullable. AD-10 makes the fixture digest a required field of the verdict, and AD-11 names it as one of the scoring version's five inputs, so an absent one leaves the scoring version uncomputable."
+    },
+    "passed": {
+      "type": "boolean"
+    },
+    "checks": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "kind": {
+            "type": "string",
+            "enum": [
+              "interface-present",
+              "input-sensitivity",
+              "state-reset",
+              "clean-control",
+              "seeded-faults-scoped",
+              "seeded-fault-fired"
+            ]
+          },
+          "operationId": {
+            "anyOf": [
+              {
+                "type": "string",
+                "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$",
+                "description": "A kebab-case slug. Excludes \"/\" and \"~\" so an identifier can be embedded in an interaction-rooted pointer without escaping."
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "description": "Nullable because AD-10 scopes input sensitivity and state reset per operation and interface presence per interface, so a check that is not per-operation has nothing to name here."
+          },
+          "outcome": {
+            "type": "string",
+            "enum": [
+              "satisfied",
+              "failed",
+              "exempt"
+            ],
+            "description": "`exempt` exists because AD-10 says \"an operation declaring no inputs in any channel is exempt and records the exemption\", and an exemption with no spelling is an exemption nobody records. That a failed pre-flight invalidates the run, and that a sensitivity witness resolving `insufficient-evidence` fails rather than passes, are AD-10 semantics for `core/preflight`; the schema carries the outcome and refines nothing."
+          },
+          "note": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ]
+          }
+        },
+        "required": [
+          "kind",
+          "operationId",
+          "outcome",
+          "note"
+        ],
+        "additionalProperties": false
+      }
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "parentDigest",
+    "revisionCount",
+    "runId",
+    "fixtureDigest",
+    "passed",
+    "checks"
+  ],
+  "additionalProperties": false,
+  "description": "The pre-flight verdict: a pure function of the observations the environment-probe port returned, with no prior art. AD-10 bounds it to a closed list of check kinds, all six transcribed from that AD's own prose, and requires the fixture digest as a required field. Probing a fixture is not executing the system under test, and a failed pre-flight invalidates the run rather than becoming a contract verdict."
+}

package/schemas/private-artifact-manifest.schema.json

@@ -0,0 +1,103 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:eval-quality:schema:private-artifact-manifest",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 9007199254740991,
+      "description": "AD-11 requires an integer under this exact name. Deliberately not `z.literal(1)`: the literal exports as `{\"type\":\"number\",\"const\":1}`, losing `integer` for a non-TypeScript consumer, and it would turn a version-2 artifact into an anonymous schema-parse failure instead of AD-28's dedicated `schema-version-mismatch` fault. Version equality belongs to the reader that throws that fault."
+    },
+    "parentDigest": {
+      "anyOf": [
+        {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$",
+          "description": "AD-27 digest: \"sha256:\" plus 64 lowercase hexadecimal characters."
+        },
+        {
+          "type": "null"
+        }
+      ],
+      "description": "AD-29 lineage. `null` if and only if `revisionCount` is 0. That biconditional is stated here rather than refined: a refinement is silently dropped from the published schema, so a non-TypeScript consumer would never see it, and the constraint ledger records it as not expressible."
+    },
+    "revisionCount": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991,
+      "description": "AD-29: one greater than the parent artifact's."
+    },
+    "entries": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "privateRef": {
+            "type": "string",
+            "minLength": 1,
+            "description": "The opaque handle the corpus port resolves under AD-8. Only emptiness is excluded: an opaque reference with a shape this package understands would stop being opaque."
+          },
+          "digest": {
+            "type": "string",
+            "pattern": "^sha256:[0-9a-f]{64}$",
+            "description": "The prior art's `sha256`, renamed against the shared AD-27 primitive; the value form is byte-identical. AD-8 is explicit that \"a manifest digest is never a trusted label\": the core recomputes the digest from the resolved bytes and a mismatch is an AD-28 `digest-mismatch` fault at ingest. That is a rule about resolved bytes, which a schema over a string cannot see, so it is recorded here and enforced nowhere in this file."
+          },
+          "artifactKind": {
+            "type": "string",
+            "enum": [
+              "private-input-record",
+              "raw-trace",
+              "raw-result",
+              "human-label",
+              "implementation-patch",
+              "isolation-manifest",
+              "scripted-baseline-output",
+              "other"
+            ],
+            "description": "The prior art's `artifactType`, renamed and ratified with its eight members unchanged."
+          },
+          "publicSafeRunId": {
+            "anyOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "description": "The run this artifact belongs to, in the public-safe vocabulary. `null` for an artifact belonging to no single run."
+          },
+          "sanitizationPolicy": {
+            "anyOf": [
+              {
+                "type": "string",
+                "minLength": 1
+              },
+              {
+                "type": "null"
+              }
+            ],
+            "description": "AD-8 carries \"the sanitization policy applied to it\", and the nearest antecedent is the entry rather than the manifest: a manifest holding a raw trace beside a human label gives the two different treatment, so per-entry is the only reading that survives. Opaque rather than an enum for the reason `ScopedResource.kind` is opaque: no AD supplies a value space, and inventing one is the unshaped-declaration defect in reverse. `null` spells \"none applied\", which must stay representable."
+          }
+        },
+        "required": [
+          "privateRef",
+          "digest",
+          "artifactKind",
+          "publicSafeRunId",
+          "sanitizationPolicy"
+        ],
+        "additionalProperties": false
+      }
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "parentDigest",
+    "revisionCount",
+    "entries"
+  ],
+  "additionalProperties": false,
+  "description": "The public-safe index of every private artifact this repository references. Succeeds the prior-art `private-evidence-manifest` schema per AD-24, with three divergences: the prior art's `manifestVersion: const 1` becomes AD-11's `schemaVersion` alongside AD-29's lineage, `artifactType` becomes `artifactKind` with its eight members ratified unchanged, and each entry gains AD-8's `sanitizationPolicy`. AD-18 binds this artifact hardest: it \"never contains a private path, credential, or domain value\". That prohibition is not expressible over opaque strings, so it is stated here, recorded in the constraint ledger as not expressible, and enforced by review and by the publication guard rather than by this schema."
+}