@openplanr/protocol 0.2.0

package/schemas/v1.4.0/evaluation-aggregate-report.schema.json

@@ -0,0 +1,166 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openplanr.dev/schemas/v1.4.0/evaluation-aggregate-report.schema.json",
+  "title": "OpenPlanr Protocol v1.4 skill evaluation aggregate report",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["kind", "schemaVersion", "protocolVersion", "reportId", "generatedAt", "redaction", "inputs", "counters", "rates", "budget", "absences", "skills", "scenarios", "reportDigest"],
+  "properties": {
+    "kind": { "const": "evaluation-aggregate-report" },
+    "schemaVersion": { "const": "1.0.0" },
+    "protocolVersion": { "const": "1.4.0" },
+    "reportId": { "type": "string", "pattern": "^ear_[a-f0-9]{32}$" },
+    "generatedAt": { "type": "string", "format": "date-time" },
+    "redaction": {
+      "type": "object", "additionalProperties": false,
+      "required": ["policyVersion", "rawPromptsExcluded", "tracesExcluded", "screenshotsExcluded", "modelOutputExcluded", "absolutePathsExcluded", "credentialMaterialExcluded"],
+      "properties": {
+        "policyVersion": { "const": "1.0.0" },
+        "rawPromptsExcluded": { "const": true },
+        "tracesExcluded": { "const": true },
+        "screenshotsExcluded": { "const": true },
+        "modelOutputExcluded": { "const": true },
+        "absolutePathsExcluded": { "const": true },
+        "credentialMaterialExcluded": { "const": true }
+      }
+    },
+    "inputs": {
+      "type": "object", "additionalProperties": false,
+      "required": ["corpusDigest", "graderRegistryDigest", "hostProfileRegistryDigest", "gatePolicyDigest", "budgetBaselineDigest", "hostProfileDigests", "runResultDigests"],
+      "properties": {
+        "corpusDigest": { "$ref": "#/$defs/digest" },
+        "graderRegistryDigest": { "$ref": "#/$defs/digest" },
+        "hostProfileRegistryDigest": { "$ref": "#/$defs/digest" },
+        "gatePolicyDigest": { "$ref": "#/$defs/digest" },
+        "budgetBaselineDigest": { "$ref": "#/$defs/digest" },
+        "hostProfileDigests": { "type": "array", "minItems": 1, "maxItems": 32, "uniqueItems": true, "items": { "$ref": "#/$defs/digest" } },
+        "runResultDigests": { "type": "array", "minItems": 1, "maxItems": 256, "uniqueItems": true, "items": { "$ref": "#/$defs/digest" } }
+      }
+    },
+    "counters": {
+      "type": "object", "additionalProperties": false,
+      "required": ["scenariosTotal", "scenariosPassed", "scenariosFailed", "scenariosBlocked", "scenariosAbsent", "scenariosWaived", "triggerTruePositives", "triggerFalsePositives", "triggerFalseNegatives", "journeysAttempted", "journeysCompleted", "outputsValidated", "outputsSchemaValid", "assetsDeclared", "assetsPresent", "exportsDeclared", "exportsPresent", "packageMembersDeclared", "packageMembersPresent", "permissionPrompts", "retries", "findingsP0", "findingsP1", "findingsP2", "findingsP3"],
+      "properties": {
+        "scenariosTotal": { "$ref": "#/$defs/counter" },
+        "scenariosPassed": { "$ref": "#/$defs/counter" },
+        "scenariosFailed": { "$ref": "#/$defs/counter" },
+        "scenariosBlocked": { "$ref": "#/$defs/counter" },
+        "scenariosAbsent": { "$ref": "#/$defs/counter" },
+        "scenariosWaived": { "$ref": "#/$defs/counter" },
+        "triggerTruePositives": { "$ref": "#/$defs/counter" },
+        "triggerFalsePositives": { "$ref": "#/$defs/counter" },
+        "triggerFalseNegatives": { "$ref": "#/$defs/counter" },
+        "journeysAttempted": { "$ref": "#/$defs/counter" },
+        "journeysCompleted": { "$ref": "#/$defs/counter" },
+        "outputsValidated": { "$ref": "#/$defs/counter" },
+        "outputsSchemaValid": { "$ref": "#/$defs/counter" },
+        "assetsDeclared": { "$ref": "#/$defs/counter" },
+        "assetsPresent": { "$ref": "#/$defs/counter" },
+        "exportsDeclared": { "$ref": "#/$defs/counter" },
+        "exportsPresent": { "$ref": "#/$defs/counter" },
+        "packageMembersDeclared": { "$ref": "#/$defs/counter" },
+        "packageMembersPresent": { "$ref": "#/$defs/counter" },
+        "permissionPrompts": { "$ref": "#/$defs/counter" },
+        "retries": { "$ref": "#/$defs/counter" },
+        "findingsP0": { "$ref": "#/$defs/counter" },
+        "findingsP1": { "$ref": "#/$defs/counter" },
+        "findingsP2": { "$ref": "#/$defs/counter" },
+        "findingsP3": { "$ref": "#/$defs/counter" }
+      }
+    },
+    "rates": {
+      "type": "object", "additionalProperties": false,
+      "required": ["triggerPrecision", "triggerRecall", "journeyCompletion", "schemaValidity", "assetParity", "exportParity", "packageParity"],
+      "properties": {
+        "triggerPrecision": { "$ref": "#/$defs/rateBasisPoints" },
+        "triggerRecall": { "$ref": "#/$defs/rateBasisPoints" },
+        "journeyCompletion": { "$ref": "#/$defs/rateBasisPoints" },
+        "schemaValidity": { "$ref": "#/$defs/rateBasisPoints" },
+        "assetParity": { "$ref": "#/$defs/rateBasisPoints" },
+        "exportParity": { "$ref": "#/$defs/rateBasisPoints" },
+        "packageParity": { "$ref": "#/$defs/rateBasisPoints" }
+      }
+    },
+    "budget": {
+      "type": "object", "additionalProperties": false,
+      "required": ["baselineDigest", "latencyMsBaseline", "latencyMsObserved", "latencyRegression", "costEstimateMicrosBaseline", "costEstimateMicrosObserved", "costRegression", "permissionPromptCeiling", "retryCeiling"],
+      "properties": {
+        "baselineDigest": { "$ref": "#/$defs/digest" },
+        "latencyMsBaseline": { "$ref": "#/$defs/durationMs" },
+        "latencyMsObserved": { "$ref": "#/$defs/durationMs" },
+        "latencyRegression": { "$ref": "#/$defs/regressionBasisPoints" },
+        "costEstimateMicrosBaseline": { "$ref": "#/$defs/costMicros" },
+        "costEstimateMicrosObserved": { "$ref": "#/$defs/costMicros" },
+        "costRegression": { "$ref": "#/$defs/regressionBasisPoints" },
+        "permissionPromptCeiling": { "$ref": "#/$defs/counter" },
+        "retryCeiling": { "$ref": "#/$defs/counter" }
+      }
+    },
+    "absences": {
+      "type": "object", "additionalProperties": false,
+      "required": ["graderMissing", "graderErrored", "graderUnavailable", "hostUnavailable", "fixtureUnavailable", "budgetExceeded", "notRun"],
+      "properties": {
+        "graderMissing": { "$ref": "#/$defs/counter" },
+        "graderErrored": { "$ref": "#/$defs/counter" },
+        "graderUnavailable": { "$ref": "#/$defs/counter" },
+        "hostUnavailable": { "$ref": "#/$defs/counter" },
+        "fixtureUnavailable": { "$ref": "#/$defs/counter" },
+        "budgetExceeded": { "$ref": "#/$defs/counter" },
+        "notRun": { "$ref": "#/$defs/counter" }
+      }
+    },
+    "skills": { "type": "array", "minItems": 1, "maxItems": 128, "uniqueItems": true, "items": { "$ref": "#/$defs/skillSummary" } },
+    "scenarios": { "type": "array", "maxItems": 2048, "uniqueItems": true, "items": { "$ref": "#/$defs/scenarioSummary" } },
+    "reportDigest": { "$ref": "#/$defs/digest" }
+  },
+  "$defs": {
+    "digest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" },
+    "counter": { "type": "integer", "minimum": 0, "maximum": 1000000 },
+    "rateBasisPoints": { "type": "integer", "minimum": 0, "maximum": 10000 },
+    "regressionBasisPoints": { "type": "integer", "minimum": -10000, "maximum": 1000000 },
+    "durationMs": { "type": "integer", "minimum": 0, "maximum": 86400000 },
+    "costMicros": { "type": "integer", "minimum": 0, "maximum": 1000000000000 },
+    "skillId": { "type": "string", "pattern": "^[a-z][a-z0-9-]{1,63}$" },
+    "skillVersion": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?$" },
+    "promptClass": { "enum": ["positive", "negative", "ambiguous", "permission-denied"] },
+    "scenarioStatus": { "enum": ["passed", "failed", "blocked", "absent", "waived"] },
+    "absenceReason": { "enum": ["grader-missing", "grader-errored", "grader-unavailable", "host-unavailable", "fixture-unavailable", "budget-exceeded", "not-run"] },
+    "skillSummary": {
+      "type": "object", "additionalProperties": false,
+      "required": ["skillId", "skillVersion", "scenariosTotal", "scenariosPassed", "scenariosFailed", "scenariosBlocked", "scenariosAbsent", "scenariosWaived"],
+      "properties": {
+        "skillId": { "$ref": "#/$defs/skillId" },
+        "skillVersion": { "$ref": "#/$defs/skillVersion" },
+        "scenariosTotal": { "$ref": "#/$defs/counter" },
+        "scenariosPassed": { "$ref": "#/$defs/counter" },
+        "scenariosFailed": { "$ref": "#/$defs/counter" },
+        "scenariosBlocked": { "$ref": "#/$defs/counter" },
+        "scenariosAbsent": { "$ref": "#/$defs/counter" },
+        "scenariosWaived": { "$ref": "#/$defs/counter" }
+      }
+    },
+    "scenarioSummary": {
+      "type": "object", "additionalProperties": false,
+      "required": ["scenarioDigest", "skillId", "hostProfileDigest", "graderRegistrationDigest", "fixtureSetDigest", "promptClass", "status", "absenceReason", "waiverDigest", "observations", "latencyMs"],
+      "properties": {
+        "scenarioDigest": { "$ref": "#/$defs/digest" },
+        "skillId": { "$ref": "#/$defs/skillId" },
+        "hostProfileDigest": { "$ref": "#/$defs/digest" },
+        "graderRegistrationDigest": { "$ref": "#/$defs/digest" },
+        "fixtureSetDigest": { "$ref": "#/$defs/digest" },
+        "promptClass": { "$ref": "#/$defs/promptClass" },
+        "status": { "$ref": "#/$defs/scenarioStatus" },
+        "absenceReason": { "oneOf": [{ "const": null }, { "$ref": "#/$defs/absenceReason" }] },
+        "waiverDigest": { "oneOf": [{ "const": null }, { "$ref": "#/$defs/digest" }] },
+        "observations": { "type": "integer", "minimum": 0, "maximum": 64 },
+        "latencyMs": { "$ref": "#/$defs/durationMs" }
+      },
+      "allOf": [
+        { "if": { "properties": { "status": { "const": "absent" } }, "required": ["status"] }, "then": { "properties": { "absenceReason": { "$ref": "#/$defs/absenceReason" } } } },
+        { "if": { "properties": { "status": { "enum": ["passed", "failed", "blocked", "waived"] } }, "required": ["status"] }, "then": { "properties": { "absenceReason": { "const": null } } } },
+        { "if": { "properties": { "status": { "const": "waived" } }, "required": ["status"] }, "then": { "properties": { "waiverDigest": { "$ref": "#/$defs/digest" } } } },
+        { "if": { "properties": { "status": { "enum": ["passed", "failed", "blocked", "absent"] } }, "required": ["status"] }, "then": { "properties": { "waiverDigest": { "const": null } } } }
+      ]
+    }
+  }
+}

package/schemas/v1.4.0/evaluation-budget.schema.json

@@ -0,0 +1,49 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openplanr.dev/schemas/v1.4.0/evaluation-budget.schema.json",
+  "title": "OpenPlanr Protocol 1.4 evaluation budget",
+  "type": "object", "additionalProperties": false,
+  "required": ["kind", "schemaVersion", "protocolVersion", "budgetId", "carriesMeasurement", "scope", "ceilings", "baseline", "budgetDigest"],
+  "properties": {
+    "kind": { "const": "evaluation-budget" }, "schemaVersion": { "const": "1.0.0" }, "protocolVersion": { "const": "1.4.0" },
+    "budgetId": { "type": "string", "pattern": "^ebg_[a-f0-9]{32}$" },
+    "carriesMeasurement": { "const": false, "description": "A budget states ceilings. Observed values live in a run result, never here." },
+    "scope": {
+      "type": "object", "additionalProperties": false, "required": ["kind", "id"],
+      "properties": { "kind": { "enum": ["corpus", "skill", "scenario"] }, "id": { "$ref": "#/$defs/identifier" } }
+    },
+    "ceilings": {
+      "type": "object", "additionalProperties": false,
+      "required": ["latencyMsP50Ceiling", "latencyMsP95Ceiling", "inputTokensCeiling", "outputTokensCeiling", "totalTokensCeiling", "costEstimateCurrency", "costEstimateMicrosCeiling", "permissionPromptCeiling", "retryCeiling"],
+      "properties": {
+        "latencyMsP50Ceiling": { "$ref": "#/$defs/durationMs" },
+        "latencyMsP95Ceiling": { "$ref": "#/$defs/durationMs" },
+        "inputTokensCeiling": { "$ref": "#/$defs/tokenCount" },
+        "outputTokensCeiling": { "$ref": "#/$defs/tokenCount" },
+        "totalTokensCeiling": { "$ref": "#/$defs/tokenCount" },
+        "costEstimateCurrency": { "const": "USD" },
+        "costEstimateMicrosCeiling": { "$ref": "#/$defs/costMicros" },
+        "permissionPromptCeiling": { "$ref": "#/$defs/counter" },
+        "retryCeiling": { "$ref": "#/$defs/counter" }
+      }
+    },
+    "baseline": {
+      "type": "object", "additionalProperties": false,
+      "required": ["frozen", "baselineDigest", "capturedAt"],
+      "properties": {
+        "frozen": { "const": true },
+        "baselineDigest": { "$ref": "#/$defs/digest", "description": "Exact bytes a latency or cost regression is measured against. A different baseline is a different budget." },
+        "capturedAt": { "type": "string", "format": "date-time" }
+      }
+    },
+    "budgetDigest": { "$ref": "#/$defs/digest" }
+  },
+  "$defs": {
+    "digest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" },
+    "identifier": { "type": "string", "minLength": 2, "maxLength": 128, "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$" },
+    "durationMs": { "type": "integer", "minimum": 1, "maximum": 86400000 },
+    "tokenCount": { "type": "integer", "minimum": 0, "maximum": 100000000 },
+    "costMicros": { "type": "integer", "minimum": 0, "maximum": 1000000000000 },
+    "counter": { "type": "integer", "minimum": 0, "maximum": 1000000 }
+  }
+}

package/schemas/v1.4.0/evaluation-corpus.schema.json

@@ -0,0 +1,93 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openplanr.dev/schemas/v1.4.0/evaluation-corpus.schema.json",
+  "x-openplanr-contract": { "id": "evaluation-corpus", "version": "1.4.0" },
+  "title": "OpenPlanr Protocol v1.4 skill evaluation corpus",
+  "description": "The closed, self-digested membership envelope for one skill's evaluation scenarios. `corpusDigest` is SHA-256 over the RFC 8785 canonical form of this record with `corpusId` and `corpusDigest` removed, and `corpusId` is `ecp_` followed by that digest hex. Members ascend by `scenarioId`. A duplicate `scenarioId`, a `scenarioId` whose hex differs from its own `scenarioDigest`, a member whose recorded digest differs from the digest recomputed from its source bytes, and a member scenario bound to a skill identity other than `skill` are each refused rather than skipped.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "kind",
+    "schemaVersion",
+    "protocolVersion",
+    "corpusId",
+    "corpusDigest",
+    "title",
+    "skill",
+    "members"
+  ],
+  "properties": {
+    "kind": { "const": "evaluation-corpus" },
+    "schemaVersion": { "const": "1.0.0" },
+    "protocolVersion": { "const": "1.4.0" },
+    "corpusId": {
+      "$ref": "#/$defs/corpusId",
+      "description": "The literal prefix `ecp_` followed by the hexadecimal part of `corpusDigest`. Any other value is a foreign identity."
+    },
+    "corpusDigest": { "$ref": "#/$defs/digest" },
+    "title": { "$ref": "#/$defs/title" },
+    "skill": {
+      "$ref": "#/$defs/skillIdentity",
+      "description": "The single skill every member belongs to. A corpus never spans skills or versions."
+    },
+    "members": {
+      "type": "array",
+      "minItems": 1,
+      "maxItems": 1024,
+      "uniqueItems": true,
+      "items": { "$ref": "#/$defs/member" },
+      "description": "Canonical member order is ascending `scenarioId`, so an identical membership always canonicalizes to an identical corpus digest."
+    }
+  },
+  "$defs": {
+    "digest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" },
+    "corpusId": { "type": "string", "pattern": "^ecp_[a-f0-9]{64}$" },
+    "scenarioId": { "type": "string", "pattern": "^esc_[a-f0-9]{64}$" },
+    "skillId": {
+      "type": "string",
+      "minLength": 2,
+      "maxLength": 64,
+      "pattern": "^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$"
+    },
+    "semver": {
+      "type": "string",
+      "pattern": "^(?:0|[1-9][0-9]*)\\.(?:0|[1-9][0-9]*)\\.(?:0|[1-9][0-9]*)$"
+    },
+    "title": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 160,
+      "pattern": "^\\S(?:[\\s\\S]*\\S)?$",
+      "not": { "pattern": "(?:<[^>]+>|\\{\\{[^}]+\\}\\})" }
+    },
+    "scenarioSourcePath": {
+      "type": "string",
+      "minLength": 6,
+      "maxLength": 512,
+      "pattern": "^[A-Za-z0-9_][A-Za-z0-9._-]*(?:/[A-Za-z0-9_][A-Za-z0-9._-]*)*\\.json$",
+      "description": "Normalized repository-relative path to the member's source document. Absolute paths, drive letters, backslashes, and dot segments are refused."
+    },
+    "skillIdentity": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "version"],
+      "properties": {
+        "id": { "$ref": "#/$defs/skillId" },
+        "version": { "$ref": "#/$defs/semver" }
+      }
+    },
+    "member": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["scenarioId", "scenarioDigest", "sourcePath"],
+      "properties": {
+        "scenarioId": { "$ref": "#/$defs/scenarioId" },
+        "scenarioDigest": {
+          "$ref": "#/$defs/digest",
+          "description": "The digest the corpus was sealed against. It is recomputed from `sourcePath` on every read and a mismatch invalidates the member."
+        },
+        "sourcePath": { "$ref": "#/$defs/scenarioSourcePath" }
+      }
+    }
+  }
+}

package/schemas/v1.4.0/evaluation-fixture.schema.json

@@ -0,0 +1,64 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openplanr.dev/schemas/v1.4.0/evaluation-fixture.schema.json",
+  "x-openplanr-contract": { "id": "evaluation-fixture", "version": "1.4.0" },
+  "title": "OpenPlanr Protocol v1.4 skill evaluation fixture",
+  "description": "One immutable evaluation input, bound to exact bytes by content digest and to exact handling by media kind. The record carries no field able to hold fixture bytes, so raw content and credential material can never travel inside it.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "kind",
+    "schemaVersion",
+    "protocolVersion",
+    "fixtureId",
+    "fixtureDigest",
+    "mediaKind",
+    "byteLength",
+    "contentDigest",
+    "sourcePath",
+    "retention",
+    "containsCredentialMaterial"
+  ],
+  "properties": {
+    "kind": { "const": "evaluation-fixture" },
+    "schemaVersion": { "const": "1.0.0" },
+    "protocolVersion": { "const": "1.4.0" },
+    "fixtureId": {
+      "$ref": "#/$defs/fixtureId",
+      "description": "The literal prefix `efx_` followed by the hexadecimal part of `fixtureDigest`. Any other value is a foreign identity."
+    },
+    "fixtureDigest": {
+      "$ref": "#/$defs/digest",
+      "description": "SHA-256 over the RFC 8785 canonical form of this record with `fixtureId` and `fixtureDigest` removed, so media kind and content digest are both identity-bearing."
+    },
+    "mediaKind": {
+      "enum": ["text", "markdown", "json", "source", "html", "image", "archive", "binary"],
+      "description": "Closed handling class. A reader that does not support the declared kind refuses the fixture instead of guessing."
+    },
+    "byteLength": { "type": "integer", "minimum": 0, "maximum": 67108864 },
+    "contentDigest": {
+      "$ref": "#/$defs/digest",
+      "description": "SHA-256 over the fixture bytes at `sourcePath`. A recomputed digest that differs invalidates every observation bound to this fixture."
+    },
+    "sourcePath": { "$ref": "#/$defs/relativePath" },
+    "retention": {
+      "const": "local-only",
+      "description": "Fixture bytes stay on the machine that authored them. No other retention is representable."
+    },
+    "containsCredentialMaterial": {
+      "const": false,
+      "description": "Explicit negative declaration. A fixture carrying credential material cannot be expressed by this contract."
+    }
+  },
+  "$defs": {
+    "digest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" },
+    "fixtureId": { "type": "string", "pattern": "^efx_[a-f0-9]{64}$" },
+    "relativePath": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 512,
+      "pattern": "^[A-Za-z0-9_][A-Za-z0-9._-]*(?:/[A-Za-z0-9_][A-Za-z0-9._-]*)*$",
+      "description": "Normalized repository-relative path. Absolute paths, drive letters, backslashes, and dot segments are refused."
+    }
+  }
+}

package/schemas/v1.4.0/evaluation-gate-policy.schema.json

@@ -0,0 +1,33 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openplanr.dev/schemas/v1.4.0/evaluation-gate-policy.schema.json",
+  "title": "OpenPlanr Protocol 1.4 evaluation gate policy",
+  "type": "object", "additionalProperties": false,
+  "required": ["kind", "schemaVersion", "protocolVersion", "policyId", "budgetDigest", "baselineDigest", "gates", "gatePolicyDigest"],
+  "properties": {
+    "kind": { "const": "evaluation-gate-policy" }, "schemaVersion": { "const": "1.0.0" }, "protocolVersion": { "const": "1.4.0" },
+    "policyId": { "type": "string", "pattern": "^egp_[a-f0-9]{32}$" },
+    "budgetDigest": { "$ref": "#/$defs/digest" },
+    "baselineDigest": { "$ref": "#/$defs/digest", "description": "Frozen baseline the regression gates are measured against." },
+    "gates": {
+      "type": "object", "additionalProperties": false,
+      "description": "Keyed by metric identity. Every mandatory threshold is a constant, so no caller can lower a gate and still be a valid policy.",
+      "required": ["trigger-precision", "trigger-recall", "journey-completion", "schema-validity", "asset-parity", "export-parity", "package-parity", "finding-severity-p0", "finding-severity-p1", "latency-regression", "cost-regression"],
+      "properties": {
+        "trigger-precision": { "type": "object", "additionalProperties": false, "required": ["comparator", "unit", "threshold", "waivable"], "properties": { "comparator": { "const": ">=" }, "unit": { "const": "basis-points" }, "threshold": { "const": 9500 }, "waivable": { "const": true } } },
+        "trigger-recall": { "type": "object", "additionalProperties": false, "required": ["comparator", "unit", "threshold", "waivable"], "properties": { "comparator": { "const": ">=" }, "unit": { "const": "basis-points" }, "threshold": { "const": 9500 }, "waivable": { "const": true } } },
+        "journey-completion": { "type": "object", "additionalProperties": false, "required": ["comparator", "unit", "threshold", "waivable"], "properties": { "comparator": { "const": ">=" }, "unit": { "const": "basis-points" }, "threshold": { "const": 9000 }, "waivable": { "const": true } } },
+        "schema-validity": { "type": "object", "additionalProperties": false, "required": ["comparator", "unit", "threshold", "waivable"], "properties": { "comparator": { "const": "==" }, "unit": { "const": "basis-points" }, "threshold": { "const": 10000 }, "waivable": { "const": false } } },
+        "asset-parity": { "type": "object", "additionalProperties": false, "required": ["comparator", "unit", "threshold", "waivable"], "properties": { "comparator": { "const": "==" }, "unit": { "const": "basis-points" }, "threshold": { "const": 10000 }, "waivable": { "const": true } } },
+        "export-parity": { "type": "object", "additionalProperties": false, "required": ["comparator", "unit", "threshold", "waivable"], "properties": { "comparator": { "const": "==" }, "unit": { "const": "basis-points" }, "threshold": { "const": 10000 }, "waivable": { "const": true } } },
+        "package-parity": { "type": "object", "additionalProperties": false, "required": ["comparator", "unit", "threshold", "waivable"], "properties": { "comparator": { "const": "==" }, "unit": { "const": "basis-points" }, "threshold": { "const": 10000 }, "waivable": { "const": false } } },
+        "finding-severity-p0": { "type": "object", "additionalProperties": false, "required": ["comparator", "unit", "threshold", "waivable"], "properties": { "comparator": { "const": "<=" }, "unit": { "const": "count" }, "threshold": { "const": 0 }, "waivable": { "const": false } } },
+        "finding-severity-p1": { "type": "object", "additionalProperties": false, "required": ["comparator", "unit", "threshold", "waivable"], "properties": { "comparator": { "const": "<=" }, "unit": { "const": "count" }, "threshold": { "const": 0 }, "waivable": { "const": false } } },
+        "latency-regression": { "type": "object", "additionalProperties": false, "required": ["comparator", "unit", "threshold", "waivable", "measuredAgainst"], "properties": { "comparator": { "const": "<=" }, "unit": { "const": "basis-points" }, "threshold": { "const": 2000 }, "waivable": { "const": true }, "measuredAgainst": { "const": "frozen-baseline" } } },
+        "cost-regression": { "type": "object", "additionalProperties": false, "required": ["comparator", "unit", "threshold", "waivable", "measuredAgainst"], "properties": { "comparator": { "const": "<=" }, "unit": { "const": "basis-points" }, "threshold": { "const": 2000 }, "waivable": { "const": true }, "measuredAgainst": { "const": "frozen-baseline" } } }
+      }
+    },
+    "gatePolicyDigest": { "$ref": "#/$defs/digest" }
+  },
+  "$defs": { "digest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" } }
+}

package/schemas/v1.4.0/evaluation-grader-registration.schema.json

@@ -0,0 +1,141 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openplanr.dev/schemas/v1.4.0/evaluation-grader-registration.schema.json",
+  "title": "Protocol 1.4 evaluation grader registration",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["kind", "schemaVersion", "protocolVersion", "graderId", "graderName", "graderVersion", "graderType", "declaredInputs", "determinism", "telemetry", "consent", "credentialReference", "outputContract", "unavailableBehavior", "blocksContractValidation", "provenance", "registrationDigest"],
+  "properties": {
+    "kind": { "const": "evaluation-grader-registration" },
+    "schemaVersion": { "const": "1.0.0" },
+    "protocolVersion": { "const": "1.4.0" },
+    "graderId": { "type": "string", "pattern": "^egr_[a-f0-9]{32}$" },
+    "graderName": { "$ref": "#/$defs/name" },
+    "graderVersion": { "$ref": "#/$defs/version" },
+    "graderType": { "enum": ["deterministic-schema", "deterministic-behaviour", "live-model-judgement"] },
+    "declaredInputs": { "type": "array", "minItems": 1, "maxItems": 16, "uniqueItems": true, "items": { "$ref": "#/$defs/declaredInput" } },
+    "determinism": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["deterministic", "modelCall", "network", "credentialUse", "filesystemWrite", "replay"],
+      "properties": {
+        "deterministic": { "type": "boolean" },
+        "modelCall": { "type": "boolean" },
+        "network": { "type": "boolean" },
+        "credentialUse": { "type": "boolean" },
+        "filesystemWrite": { "const": false },
+        "replay": { "enum": ["identical-output", "not-guaranteed"] }
+      }
+    },
+    "telemetry": { "const": false },
+    "consent": {
+      "oneOf": [
+        { "type": "null" },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["required", "mode", "scope", "recordDigest", "grantedAt", "expiresAt", "noDefaultChoice"],
+          "properties": {
+            "required": { "const": true },
+            "mode": { "const": "named-owner-current" },
+            "scope": { "enum": ["single-run", "single-corpus"] },
+            "recordDigest": { "$ref": "#/$defs/digest" },
+            "grantedAt": { "type": "string", "format": "date-time" },
+            "expiresAt": { "type": "string", "format": "date-time" },
+            "noDefaultChoice": { "const": true }
+          }
+        }
+      ]
+    },
+    "credentialReference": {
+      "oneOf": [
+        { "type": "null" },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["refKind", "refName", "custody"],
+          "properties": {
+            "refKind": { "enum": ["environment-name", "os-custody-key"] },
+            "refName": { "type": "string", "minLength": 1, "maxLength": 128, "pattern": "^[A-Za-z][A-Za-z0-9_.-]*$" },
+            "custody": { "const": "host-owned" }
+          }
+        }
+      ]
+    },
+    "outputContract": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["contractId", "schemaVersion", "protocolVersion"],
+      "properties": {
+        "contractId": { "const": "evaluation-observation" },
+        "schemaVersion": { "const": "1.0.0" },
+        "protocolVersion": { "const": "1.4.0" }
+      }
+    },
+    "unavailableBehavior": { "const": "typed-absence" },
+    "blocksContractValidation": { "const": false },
+    "provenance": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["packageName", "packageVersion", "sourceDigest"],
+      "properties": {
+        "packageName": { "$ref": "#/$defs/name" },
+        "packageVersion": { "$ref": "#/$defs/version" },
+        "sourceDigest": { "$ref": "#/$defs/digest" }
+      }
+    },
+    "registrationDigest": { "$ref": "#/$defs/digest" }
+  },
+  "allOf": [
+    {
+      "if": { "properties": { "graderType": { "enum": ["deterministic-schema", "deterministic-behaviour"] } }, "required": ["graderType"] },
+      "then": {
+        "properties": {
+          "determinism": {
+            "properties": {
+              "deterministic": { "const": true },
+              "modelCall": { "const": false },
+              "network": { "const": false },
+              "credentialUse": { "const": false },
+              "replay": { "const": "identical-output" }
+            }
+          },
+          "consent": { "const": null },
+          "credentialReference": { "const": null }
+        }
+      }
+    },
+    {
+      "if": { "properties": { "graderType": { "const": "live-model-judgement" } }, "required": ["graderType"] },
+      "then": {
+        "required": ["consent", "credentialReference"],
+        "properties": {
+          "determinism": {
+            "properties": {
+              "deterministic": { "const": false },
+              "modelCall": { "const": true },
+              "replay": { "const": "not-guaranteed" }
+            }
+          },
+          "consent": { "type": "object" },
+          "credentialReference": { "type": "object" }
+        }
+      }
+    }
+  ],
+  "$defs": {
+    "digest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" },
+    "name": { "type": "string", "maxLength": 64, "pattern": "^[a-z][a-z0-9]*(?:-[a-z0-9]+)*$" },
+    "version": { "type": "string", "pattern": "^(?:0|[1-9][0-9]*)\\.(?:0|[1-9][0-9]*)\\.(?:0|[1-9][0-9]*)$" },
+    "declaredInput": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["inputKind", "required", "binding"],
+      "properties": {
+        "inputKind": { "enum": ["scenario", "fixture", "host-profile", "budget", "skill-source", "declared-output", "package-manifest", "run-trace"] },
+        "required": { "type": "boolean" },
+        "binding": { "const": "digest" }
+      }
+    }
+  }
+}

package/schemas/v1.4.0/evaluation-grader-registry.schema.json

@@ -0,0 +1,20 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openplanr.dev/schemas/v1.4.0/evaluation-grader-registry.schema.json",
+  "title": "Protocol 1.4 evaluation grader registry",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["kind", "schemaVersion", "protocolVersion", "registryId", "registryVersion", "digestAlgorithm", "canonicalization", "memberOrder", "graders", "registryDigest"],
+  "properties": {
+    "kind": { "const": "evaluation-grader-registry" },
+    "schemaVersion": { "const": "1.0.0" },
+    "protocolVersion": { "const": "1.4.0" },
+    "registryId": { "type": "string", "pattern": "^egrr_[a-f0-9]{32}$" },
+    "registryVersion": { "const": "1.0.0" },
+    "digestAlgorithm": { "const": "sha256" },
+    "canonicalization": { "const": "jcs" },
+    "memberOrder": { "const": "graderName-ascending" },
+    "graders": { "type": "array", "minItems": 1, "maxItems": 64, "uniqueItems": true, "items": { "$ref": "./evaluation-grader-registration.schema.json" } },
+    "registryDigest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }
+  }
+}

package/schemas/v1.4.0/evaluation-host-profile-registry.schema.json

@@ -0,0 +1,20 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://openplanr.dev/schemas/v1.4.0/evaluation-host-profile-registry.schema.json",
+  "title": "Protocol 1.4 evaluation host profile registry",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["kind", "schemaVersion", "protocolVersion", "registryId", "registryVersion", "digestAlgorithm", "canonicalization", "memberOrder", "profiles", "registryDigest"],
+  "properties": {
+    "kind": { "const": "evaluation-host-profile-registry" },
+    "schemaVersion": { "const": "1.0.0" },
+    "protocolVersion": { "const": "1.4.0" },
+    "registryId": { "type": "string", "pattern": "^ehpr_[a-f0-9]{32}$" },
+    "registryVersion": { "const": "1.0.0" },
+    "digestAlgorithm": { "const": "sha256" },
+    "canonicalization": { "const": "jcs" },
+    "memberOrder": { "const": "host-ascending" },
+    "profiles": { "type": "array", "minItems": 1, "maxItems": 64, "uniqueItems": true, "items": { "$ref": "./evaluation-host-profile.schema.json" } },
+    "registryDigest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }
+  }
+}