@bjoernrochel/sevro 0.1.0-rc.1

package/schemas/report-v1.schema.json

@@ -0,0 +1,164 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:sevro:schema:report:v1",
+  "title": "Sevro report v1",
+  "type": "object",
+  "required": ["format", "inputs", "rows", "summary"],
+  "properties": {
+    "format": { "const": "sevro.report.v1" },
+    "inputs": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/$defs/input" }
+    },
+    "rows": { "type": "array", "items": { "$ref": "#/$defs/row" } },
+    "summary": {
+      "type": "object",
+      "required": ["cases", "passed", "failed", "notAssessed"],
+      "properties": {
+        "cases": { "type": "integer", "minimum": 0 },
+        "passed": { "type": "integer", "minimum": 0 },
+        "failed": { "type": "integer", "minimum": 0 },
+        "notAssessed": { "type": "integer", "minimum": 0 }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false,
+  "$defs": {
+    "path": { "type": "string", "minLength": 1 },
+    "digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+    "state": {
+      "enum": [
+        "completed",
+        "failed",
+        "cancelled",
+        "not_run",
+        "error",
+        "unavailable",
+        "not_requested"
+      ]
+    },
+    "verdict": { "enum": ["passed", "failed", "not_assessed"] },
+    "input": {
+      "type": "object",
+      "required": [
+        "resultFile",
+        "execution",
+        "grading",
+        "task",
+        "exitCode",
+        "caseCount"
+      ],
+      "properties": {
+        "resultFile": { "$ref": "#/$defs/path" },
+        "execution": {
+          "enum": ["completed", "failed", "cancelled", "not_run"]
+        },
+        "grading": {
+          "enum": ["completed", "error", "unavailable", "not_requested"]
+        },
+        "task": { "$ref": "#/$defs/verdict" },
+        "exitCode": { "enum": [0, 1, 2, 3, 4, 64, 70, 130, 143] },
+        "caseCount": { "type": "integer", "minimum": 0 }
+      },
+      "additionalProperties": false
+    },
+    "route": {
+      "type": "object",
+      "required": ["host", "model", "effort"],
+      "properties": {
+        "host": { "type": "string", "minLength": 1 },
+        "model": { "type": "string", "minLength": 1 },
+        "effort": { "type": "string", "minLength": 1 }
+      },
+      "additionalProperties": false
+    },
+    "outcome": {
+      "type": "object",
+      "required": ["trial", "id", "status"],
+      "properties": {
+        "trial": { "type": "integer", "minimum": 1 },
+        "id": { "type": "string", "minLength": 1 },
+        "status": { "enum": ["passed", "failed", "unavailable"] }
+      },
+      "additionalProperties": false
+    },
+    "row": {
+      "type": "object",
+      "required": [
+        "resultFile",
+        "evidencePath",
+        "runId",
+        "caseId",
+        "execution",
+        "grading",
+        "task",
+        "exitCode",
+        "trialCount",
+        "taskPassRate",
+        "candidateDurationMs",
+        "inputTokens",
+        "outputTokens",
+        "costUsd",
+        "runnerBuildDigest",
+        "evaluationDigest",
+        "requestedCondition",
+        "actualCondition",
+        "candidateRoute",
+        "routes",
+        "domainOutcomes"
+      ],
+      "properties": {
+        "resultFile": { "$ref": "#/$defs/path" },
+        "evidencePath": { "type": ["string", "null"] },
+        "runId": { "type": ["string", "null"] },
+        "caseId": { "type": "string", "minLength": 1 },
+        "execution": { "$ref": "#/$defs/state" },
+        "grading": { "$ref": "#/$defs/state" },
+        "task": { "$ref": "#/$defs/verdict" },
+        "exitCode": { "enum": [0, 1, 2, 3, 4, 64, 70, 130, 143] },
+        "trialCount": { "type": "integer", "minimum": 0 },
+        "taskPassRate": {
+          "type": ["number", "null"],
+          "minimum": 0,
+          "maximum": 1
+        },
+        "candidateDurationMs": { "type": ["number", "null"], "minimum": 0 },
+        "inputTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "outputTokens": { "type": ["integer", "null"], "minimum": 0 },
+        "costUsd": { "type": ["number", "null"], "minimum": 0 },
+        "runnerBuildDigest": {
+          "oneOf": [{ "$ref": "#/$defs/digest" }, { "type": "null" }]
+        },
+        "evaluationDigest": {
+          "oneOf": [{ "$ref": "#/$defs/digest" }, { "type": "null" }]
+        },
+        "requestedCondition": { "enum": ["passive", "enforced", null] },
+        "actualCondition": { "enum": ["passive", "enforced", "unknown", null] },
+        "candidateRoute": {
+          "oneOf": [{ "$ref": "#/$defs/route" }, { "type": "null" }]
+        },
+        "routes": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["role", "host", "model", "effort"],
+            "properties": {
+              "role": { "enum": ["candidate", "semantic", "advisory"] },
+              "host": { "type": "string", "minLength": 1 },
+              "model": { "type": "string", "minLength": 1 },
+              "effort": { "type": "string", "minLength": 1 }
+            },
+            "additionalProperties": false
+          }
+        },
+        "domainOutcomes": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/outcome" }
+        }
+      },
+      "additionalProperties": false
+    }
+  }
+}

package/schemas/run-evidence-v1.schema.json

@@ -0,0 +1,510 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "urn:sevro:schema:run-evidence:v1",
+  "title": "Sevro retained run evidence v1",
+  "type": "object",
+  "required": [
+    "format",
+    "runId",
+    "evaluationIdentity",
+    "configuration",
+    "runner",
+    "project",
+    "extension",
+    "condition",
+    "graders",
+    "routes",
+    "result",
+    "trials"
+  ],
+  "properties": {
+    "format": { "const": "sevro.run-evidence.v1" },
+    "runId": { "type": "string", "minLength": 1 },
+    "evaluationIdentity": {
+      "type": "object",
+      "required": ["algorithm", "digest", "dimensions"],
+      "properties": {
+        "algorithm": { "const": "sevro.identity.v1" },
+        "digest": { "$ref": "#/$defs/digest" },
+        "dimensions": {
+          "type": "object",
+          "required": [
+            "runnerBuildDigest",
+            "projectDigest",
+            "configurationDigest",
+            "extensionDigest",
+            "extensionProtocol",
+            "caseDigest",
+            "fixtureDigest",
+            "checksDigest",
+            "requiredEvidenceDigest",
+            "evaluatorDigest",
+            "graderDigest",
+            "instrumentationDigest",
+            "routeDigest",
+            "condition",
+            "trialCount",
+            "passThreshold"
+          ],
+          "properties": {
+            "runnerBuildDigest": { "$ref": "#/$defs/digest" },
+            "projectDigest": { "$ref": "#/$defs/digest" },
+            "configurationDigest": { "$ref": "#/$defs/digest" },
+            "extensionDigest": {
+              "oneOf": [{ "$ref": "#/$defs/digest" }, { "type": "null" }]
+            },
+            "extensionProtocol": { "enum": ["sevro.extension.v1", null] },
+            "caseDigest": { "$ref": "#/$defs/digest" },
+            "fixtureDigest": { "$ref": "#/$defs/digest" },
+            "checksDigest": { "$ref": "#/$defs/digest" },
+            "requiredEvidenceDigest": { "$ref": "#/$defs/digest" },
+            "evaluatorDigest": { "$ref": "#/$defs/digest" },
+            "graderDigest": { "$ref": "#/$defs/digest" },
+            "instrumentationDigest": { "$ref": "#/$defs/digest" },
+            "routeDigest": { "$ref": "#/$defs/digest" },
+            "condition": { "enum": ["passive", "enforced"] },
+            "trialCount": { "type": "integer", "minimum": 1 },
+            "passThreshold": {
+              "type": "number",
+              "exclusiveMinimum": 0,
+              "maximum": 1
+            }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+    "configuration": {
+      "type": "object",
+      "required": ["digest", "redacted"],
+      "properties": {
+        "digest": { "$ref": "#/$defs/digest" },
+        "redacted": { "type": "object" }
+      },
+      "additionalProperties": false
+    },
+    "runner": { "$ref": "#/$defs/runner" },
+    "project": {
+      "type": "object",
+      "required": ["root", "revision", "dirtyPatchDigest"],
+      "properties": {
+        "root": { "$ref": "#/$defs/fileUrl" },
+        "revision": { "type": ["string", "null"] },
+        "dirtyPatchDigest": {
+          "oneOf": [{ "$ref": "#/$defs/digest" }, { "type": "null" }]
+        }
+      },
+      "additionalProperties": false
+    },
+    "extension": {
+      "oneOf": [{ "type": "null" }, { "$ref": "#/$defs/extension" }]
+    },
+    "condition": {
+      "type": "object",
+      "required": [
+        "requested",
+        "actual",
+        "requestedInstrumentation",
+        "appliedInstrumentation"
+      ],
+      "properties": {
+        "requested": { "enum": ["passive", "enforced"] },
+        "actual": { "enum": ["passive", "enforced", "unknown"] },
+        "requestedInstrumentation": { "$ref": "#/$defs/instrumentationList" },
+        "appliedInstrumentation": { "$ref": "#/$defs/instrumentationList" }
+      },
+      "additionalProperties": false
+    },
+    "graders": {
+      "type": "object",
+      "required": ["active", "replacedDefaults"],
+      "properties": {
+        "active": { "type": "array", "items": { "$ref": "#/$defs/grader" } },
+        "replacedDefaults": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 },
+          "uniqueItems": true
+        }
+      },
+      "additionalProperties": false
+    },
+    "routes": { "type": "array", "items": { "$ref": "#/$defs/route" } },
+    "result": { "$ref": "urn:sevro:schema:cli-result:v1" },
+    "trials": { "type": "array", "items": { "$ref": "#/$defs/trialEvidence" } },
+    "diagnostic": { "$ref": "#/$defs/diagnostic" }
+  },
+  "additionalProperties": false,
+  "$defs": {
+    "digest": { "type": "string", "pattern": "^[a-f0-9]{64}$" },
+    "fileUrl": { "type": "string", "pattern": "^file:///" },
+    "runner": {
+      "oneOf": [
+        {
+          "type": "object",
+          "required": ["source", "packageName", "version", "buildDigest"],
+          "properties": {
+            "source": { "const": "package" },
+            "packageName": { "type": "string", "minLength": 1 },
+            "version": { "type": "string", "minLength": 1 },
+            "buildDigest": { "$ref": "#/$defs/digest" }
+          },
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "required": [
+            "source",
+            "root",
+            "revision",
+            "dirtyPatchDigest",
+            "buildDigest"
+          ],
+          "properties": {
+            "source": { "const": "checkout" },
+            "root": { "$ref": "#/$defs/fileUrl" },
+            "revision": { "type": "string", "minLength": 1 },
+            "dirtyPatchDigest": {
+              "oneOf": [{ "$ref": "#/$defs/digest" }, { "type": "null" }]
+            },
+            "buildDigest": { "$ref": "#/$defs/digest" }
+          },
+          "additionalProperties": false
+        }
+      ]
+    },
+    "extension": {
+      "type": "object",
+      "required": [
+        "id",
+        "version",
+        "sourceDigest",
+        "configurationDigest",
+        "protocol",
+        "capabilities",
+        "replacements"
+      ],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "version": { "type": "string", "minLength": 1 },
+        "sourceDigest": { "$ref": "#/$defs/digest" },
+        "configurationDigest": { "$ref": "#/$defs/digest" },
+        "protocol": { "const": "sevro.extension.v1" },
+        "capabilities": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 },
+          "uniqueItems": true
+        },
+        "replacements": {
+          "type": "object",
+          "required": ["graders", "taskVerdictPolicy"],
+          "properties": {
+            "graders": {
+              "type": "array",
+              "items": { "type": "string", "minLength": 1 },
+              "uniqueItems": true
+            },
+            "taskVerdictPolicy": { "type": ["string", "null"] }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+    "instrumentationList": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["id", "configuration"],
+        "properties": {
+          "id": { "type": "string", "minLength": 1 },
+          "configuration": { "type": "object" }
+        },
+        "additionalProperties": false
+      }
+    },
+    "grader": {
+      "type": "object",
+      "required": ["id", "source", "version"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "source": { "enum": ["builtin", "extension"] },
+        "version": { "type": "string", "minLength": 1 }
+      },
+      "additionalProperties": false
+    },
+    "route": {
+      "type": "object",
+      "required": ["role", "host", "model", "effort"],
+      "properties": {
+        "role": { "enum": ["candidate", "semantic", "advisory"] },
+        "host": { "type": "string", "minLength": 1 },
+        "model": { "type": "string", "minLength": 1 },
+        "effort": { "type": "string", "minLength": 1 }
+      },
+      "additionalProperties": false
+    },
+    "advisoryReview": {
+      "oneOf": [
+        { "type": "null" },
+        {
+          "type": "object",
+          "required": ["status", "assessment", "usage", "rawResult"],
+          "properties": {
+            "status": { "enum": ["completed", "failed", "not_run"] },
+            "assessment": {
+              "oneOf": [
+                { "type": "null" },
+                {
+                  "type": "object",
+                  "required": [
+                    "verdict",
+                    "overallScore",
+                    "dimensions",
+                    "strengths",
+                    "weaknesses",
+                    "summary"
+                  ],
+                  "properties": {
+                    "verdict": { "enum": ["pass", "fail"] },
+                    "overallScore": {
+                      "type": "integer",
+                      "minimum": 1,
+                      "maximum": 5
+                    },
+                    "dimensions": {
+                      "type": "object",
+                      "required": [
+                        "correctness",
+                        "maintainability",
+                        "testQuality",
+                        "scopeDiscipline"
+                      ],
+                      "properties": {
+                        "correctness": {
+                          "type": "integer",
+                          "minimum": 1,
+                          "maximum": 5
+                        },
+                        "maintainability": {
+                          "type": "integer",
+                          "minimum": 1,
+                          "maximum": 5
+                        },
+                        "testQuality": {
+                          "type": "integer",
+                          "minimum": 1,
+                          "maximum": 5
+                        },
+                        "scopeDiscipline": {
+                          "type": "integer",
+                          "minimum": 1,
+                          "maximum": 5
+                        }
+                      },
+                      "additionalProperties": false
+                    },
+                    "strengths": {
+                      "type": "array",
+                      "items": { "type": "string" }
+                    },
+                    "weaknesses": {
+                      "type": "array",
+                      "items": { "type": "string" }
+                    },
+                    "summary": { "type": "string", "minLength": 1 }
+                  },
+                  "additionalProperties": false
+                }
+              ]
+            },
+            "usage": {
+              "type": "object",
+              "required": [
+                "inputTokens",
+                "outputTokens",
+                "costUsd",
+                "complete"
+              ],
+              "properties": {
+                "inputTokens": { "type": ["integer", "null"], "minimum": 0 },
+                "outputTokens": { "type": ["integer", "null"], "minimum": 0 },
+                "costUsd": { "type": ["number", "null"], "minimum": 0 },
+                "complete": { "type": "boolean" }
+              },
+              "additionalProperties": false
+            },
+            "rawResult": {
+              "type": "object",
+              "required": ["source", "path", "sha256"],
+              "properties": {
+                "source": { "type": "string", "minLength": 1 },
+                "path": {
+                  "oneOf": [{ "$ref": "#/$defs/fileUrl" }, { "type": "null" }]
+                },
+                "sha256": {
+                  "oneOf": [{ "$ref": "#/$defs/digest" }, { "type": "null" }]
+                }
+              },
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        }
+      ]
+    },
+    "trialEvidence": {
+      "type": "object",
+      "required": [
+        "caseId",
+        "trial",
+        "executionMode",
+        "condition",
+        "observationCompleteness",
+        "observations",
+        "metrics",
+        "taskVerdictPolicy",
+        "routes",
+        "usage",
+        "rawResult",
+        "artifactRefs"
+      ],
+      "properties": {
+        "caseId": { "type": "string", "minLength": 1 },
+        "trial": { "type": "integer", "minimum": 1 },
+        "executionMode": { "enum": ["executed", "dry", "unknown"] },
+        "candidateDurationMs": { "type": ["number", "null"], "minimum": 0 },
+        "condition": {
+          "type": "object",
+          "required": ["requested", "actual", "appliedInstrumentation"],
+          "properties": {
+            "requested": { "enum": ["passive", "enforced"] },
+            "actual": { "enum": ["passive", "enforced", "unknown"] },
+            "appliedInstrumentation": { "$ref": "#/$defs/instrumentationList" }
+          },
+          "additionalProperties": false
+        },
+        "observationCompleteness": {
+          "enum": ["complete", "partial", "unavailable"]
+        },
+        "observations": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["id", "source", "completeness", "data"],
+            "properties": {
+              "id": { "type": "string", "minLength": 1 },
+              "source": { "type": "string", "minLength": 1 },
+              "completeness": {
+                "enum": ["complete", "partial", "unavailable"]
+              },
+              "data": { "type": "object" }
+            },
+            "additionalProperties": false
+          }
+        },
+        "metrics": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/metric" }
+        },
+        "domainOutcomes": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["id", "status", "evidenceRefs"],
+            "properties": {
+              "id": { "type": "string", "minLength": 1 },
+              "status": { "enum": ["passed", "failed", "unavailable"] },
+              "detail": { "type": "string", "maxLength": 4096 },
+              "evidenceRefs": {
+                "type": "array",
+                "items": { "type": "string", "minLength": 1 },
+                "uniqueItems": true
+              },
+              "data": { "type": "object" }
+            },
+            "additionalProperties": false
+          }
+        },
+        "taskVerdictPolicy": {
+          "oneOf": [
+            { "type": "null" },
+            {
+              "type": "object",
+              "required": ["id", "recommendation"],
+              "properties": {
+                "id": { "type": "string", "minLength": 1 },
+                "recommendation": {
+                  "enum": ["passed", "failed", "not_assessed", null]
+                }
+              },
+              "additionalProperties": false
+            }
+          ]
+        },
+        "advisoryReview": { "$ref": "#/$defs/advisoryReview" },
+        "routes": { "type": "array", "items": { "$ref": "#/$defs/route" } },
+        "usage": {
+          "type": "object",
+          "required": ["inputTokens", "outputTokens", "costUsd", "complete"],
+          "properties": {
+            "inputTokens": { "type": ["integer", "null"], "minimum": 0 },
+            "outputTokens": { "type": ["integer", "null"], "minimum": 0 },
+            "costUsd": { "type": ["number", "null"], "minimum": 0 },
+            "complete": { "type": "boolean" }
+          },
+          "additionalProperties": false
+        },
+        "rawResult": {
+          "type": "object",
+          "required": ["source", "path", "sha256"],
+          "properties": {
+            "source": { "type": "string", "minLength": 1 },
+            "path": {
+              "oneOf": [{ "$ref": "#/$defs/fileUrl" }, { "type": "null" }]
+            },
+            "sha256": {
+              "oneOf": [{ "$ref": "#/$defs/digest" }, { "type": "null" }]
+            }
+          },
+          "additionalProperties": false
+        },
+        "artifactRefs": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "required": ["id", "path", "sha256"],
+            "properties": {
+              "id": { "type": "string", "minLength": 1 },
+              "path": { "$ref": "#/$defs/fileUrl" },
+              "sha256": { "$ref": "#/$defs/digest" },
+              "gitExclude": { "type": "boolean" },
+              "executable": { "type": "boolean" }
+            },
+            "additionalProperties": false
+          }
+        }
+      },
+      "additionalProperties": false
+    },
+    "diagnostic": {
+      "type": "object",
+      "required": ["code", "message"],
+      "properties": {
+        "code": { "type": "string", "minLength": 1 },
+        "message": { "type": "string", "minLength": 1, "maxLength": 4096 }
+      },
+      "additionalProperties": false
+    },
+    "metric": {
+      "type": "object",
+      "required": ["id", "value", "unit"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "value": { "type": ["number", "null"] },
+        "unit": { "type": "string", "minLength": 1 }
+      },
+      "additionalProperties": false
+    }
+  }
+}