@skill-harness/core 0.8.0 → 0.10.0

package/schemas/results-v2.schema.json

@@ -0,0 +1,133 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://skill-harness.dev/schemas/results-v2.schema.json",
+  "title": "skill-harness results v2",
+  "type": "object",
+  "required": ["schema", "skill", "harness", "model", "judge", "timestamp", "label", "mode", "effective_grade", "scenarios"],
+  "properties": {
+    "schema": { "const": 2 },
+    "harness_version": { "type": "string" },
+    "harness_cli_version": { "type": "string" },
+    "delivery_canary": { "enum": ["pass", "skipped"] },
+    "skill": { "type": "string" },
+    "harness": { "type": "string" },
+    "model": { "type": "string" },
+    "judge": {
+      "type": "object",
+      "required": ["provider", "model"],
+      "properties": { "provider": { "type": "string" }, "model": { "type": "string" } }
+    },
+    "timestamp": { "type": "string" },
+    "label": { "type": ["string", "null"] },
+    "mode": { "enum": ["red", "green", "force"] },
+    "partial": { "type": "boolean" },
+    "source_hashes": { "type": "object", "additionalProperties": { "type": "string" } },
+    "effective_grade": { "$ref": "#/$defs/grade" },
+    "scenarios": { "type": "array", "items": { "$ref": "#/$defs/scenario" } }
+  },
+  "$defs": {
+    "verdict": { "enum": ["PASS", "FAIL", "ERROR", "JUDGE-AMBIGUOUS"] },
+    "grade": {
+      "type": "object",
+      "required": ["passed", "total", "pct", "letter", "ship", "note"],
+      "properties": {
+        "passed": { "type": "integer" }, "total": { "type": "integer" }, "pct": { "type": "number" },
+        "letter": { "type": "string" }, "ship": { "type": "boolean" }, "note": { "type": "string" }
+      }
+    },
+    "metrics": {
+      "type": "object",
+      "required": ["wall_time_ms", "judge_calls", "judge_rejudge_calls", "subject_metrics_reps", "total_reps"],
+      "properties": {
+        "wall_time_ms": { "type": "number", "minimum": 0 },
+        "judge_calls": { "type": "integer", "minimum": 0 },
+        "judge_rejudge_calls": { "type": "integer", "minimum": 0 },
+        "subject_metrics_reps": { "type": "integer", "minimum": 0 },
+        "total_reps": { "type": "integer", "minimum": 1 },
+        "input_tokens": { "type": "integer", "minimum": 0 },
+        "output_tokens": { "type": "integer", "minimum": 0 },
+        "cache_read_tokens": { "type": "integer", "minimum": 0 },
+        "cache_write_tokens": { "type": "integer", "minimum": 0 },
+        "subject_cost_usd": { "type": "number", "minimum": 0 },
+        "tool_calls": { "type": "integer", "minimum": 0 },
+        "delegated_children": { "type": "integer", "minimum": 0 },
+        "max_concurrency": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "scenario": {
+      "type": "object",
+      "required": ["id", "judge_verdict", "judge_reason", "suspect", "override", "note"],
+      "properties": {
+        "id": { "type": "string" },
+        "judge_verdict": { "$ref": "#/$defs/verdict" },
+        "judge_reason": { "type": "string" },
+        "suspect": { "type": "boolean" },
+        "override": { "oneOf": [{ "const": null }, { "enum": ["PASS", "FAIL"] }] },
+        "note": { "type": "string" },
+        "reps": { "type": "integer", "minimum": 1 },
+        "passes": { "type": "integer", "minimum": 0 },
+        "clean": { "type": "integer", "minimum": 0 },
+        "flakiness": { "type": "number", "minimum": 0, "maximum": 1 },
+        "pass_threshold": { "type": "number", "minimum": 0, "maximum": 1 },
+        "metrics": { "$ref": "#/$defs/metrics" },
+        "objective": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["status", "assertions"],
+          "properties": {
+            "status": { "enum": ["PASS", "FAIL", "ERROR"] },
+            "trace_version": { "type": "integer" },
+            "trace_sha256": { "type": "string" },
+            "trajectory_version": { "const": "1.0" },
+            "events_sha256": { "type": "string" },
+            "rep_events_sha256": { "type": "array", "items": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" } },
+            "rep_trace_sha256": { "type": "array", "items": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" } },
+            "assertions": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["kind", "status", "detail"],
+                "properties": {
+                  "kind": { "type": "string", "minLength": 1 },
+                  "status": { "enum": ["PASS", "FAIL", "ERROR"] },
+                  "detail": { "type": "string" }
+                }
+              }
+            }
+          }
+        },
+        "adjudication": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["state", "trigger", "judgments"],
+          "properties": {
+            "state": { "enum": ["confirmed", "tie_broken", "unresolved"] },
+            "trigger": { "enum": ["ambiguous", "contradictory", "non_unanimous", "ship_deciding"] },
+            "verdict": { "enum": ["PASS", "FAIL"] },
+            "judgments": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["ordinal", "judge", "verdict", "reason", "suspect"],
+                "properties": {
+                  "ordinal": { "type": "integer", "minimum": 1, "maximum": 3 },
+                  "judge": {
+                    "type": "object",
+                    "additionalProperties": false,
+                    "required": ["provider", "model"],
+                    "properties": { "provider": { "type": "string" }, "model": { "type": "string" } }
+                  },
+                  "verdict": { "$ref": "#/$defs/verdict" },
+                  "reason": { "type": "string" },
+                  "suspect": { "type": "boolean" }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/schemas/specification-v1.schema.json

@@ -0,0 +1,217 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://skill-harness.dev/schemas/specification-v1.schema.json",
+  "title": "skill-harness specification v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["skill", "judge_persona", "ship_bar", "scenarios"],
+  "properties": {
+    "schema": { "const": 1 },
+    "skill": { "type": "string", "minLength": 1 },
+    "judge_persona": { "type": "string", "minLength": 1 },
+    "ship_bar": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["total", "min_pass"],
+      "properties": {
+        "total": { "type": "integer", "minimum": 1 },
+        "min_pass": { "type": "integer", "minimum": 1 },
+        "no_critical_fail": { "type": "boolean" }
+      }
+    },
+    "critical": { "type": "array", "items": { "type": "string" }, "uniqueItems": true },
+    "adjudication": { "type": "object" },
+    "scenarios": { "type": "array", "items": { "$ref": "#/$defs/scenario" } }
+  },
+  "$defs": {
+    "stringList": { "type": "array", "items": { "type": "string", "minLength": 1 }, "minItems": 1, "uniqueItems": true },
+    "predicate": {
+      "oneOf": [
+        { "type": ["string", "number", "boolean"] },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "equals": {},
+            "contains": { "type": "string" },
+            "starts_with": { "type": "string" },
+            "ends_with": { "type": "string" },
+            "matches": { "type": "string" },
+            "exists": { "type": "boolean" },
+            "any": { "$ref": "#/$defs/predicate" }
+          },
+          "minProperties": 1
+        }
+      ]
+    },
+    "count": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "min": { "type": "integer", "minimum": 0 },
+        "max": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "selector": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["event"],
+      "properties": {
+        "event": { "type": "string", "minLength": 1 },
+        "where": { "type": "object", "additionalProperties": { "$ref": "#/$defs/predicate" } },
+        "select": { "enum": ["first", "last", "all"] }
+      }
+    },
+    "requiredSelector": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["event"],
+      "properties": {
+        "event": { "type": "string", "minLength": 1 },
+        "where": { "type": "object", "additionalProperties": { "$ref": "#/$defs/predicate" } },
+        "select": { "enum": ["first", "last", "all"] },
+        "count": { "$ref": "#/$defs/count" }
+      }
+    },
+    "correlation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["left", "right"],
+      "properties": {
+        "left": { "$ref": "#/$defs/selector" },
+        "right": { "$ref": "#/$defs/selector" },
+        "same": { "$ref": "#/$defs/stringList" },
+        "different": { "$ref": "#/$defs/stringList" },
+        "order": { "enum": ["before", "after"] }
+      },
+      "anyOf": [{ "required": ["same"] }, { "required": ["different"] }, { "required": ["order"] }]
+    },
+    "freshness": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["subject", "after"],
+      "properties": {
+        "subject": { "$ref": "#/$defs/selector" },
+        "after": { "type": "array", "items": { "$ref": "#/$defs/selector" }, "minItems": 1 },
+        "same": { "$ref": "#/$defs/stringList" }
+      }
+    },
+    "unique": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["events", "fields"],
+      "properties": {
+        "events": { "$ref": "#/$defs/selector" },
+        "fields": { "$ref": "#/$defs/stringList" }
+      }
+    },
+    "forbidAfter": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["anchor", "forbidden"],
+      "properties": {
+        "anchor": { "$ref": "#/$defs/selector" },
+        "forbidden": { "type": "array", "items": { "$ref": "#/$defs/selector" }, "minItems": 1 },
+        "same": { "$ref": "#/$defs/stringList" },
+        "anchor_optional": { "type": "boolean" }
+      }
+    },
+    "approvalAssertion": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["grant", "use"],
+      "properties": {
+        "grant": { "$ref": "#/$defs/selector" },
+        "use": { "$ref": "#/$defs/selector" },
+        "same": { "$ref": "#/$defs/stringList" },
+        "scopes": { "$ref": "#/$defs/stringList" },
+        "sources": { "$ref": "#/$defs/stringList" },
+        "unexpired": { "type": "boolean" },
+        "max_uses": { "type": "integer", "minimum": 1 }
+      }
+    },
+    "coverageAssertion": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["requirements"],
+      "properties": {
+        "requirements": { "$ref": "#/$defs/stringList" },
+        "events": { "$ref": "#/$defs/selector" }
+      }
+    },
+    "trajectory": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["version"],
+      "properties": {
+        "version": { "const": "1.0" },
+        "require": { "type": "array", "items": { "$ref": "#/$defs/requiredSelector" }, "minItems": 1 },
+        "forbid": { "type": "array", "items": { "$ref": "#/$defs/selector" }, "minItems": 1 },
+        "ordered": { "type": "array", "items": { "type": "array", "items": { "$ref": "#/$defs/selector" }, "minItems": 1 }, "minItems": 1 },
+        "correlate": { "type": "array", "items": { "$ref": "#/$defs/correlation" }, "minItems": 1 },
+        "freshness": { "type": "array", "items": { "$ref": "#/$defs/freshness" }, "minItems": 1 },
+        "unique": { "type": "array", "items": { "$ref": "#/$defs/unique" }, "minItems": 1 },
+        "forbid_after": { "type": "array", "items": { "$ref": "#/$defs/forbidAfter" }, "minItems": 1 },
+        "approvals": { "type": "array", "items": { "$ref": "#/$defs/approvalAssertion" }, "minItems": 1 },
+        "coverage": { "type": "array", "items": { "$ref": "#/$defs/coverageAssertion" }, "minItems": 1 }
+      },
+      "minProperties": 2
+    },
+    "scenario": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "title", "turns", "checklist"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "title": { "type": "string", "minLength": 1 },
+        "critical": { "type": "boolean" },
+        "mode": { "enum": ["inline", "seeded"] },
+        "fixture": { "type": "string" },
+        "turns": { "type": "array", "items": { "type": "string" }, "minItems": 1 },
+        "checklist": { "type": "array", "items": { "type": "string" }, "minItems": 1 },
+        "covers": { "type": "array", "items": { "type": "string" } },
+        "reps": { "type": "integer", "minimum": 1 },
+        "pass_threshold": { "type": "number", "minimum": 0, "maximum": 1 },
+        "system_prompt_file": { "type": "string" },
+        "env": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "workspace": {},
+            "remote": { "type": "boolean" },
+            "extensions": { "type": "array", "items": { "type": "string" } },
+            "event_sources": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "additionalProperties": false,
+                "required": ["adapter", "path"],
+                "properties": {
+                  "adapter": { "enum": ["normalized-v1", "principal-assurance-v1", "pi-daddy-v1"] },
+                  "path": {
+                    "type": "string",
+                    "minLength": 1,
+                    "pattern": "^(?!/)(?![A-Za-z]:[\\\\/])(?!.*\\\\)(?!.*//)(?!.*(?:^|/)\\.\\.?(?:/|$)).+$"
+                  },
+                  "required": { "type": "boolean" }
+                }
+              }
+            }
+          }
+        },
+        "assert": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "vitest": { "type": "boolean" },
+            "diff_contains": { "type": "array", "items": { "type": "string", "minLength": 1 } },
+            "diff_excludes": { "type": "array", "items": { "type": "string", "minLength": 1 } },
+            "post_test": { "type": "string" },
+            "trace": { "type": "object" },
+            "trajectory": { "$ref": "#/$defs/trajectory" }
+          }
+        }
+      }
+    }
+  }
+}

package/schemas/trajectory-event-v1.schema.json

@@ -0,0 +1,61 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://skill-harness.dev/schemas/trajectory-event-v1.schema.json",
+  "title": "skill-harness normalized trajectory event v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["event_version", "seq", "type", "source"],
+  "properties": {
+    "event_version": { "const": "1.0" },
+    "seq": { "type": "integer", "minimum": 1 },
+    "type": { "type": "string", "minLength": 1 },
+    "source": { "type": "string", "minLength": 1 },
+    "at": { "type": "string", "format": "date-time" },
+    "run_id": { "$ref": "#/$defs/id" },
+    "task_id": { "$ref": "#/$defs/id" },
+    "workspace_id": { "$ref": "#/$defs/id" },
+    "context_id": { "$ref": "#/$defs/id" },
+    "finding_id": { "$ref": "#/$defs/id" },
+    "parent_id": { "$ref": "#/$defs/id" },
+    "child_id": { "$ref": "#/$defs/id" },
+    "phase": { "type": "string", "minLength": 1 },
+    "tool": { "type": "string", "minLength": 1 },
+    "capability": { "type": "string", "minLength": 1 },
+    "requested_capabilities": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
+    "effective_capabilities": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
+    "refusal_code": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]*$" },
+    "exit_code": { "type": "integer" },
+    "digests": {
+      "type": "object",
+      "properties": {
+        "plan": { "$ref": "#/$defs/sha256" },
+        "task": { "$ref": "#/$defs/sha256" },
+        "definition": { "$ref": "#/$defs/sha256" },
+        "head": { "$ref": "#/$defs/gitSha" },
+        "tree": { "$ref": "#/$defs/gitSha" }
+      },
+      "additionalProperties": { "type": "string" }
+    },
+    "approval": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "id": { "$ref": "#/$defs/id" },
+        "capability": { "type": "string", "minLength": 1 },
+        "subject": { "type": "string", "minLength": 1 },
+        "source": { "type": "string", "minLength": 1 },
+        "scope": { "type": "string", "minLength": 1 },
+        "approved_at": { "type": "string", "format": "date-time" },
+        "expires_at": { "type": "string", "format": "date-time" },
+        "used_at": { "type": "string", "format": "date-time" }
+      }
+    },
+    "requirements": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
+    "attributes": { "type": "object" }
+  },
+  "$defs": {
+    "id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$" },
+    "sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" },
+    "gitSha": { "type": "string", "pattern": "^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$" }
+  }
+}