@skill-harness/core 0.17.0 → 0.19.0

package/dist/trajectory-events.js

@@ -0,0 +1,119 @@
+export const LEGACY_TRAJECTORY_EVENT_VERSION = "1.0";
+export const TRAJECTORY_EVENT_VERSION = "1.1";
+export function deserializeTrajectoryEvents(text) {
+    const out = [];
+    try {
+        for (const line of text.split("\n").filter((entry) => entry.trim())) {
+            const event = JSON.parse(line);
+            if (validateEvent(event) !== null)
+                return null;
+            out.push(event);
+        }
+    }
+    catch {
+        return null;
+    }
+    if (!out.length)
+        return null;
+    const sequences = out.map((event) => event.seq).sort((a, b) => a - b);
+    if (new Set(sequences).size !== out.length || sequences.some((seq, index) => seq !== index + 1))
+        return null;
+    return out;
+}
+const V11_EVENT_KEYS = new Set(["execution_id", "parent_execution_id", "task_from_execution_id", "workflow_fact_id", "deadline_at"]);
+const EVENT_KEYS = new Set([
+    "event_version", "seq", "type", "source", "at", "run_id", "task_id", "workspace_id", "context_id", "finding_id",
+    "parent_id", "child_id", ...V11_EVENT_KEYS,
+    "phase", "tool", "capability", "requested_capabilities", "effective_capabilities", "refusal_code",
+    "exit_code", "digests", "approval", "requirements", "attributes",
+]);
+const APPROVAL_KEYS = new Set(["id", "capability", "subject", "source", "scope", "approved_at", "expires_at", "used_at"]);
+const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+const SHA256_RE = /^[a-fA-F0-9]{64}$/;
+const GIT_SHA_RE = /^(?:[a-fA-F0-9]{40}|[a-fA-F0-9]{64})$/;
+const REFUSAL_RE = /^[A-Z][A-Z0-9_]*$/;
+function validateEvent(event) {
+    if (!event || typeof event !== "object" || Array.isArray(event))
+        return "event must be an object";
+    const object = event;
+    const unknown = Object.keys(object).find((key) => !EVENT_KEYS.has(key));
+    if (unknown)
+        return `unknown field ${unknown}`;
+    if (event.event_version !== LEGACY_TRAJECTORY_EVENT_VERSION && event.event_version !== TRAJECTORY_EVENT_VERSION)
+        return `unsupported event_version ${String(event.event_version)}`;
+    if (event.event_version === LEGACY_TRAJECTORY_EVENT_VERSION) {
+        const versionedField = Object.keys(object).find((key) => V11_EVENT_KEYS.has(key));
+        if (versionedField)
+            return `${versionedField} requires event_version ${TRAJECTORY_EVENT_VERSION}`;
+    }
+    if (!Number.isInteger(event.seq) || event.seq < 1)
+        return "seq must be a positive integer";
+    if (typeof event.type !== "string" || !event.type)
+        return "type must be a non-empty string";
+    if (typeof event.source !== "string" || !event.source)
+        return "source must be a non-empty string";
+    if (event.at !== undefined && !validDate(event.at))
+        return "at must be an RFC 3339 date-time";
+    for (const field of ["run_id", "task_id", "workspace_id", "context_id", "finding_id", "parent_id", "child_id", "execution_id", "task_from_execution_id", "workflow_fact_id"]) {
+        if (event[field] !== undefined && (typeof event[field] !== "string" || !ID_RE.test(event[field])))
+            return `${field} is not a valid bounded identifier`;
+    }
+    if (event.parent_execution_id !== undefined && event.parent_execution_id !== null && (typeof event.parent_execution_id !== "string" || !ID_RE.test(event.parent_execution_id)))
+        return "parent_execution_id is not a valid bounded identifier or null";
+    if (event.deadline_at !== undefined && !validDate(event.deadline_at))
+        return "deadline_at must be an RFC 3339 date-time";
+    for (const field of ["phase", "tool", "capability"])
+        if (event[field] !== undefined && (typeof event[field] !== "string" || !event[field]))
+            return `${field} must be a non-empty string`;
+    for (const field of ["requested_capabilities", "effective_capabilities", "requirements"]) {
+        const values = event[field];
+        if (values !== undefined && (!Array.isArray(values) || values.some((value) => typeof value !== "string" || !value) || new Set(values).size !== values.length))
+            return `${field} must be an array of unique non-empty strings`;
+    }
+    if (event.refusal_code !== undefined && !REFUSAL_RE.test(event.refusal_code))
+        return "refusal_code is invalid";
+    if (event.exit_code !== undefined && !Number.isInteger(event.exit_code))
+        return "exit_code must be an integer";
+    if (event.digests !== undefined) {
+        if (!event.digests || typeof event.digests !== "object" || Array.isArray(event.digests))
+            return "digests must be an object";
+        for (const [key, value] of Object.entries(event.digests)) {
+            if (typeof value !== "string")
+                return `digests.${key} must be a string`;
+            if (["plan", "task", "definition"].includes(key) && !SHA256_RE.test(value))
+                return `digests.${key} must be sha256`;
+            if (["head", "tree"].includes(key) && !GIT_SHA_RE.test(value))
+                return `digests.${key} must be a git object id`;
+        }
+    }
+    if (event.approval !== undefined) {
+        if (!event.approval || typeof event.approval !== "object" || Array.isArray(event.approval))
+            return "approval must be an object";
+        const unknownApproval = Object.keys(event.approval).find((key) => !APPROVAL_KEYS.has(key));
+        if (unknownApproval)
+            return `approval.${unknownApproval} is unknown`;
+        for (const [key, value] of Object.entries(event.approval)) {
+            if (typeof value !== "string" || !value)
+                return `approval.${key} must be a non-empty string`;
+            if (key === "id" && !ID_RE.test(value))
+                return "approval.id is invalid";
+            if (["approved_at", "expires_at", "used_at"].includes(key) && !validDate(value))
+                return `approval.${key} must be an RFC 3339 date-time`;
+        }
+    }
+    if (event.attributes !== undefined && (!event.attributes || typeof event.attributes !== "object" || Array.isArray(event.attributes)))
+        return "attributes must be an object";
+    return null;
+}
+function validDate(value) {
+    if (typeof value !== "string")
+        return false;
+    const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/.exec(value);
+    if (!match || !Number.isFinite(Date.parse(value)))
+        return false;
+    const [, year, month, day, hour, minute, second] = match.map(Number);
+    if (month < 1 || month > 12 || hour > 23 || minute > 59 || second > 59)
+        return false;
+    return day >= 1 && day <= new Date(Date.UTC(year, month, 0)).getUTCDate();
+}
+//# sourceMappingURL=trajectory-events.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@skill-harness/core",
-    "version": "0.17.0",
+    "version": "0.19.0",
     "description": "skill-harness engine — spec, discover, run, LLM-judge grade, score, results (internal API)",
     "type": "module",
     "license": "MIT",

package/schemas/results-v2.schema.json

@@ -3,128 +3,447 @@
   "$id": "https://skill-harness.dev/schemas/results-v2.schema.json",
   "title": "skill-harness results v2",
   "type": "object",
-  "required": ["schema", "skill", "harness", "model", "judge", "timestamp", "label", "mode", "effective_grade", "scenarios"],
+  "required": [
+    "schema",
+    "skill",
+    "harness",
+    "model",
+    "judge",
+    "timestamp",
+    "label",
+    "mode",
+    "effective_grade",
+    "scenarios"
+  ],
   "properties": {
-    "schema": { "const": 2 },
-    "harness_version": { "type": "string" },
-    "harness_cli_version": { "type": "string" },
-    "delivery_canary": { "enum": ["pass", "skipped"] },
-    "skill": { "type": "string" },
-    "harness": { "type": "string" },
-    "model": { "type": "string" },
+    "schema": {
+      "const": 2
+    },
+    "harness_version": {
+      "type": "string"
+    },
+    "harness_cli_version": {
+      "type": "string"
+    },
+    "delivery_canary": {
+      "enum": [
+        "pass",
+        "skipped"
+      ]
+    },
+    "skill": {
+      "type": "string"
+    },
+    "harness": {
+      "type": "string"
+    },
+    "model": {
+      "type": "string"
+    },
     "judge": {
       "type": "object",
-      "required": ["provider", "model"],
-      "properties": { "provider": { "type": "string" }, "model": { "type": "string" } }
-    },
-    "timestamp": { "type": "string" },
-    "label": { "type": ["string", "null"] },
-    "mode": { "enum": ["red", "green", "force"] },
-    "partial": { "type": "boolean" },
-    "source_hashes": { "type": "object", "additionalProperties": { "type": "string" } },
-    "effective_grade": { "$ref": "#/$defs/grade" },
-    "scenarios": { "type": "array", "items": { "$ref": "#/$defs/scenario" } }
+      "required": [
+        "provider",
+        "model"
+      ],
+      "properties": {
+        "provider": {
+          "type": "string"
+        },
+        "model": {
+          "type": "string"
+        }
+      }
+    },
+    "timestamp": {
+      "type": "string"
+    },
+    "label": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "mode": {
+      "enum": [
+        "red",
+        "green",
+        "force"
+      ]
+    },
+    "partial": {
+      "type": "boolean"
+    },
+    "source_hashes": {
+      "type": "object",
+      "additionalProperties": {
+        "type": "string"
+      }
+    },
+    "effective_grade": {
+      "$ref": "#/$defs/grade"
+    },
+    "scenarios": {
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/scenario"
+      }
+    }
   },
   "$defs": {
-    "verdict": { "enum": ["PASS", "FAIL", "ERROR", "JUDGE-AMBIGUOUS"] },
+    "verdict": {
+      "enum": [
+        "PASS",
+        "FAIL",
+        "ERROR",
+        "JUDGE-AMBIGUOUS"
+      ]
+    },
     "grade": {
       "type": "object",
-      "required": ["passed", "total", "pct", "letter", "ship", "note"],
+      "required": [
+        "passed",
+        "total",
+        "pct",
+        "letter",
+        "ship",
+        "note"
+      ],
       "properties": {
-        "passed": { "type": "integer" }, "total": { "type": "integer" }, "pct": { "type": "number" },
-        "letter": { "type": "string" }, "ship": { "type": "boolean" }, "note": { "type": "string" }
+        "passed": {
+          "type": "integer"
+        },
+        "total": {
+          "type": "integer"
+        },
+        "pct": {
+          "type": "number"
+        },
+        "letter": {
+          "type": "string"
+        },
+        "ship": {
+          "type": "boolean"
+        },
+        "note": {
+          "type": "string"
+        }
+      }
+    },
+    "usage": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["repetition", "inputTokens", "outputTokens", "cacheReadTokens", "costUsd", "priceAsOf"],
+      "properties": {
+        "repetition": { "type": "integer", "minimum": 0 },
+        "inputTokens": { "type": ["integer", "null"], "minimum": 1 },
+        "outputTokens": { "type": ["integer", "null"], "minimum": 1 },
+        "cacheReadTokens": { "type": ["integer", "null"], "minimum": 1 },
+        "costUsd": { "type": ["number", "null"], "exclusiveMinimum": 0 },
+        "priceAsOf": { "type": ["string", "null"] }
       }
     },
     "metrics": {
       "type": "object",
-      "required": ["wall_time_ms", "judge_calls", "judge_rejudge_calls", "subject_metrics_reps", "total_reps"],
+      "required": [
+        "wall_time_ms",
+        "judge_calls",
+        "judge_rejudge_calls",
+        "subject_metrics_reps",
+        "total_reps"
+      ],
       "properties": {
-        "wall_time_ms": { "type": "number", "minimum": 0 },
-        "judge_calls": { "type": "integer", "minimum": 0 },
-        "judge_rejudge_calls": { "type": "integer", "minimum": 0 },
-        "subject_metrics_reps": { "type": "integer", "minimum": 0 },
-        "total_reps": { "type": "integer", "minimum": 1 },
-        "input_tokens": { "type": "integer", "minimum": 0 },
-        "output_tokens": { "type": "integer", "minimum": 0 },
-        "cache_read_tokens": { "type": "integer", "minimum": 0 },
-        "cache_write_tokens": { "type": "integer", "minimum": 0 },
-        "subject_cost_usd": { "type": "number", "minimum": 0 },
-        "cost_source": { "enum": ["provider-reported", "subscription", "unreported"] },
-        "tool_calls": { "type": "integer", "minimum": 0 },
-        "delegated_children": { "type": "integer", "minimum": 0 },
-        "max_concurrency": { "type": "integer", "minimum": 0 }
+        "wall_time_ms": {
+          "type": "number",
+          "minimum": 0
+        },
+        "judge_calls": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "judge_rejudge_calls": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "subject_metrics_reps": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "total_reps": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "input_tokens": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "output_tokens": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "cache_read_tokens": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "cache_write_tokens": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "subject_cost_usd": {
+          "type": "number",
+          "minimum": 0
+        },
+        "cost_source": {
+          "enum": [
+            "provider-reported",
+            "subscription",
+            "price-table",
+            "unreported"
+          ]
+        },
+        "tool_calls": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "delegated_children": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "max_concurrency": {
+          "type": "integer",
+          "minimum": 0
+        }
       }
     },
     "judgment": {
       "type": "object",
       "additionalProperties": false,
-      "required": ["ordinal", "judge", "verdict", "reason", "suspect"],
+      "required": [
+        "ordinal",
+        "judge",
+        "verdict",
+        "reason",
+        "suspect"
+      ],
       "properties": {
-        "ordinal": { "type": "integer", "minimum": 1, "maximum": 3 },
+        "ordinal": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 3
+        },
         "judge": {
-          "type": "object", "additionalProperties": false, "required": ["provider", "model"],
-          "properties": { "provider": { "type": "string" }, "model": { "type": "string" } }
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "provider",
+            "model"
+          ],
+          "properties": {
+            "provider": {
+              "type": "string"
+            },
+            "model": {
+              "type": "string"
+            }
+          }
+        },
+        "verdict": {
+          "$ref": "#/$defs/verdict"
         },
-        "verdict": { "$ref": "#/$defs/verdict" },
-        "reason": { "type": "string" },
-        "suspect": { "type": "boolean" }
+        "reason": {
+          "type": "string"
+        },
+        "suspect": {
+          "type": "boolean"
+        }
       }
     },
     "scenario": {
       "type": "object",
-      "required": ["id", "judge_verdict", "judge_reason", "suspect", "override", "note"],
+      "required": [
+        "id",
+        "judge_verdict",
+        "judge_reason",
+        "suspect",
+        "override",
+        "note"
+      ],
       "properties": {
-        "id": { "type": "string" },
-        "judge_verdict": { "$ref": "#/$defs/verdict" },
-        "judge_reason": { "type": "string" },
-        "suspect": { "type": "boolean" },
-        "override": { "oneOf": [{ "const": null }, { "enum": ["PASS", "FAIL"] }] },
-        "note": { "type": "string" },
-        "reps": { "type": "integer", "minimum": 1 },
-        "passes": { "type": "integer", "minimum": 0 },
-        "clean": { "type": "integer", "minimum": 0 },
-        "flakiness": { "type": "number", "minimum": 0, "maximum": 1 },
-        "pass_threshold": { "type": "number", "minimum": 0, "maximum": 1 },
-        "metrics": { "$ref": "#/$defs/metrics" },
+        "id": {
+          "type": "string"
+        },
+        "judge_verdict": {
+          "$ref": "#/$defs/verdict"
+        },
+        "judge_reason": {
+          "type": "string"
+        },
+        "suspect": {
+          "type": "boolean"
+        },
+        "override": {
+          "oneOf": [
+            {
+              "const": null
+            },
+            {
+              "enum": [
+                "PASS",
+                "FAIL"
+              ]
+            }
+          ]
+        },
+        "note": {
+          "type": "string"
+        },
+        "reps": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "passes": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "clean": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "flakiness": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "pass_threshold": {
+          "type": "number",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "metrics": {
+          "$ref": "#/$defs/metrics"
+        },
+        "usage": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/usage" }
+        },
         "objective": {
           "type": "object",
           "additionalProperties": false,
-          "required": ["status", "assertions"],
+          "required": [
+            "status",
+            "assertions"
+          ],
           "properties": {
-            "status": { "enum": ["PASS", "FAIL", "ERROR"] },
-            "trace_version": { "type": "integer" },
-            "trace_sha256": { "type": "string" },
-            "trajectory_version": { "const": "1.0" },
-            "events_sha256": { "type": "string" },
-            "rep_events_sha256": { "type": "array", "items": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" } },
-            "rep_trace_sha256": { "type": "array", "items": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" } },
+            "status": {
+              "enum": [
+                "PASS",
+                "FAIL",
+                "ERROR"
+              ]
+            },
+            "trace_version": {
+              "type": "integer"
+            },
+            "trace_sha256": {
+              "type": "string"
+            },
+            "rep_trace_sha256": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "pattern": "^[a-fA-F0-9]{64}$"
+              }
+            },
             "assertions": {
               "type": "array",
               "items": {
                 "type": "object",
                 "additionalProperties": false,
-                "required": ["kind", "status", "detail"],
+                "required": [
+                  "kind",
+                  "status",
+                  "detail"
+                ],
                 "properties": {
-                  "kind": { "type": "string", "minLength": 1 },
-                  "status": { "enum": ["PASS", "FAIL", "ERROR"] },
-                  "detail": { "type": "string" }
+                  "kind": {
+                    "type": "string",
+                    "minLength": 1
+                  },
+                  "status": {
+                    "enum": [
+                      "PASS",
+                      "FAIL",
+                      "ERROR"
+                    ]
+                  },
+                  "detail": {
+                    "type": "string"
+                  }
                 }
               }
+            },
+            "trajectory_version": {
+              "type": "string"
+            },
+            "events_sha256": {
+              "type": "string",
+              "pattern": "^[a-fA-F0-9]{64}$"
+            },
+            "rep_events_sha256": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "pattern": "^[a-fA-F0-9]{64}$"
+              }
             }
           }
         },
         "adjudication": {
           "type": "object",
           "additionalProperties": false,
-          "required": ["state", "trigger", "judgments"],
+          "required": [
+            "state",
+            "trigger",
+            "judgments"
+          ],
           "properties": {
-            "state": { "enum": ["confirmed", "tie_broken", "unresolved"] },
-            "trigger": { "enum": ["ambiguous", "contradictory", "non_unanimous", "ship_deciding"] },
-            "verdict": { "enum": ["PASS", "FAIL"] },
-            "judgments": { "type": "array", "items": { "$ref": "#/$defs/judgment" } }
+            "state": {
+              "enum": [
+                "confirmed",
+                "tie_broken",
+                "unresolved"
+              ]
+            },
+            "trigger": {
+              "enum": [
+                "ambiguous",
+                "contradictory",
+                "non_unanimous",
+                "ship_deciding"
+              ]
+            },
+            "verdict": {
+              "enum": [
+                "PASS",
+                "FAIL"
+              ]
+            },
+            "judgments": {
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/judgment"
+              }
+            }
           }
         },
-        "judge_history": { "type": "array", "minItems": 2, "maxItems": 3, "items": { "$ref": "#/$defs/judgment" } }
+        "judge_history": {
+          "type": "array",
+          "minItems": 2,
+          "maxItems": 3,
+          "items": {
+            "$ref": "#/$defs/judgment"
+          }
+        }
       }
     }
   }