agent-hitch 0.2.7 → 0.2.8

package/docs/schemas/eval-trial-publication.schema.json

@@ -9,31 +9,7 @@
     "schema_version": { "const": "1" },
     "eval_id": { "type": "string", "pattern": "^eval_[a-f0-9]{32}$" },
     "mode": { "enum": ["settle", "replace-invalid"] },
-    "trial": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["trial_id", "run_id", "task_id", "attempt", "observation_status"],
-      "properties": {
-        "trial_id": { "type": "string", "minLength": 1 },
-        "run_id": { "type": "string", "pattern": "^run_[a-f0-9]{32}$" },
-        "task_id": { "type": "string", "minLength": 1 },
-        "attempt": { "type": "integer", "minimum": 1 },
-        "observation_status": { "enum": ["valid", "invalid"] },
-        "reward": { "type": "number" },
-        "verifier_result_ref": { "type": "string", "minLength": 1 },
-        "invalid_reason": { "type": "string", "minLength": 1 }
-      },
-      "allOf": [
-        {
-          "if": { "properties": { "observation_status": { "const": "valid" } } },
-          "then": { "required": ["reward"] }
-        },
-        {
-          "if": { "properties": { "observation_status": { "const": "invalid" } } },
-          "then": { "required": ["invalid_reason"] }
-        }
-      ]
-    },
+    "trial": { "$ref": "eval-trial-reference.schema.json" },
     "created_at": { "type": "string", "format": "date-time" }
   }
 }

package/docs/schemas/eval-trial-reference.schema.json

@@ -0,0 +1,129 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-hitch.local/schemas/eval-trial-reference.schema.json",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "trial_id",
+    "task_id",
+    "attempt",
+    "observation_status"
+  ],
+  "properties": {
+    "trial_id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "run_id": {
+      "type": "string",
+      "pattern": "^run_[a-f0-9]{32}$"
+    },
+    "task_id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "attempt": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "observation_status": {
+      "enum": [
+        "valid",
+        "invalid"
+      ]
+    },
+    "reward": {
+      "type": "number"
+    },
+    "scores": {
+      "$ref": "verifier-evidence.schema.json#/$defs/scores"
+    },
+    "verifier_result_ref": {
+      "type": "string",
+      "minLength": 1
+    },
+    "invalid_reason": {
+      "type": "string",
+      "minLength": 1
+    },
+    "assessment": {
+      "$ref": "regrade-assessment-reference.schema.json"
+    },
+    "run_group": {
+      "type": "object",
+      "required": [
+        "run_group_id",
+        "digest"
+      ],
+      "properties": {
+        "run_group_id": {
+          "type": "string",
+          "pattern": "^run_group_[a-f0-9]{32}$"
+        },
+        "digest": {
+          "type": "string",
+          "pattern": "^sha256:[a-f0-9]{64}$"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "oneOf": [
+    {
+      "required": [
+        "run_id"
+      ],
+      "not": {
+        "required": [
+          "run_group"
+        ]
+      }
+    },
+    {
+      "required": [
+        "run_group",
+        "assessment"
+      ],
+      "not": {
+        "required": [
+          "run_id"
+        ]
+      }
+    }
+  ],
+  "allOf": [
+    {
+      "if": {
+        "properties": {
+          "observation_status": {
+            "const": "valid"
+          }
+        }
+      },
+      "then": {
+        "required": [
+          "reward"
+        ]
+      }
+    },
+    {
+      "if": {
+        "properties": {
+          "observation_status": {
+            "const": "invalid"
+          }
+        }
+      },
+      "then": {
+        "required": [
+          "invalid_reason"
+        ],
+        "not": {
+          "required": [
+            "scores"
+          ]
+        }
+      }
+    }
+  ]
+}

package/docs/schemas/regrade-assessment-reference.schema.json

@@ -0,0 +1,11 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Hitch immutable verifier-only assessment reference",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["id", "digest"],
+  "properties": {
+    "id": { "type": "string", "pattern": "^assessment_[a-f0-9]{32}$" },
+    "digest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" }
+  }
+}

package/docs/schemas/run-context.schema.json

@@ -34,6 +34,21 @@
         "task_digest": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
         "verifier_identity": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }
       }
+    },
+    {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["kind", "benchmark_id", "benchmark_revision", "task_id", "task_digest", "verifier_identity", "run_group_id", "phase_index"],
+      "properties": {
+        "kind": { "const": "benchmark_phase" },
+        "benchmark_id": { "type": "string", "minLength": 1 },
+        "benchmark_revision": { "type": "string", "minLength": 1, "not": { "pattern": "^[Ll][Aa][Tt][Ee][Ss][Tt]$" } },
+        "task_id": { "type": "string", "minLength": 1 },
+        "task_digest": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
+        "verifier_identity": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
+        "run_group_id": { "type": "string", "pattern": "^run_group_[a-f0-9]{32}$" },
+        "phase_index": { "type": "integer", "minimum": 1, "maximum": 9007199254740991 }
+      }
     }
   ]
 }

package/docs/schemas/run-manifest.schema.json

@@ -5,6 +5,10 @@
   "type": "object",
   "additionalProperties": true,
   "required": ["schema_version", "run_id", "context", "status", "harness", "model", "protocol", "request_ref", "resolution_ref", "created_at"],
+  "allOf": [{
+    "if": { "properties": { "context": { "properties": { "kind": { "const": "benchmark_phase" } }, "required": ["kind"] } }, "required": ["context"] },
+    "then": { "required": ["parent"], "not": { "required": ["observation"] } }
+  }],
   "properties": {
     "schema_version": { "const": "1" },
     "run_id": { "type": "string", "pattern": "^run_[a-f0-9]{32}$" },

package/docs/schemas/run-request.schema.json

@@ -5,6 +5,10 @@
   "type": "object",
   "additionalProperties": false,
   "required": ["prompt"],
+  "allOf": [{
+    "if": { "properties": { "context": { "properties": { "kind": { "const": "benchmark_phase" } }, "required": ["kind"] } }, "required": ["context"] },
+    "then": { "required": ["parent"], "properties": { "defer_benchmark_observation": { "const": false } } }
+  }],
   "oneOf": [
     {
       "required": ["harness_ref"],

package/docs/schemas/trajectory-analysis.schema.json

@@ -0,0 +1,258 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-hitch.local/schemas/trajectory-analysis.schema.json",
+  "title": "Hitch bounded trajectory analysis",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "kind", "run_id", "source", "header", "surface", "events", "chunk_summaries", "omitted_event_types", "coverage"],
+  "properties": {
+    "schema_version": { "const": "1" },
+    "kind": { "const": "trajectory-analysis" },
+    "run_id": { "$ref": "#/$defs/run_id" },
+    "source": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["fidelity", "session_id", "canonical_sha256", "canonical_bytes", "event_count", "event_types"],
+      "properties": {
+        "fidelity": { "enum": ["provider_native", "normalized", "minimal"] },
+        "provider": { "type": "string", "minLength": 1 },
+        "session_id": { "type": "string", "minLength": 1 },
+        "canonical_sha256": { "$ref": "#/$defs/sha256" },
+        "canonical_bytes": { "type": "integer", "minimum": 0 },
+        "event_count": { "type": "integer", "minimum": 0 },
+        "event_types": { "$ref": "#/$defs/counts" }
+      }
+    },
+    "header": true,
+    "surface": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["fidelity", "nodes", "current_node_seqs", "replacements", "request_boundaries", "request_headers"],
+      "properties": {
+        "fidelity": { "enum": ["exact", "normalized", "partial"] },
+        "nodes": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["seq", "event_type", "surface_op", "message"],
+            "properties": {
+              "seq": { "$ref": "#/$defs/seq" },
+              "event_type": { "enum": ["user/message", "assistant/message", "tool/result"] },
+              "surface_op": { "$ref": "#/$defs/surface_op" },
+              "message": true
+            }
+          }
+        },
+        "current_node_seqs": { "type": "array", "items": { "$ref": "#/$defs/seq" } },
+        "replacements": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["seq", "start", "end", "shadowed_seqs"],
+            "properties": {
+              "seq": { "$ref": "#/$defs/seq" },
+              "start": { "$ref": "#/$defs/seq" },
+              "end": { "$ref": "#/$defs/seq" },
+              "shadowed_seqs": { "type": "array", "items": { "$ref": "#/$defs/seq" } }
+            }
+          }
+        },
+        "request_boundaries": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["turn", "step", "attempt", "boundary_seq", "surface_revision"],
+            "properties": {
+              "turn": { "$ref": "#/$defs/seq" },
+              "step": { "$ref": "#/$defs/seq" },
+              "attempt": { "$ref": "#/$defs/seq" },
+              "retry_id": { "$ref": "#/$defs/projected_string" },
+              "boundary_seq": { "$ref": "#/$defs/seq" },
+              "surface_revision": { "$ref": "#/$defs/seq" },
+              "request_header_seq": { "$ref": "#/$defs/seq" }
+            }
+          }
+        },
+        "request_headers": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["seq", "header"],
+            "properties": {
+              "seq": { "$ref": "#/$defs/seq" },
+              "header": true
+            }
+          }
+        }
+      }
+    },
+    "events": { "type": "array", "items": { "$ref": "#/$defs/diagnostic_event" } },
+    "chunk_summaries": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["turn", "step", "attempt", "first_seq", "last_seq", "count", "types", "model_boundary_seq"],
+        "properties": {
+          "turn": { "$ref": "#/$defs/seq" },
+          "step": { "$ref": "#/$defs/seq" },
+          "attempt": { "$ref": "#/$defs/seq" },
+          "retry_id": { "$ref": "#/$defs/projected_string" },
+          "first_seq": { "$ref": "#/$defs/seq" },
+          "last_seq": { "$ref": "#/$defs/seq" },
+          "count": { "type": "integer", "minimum": 1 },
+          "types": { "$ref": "#/$defs/counts" },
+          "model_boundary_seq": { "$ref": "#/$defs/seq" },
+          "usage": true,
+          "finish_reason": true,
+          "partial": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["status", "source_seq_count"],
+            "oneOf": [
+              { "required": ["content"] },
+              { "required": ["streams"] }
+            ],
+            "properties": {
+              "status": { "const": "incomplete" },
+              "content": { "$ref": "#/$defs/content_excerpt" },
+              "streams": {
+                "type": "array",
+                "minItems": 2,
+                "items": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": ["block_index", "block_start_seq", "kind", "content", "source_seq_count"],
+                  "properties": {
+                    "block_index": { "$ref": "#/$defs/seq" },
+                    "block_start_seq": { "$ref": "#/$defs/seq" },
+                    "kind": { "enum": ["text", "reasoning", "tool_arguments"] },
+                    "content": { "$ref": "#/$defs/content_excerpt" },
+                    "source_seq_count": { "type": "integer", "minimum": 0 }
+                  }
+                }
+              },
+              "source_seq_count": { "type": "integer", "minimum": 0 }
+            }
+          }
+        }
+      }
+    },
+    "omitted_event_types": { "$ref": "#/$defs/counts" },
+    "coverage": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["surface", "chunks", "content", "child_sessions"],
+      "properties": {
+        "surface": { "enum": ["complete", "partial"] },
+        "chunks": { "enum": ["coalesced", "omitted", "partial"] },
+        "content": { "enum": ["complete", "excerpted", "partial"] },
+        "child_sessions": { "enum": ["complete", "partial", "none", "unavailable"] }
+      }
+    },
+    "redactions": { "$ref": "#/$defs/redactions" }
+  },
+  "$defs": {
+    "run_id": { "type": "string", "pattern": "^run_[a-f0-9]{32}$" },
+    "sha256": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
+    "seq": { "type": "integer", "minimum": 0 },
+    "surface_op": {
+      "oneOf": [
+        { "const": "append" },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["op", "start", "end"],
+          "properties": {
+            "op": { "const": "replace" },
+            "start": { "$ref": "#/$defs/seq" },
+            "end": { "$ref": "#/$defs/seq" }
+          }
+        }
+      ]
+    },
+    "counts": { "type": "object", "additionalProperties": { "type": "integer", "minimum": 0 } },
+    "projected_string": {
+      "oneOf": [
+        { "type": "string" },
+        { "$ref": "#/$defs/content_excerpt" }
+      ]
+    },
+    "diagnostic_event": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["type", "seq", "time", "data"],
+          "properties": {
+            "type": { "type": "string", "minLength": 1, "maxLength": 1024 },
+            "seq": { "$ref": "#/$defs/seq" },
+            "time": { "type": "number" },
+            "data": true,
+            "ignorable": { "const": true },
+            "surfaceOp": { "$ref": "#/$defs/surface_op" },
+            "source_event_seqs_summary": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["count"],
+              "properties": {
+                "count": { "type": "integer", "minimum": 0 },
+                "first": { "$ref": "#/$defs/seq" },
+                "last": { "$ref": "#/$defs/seq" }
+              }
+            }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["type", "seq", "time", "event_excerpt"],
+          "properties": {
+            "type": { "type": "string", "minLength": 1, "maxLength": 1024 },
+            "seq": { "$ref": "#/$defs/seq" },
+            "time": { "type": "number" },
+            "event_excerpt": { "$ref": "#/$defs/content_excerpt" }
+          }
+        }
+      ]
+    },
+    "content_excerpt": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["preview", "bytes", "sha256", "truncated", "source"],
+      "properties": {
+        "preview": { "type": "string" },
+        "tail": { "type": "string" },
+        "bytes": { "type": "integer", "minimum": 0 },
+        "sha256": { "$ref": "#/$defs/sha256" },
+        "truncated": { "type": "boolean" },
+        "source": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["run_id", "seq", "field"],
+          "properties": {
+            "run_id": { "$ref": "#/$defs/run_id" },
+            "seq": { "$ref": "#/$defs/seq" },
+            "field": { "type": "string", "minLength": 1, "maxLength": 1024 }
+          }
+        }
+      }
+    },
+    "redactions": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["rule_id", "count"],
+        "properties": {
+          "rule_id": { "type": "string", "minLength": 1 },
+          "count": { "type": "integer", "minimum": 1 }
+        }
+      }
+    }
+  }
+}

package/docs/schemas/trajectory-events-page.schema.json

@@ -0,0 +1,40 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-hitch.local/schemas/trajectory-events-page.schema.json",
+  "title": "Hitch source-filtered trajectory event page",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "kind", "run_id", "canonical_sha256", "filter", "events", "total_matches", "eof"],
+  "properties": {
+    "schema_version": { "const": "1" },
+    "kind": { "const": "trajectory-events-page" },
+    "run_id": { "type": "string", "pattern": "^run_[a-f0-9]{32}$" },
+    "canonical_sha256": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
+    "filter": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "types": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "type": "string", "minLength": 1, "maxLength": 1024 } },
+        "seq_start": { "type": "integer", "minimum": 0 },
+        "seq_end": { "type": "integer", "minimum": 0 },
+        "field": { "type": "string", "minLength": 1, "maxLength": 1024 }
+      }
+    },
+    "events": { "type": "array" },
+    "total_matches": { "type": "integer", "minimum": 0 },
+    "next_cursor": { "type": "string", "minLength": 1 },
+    "eof": { "type": "boolean" },
+    "redactions": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["rule_id", "count"],
+        "properties": {
+          "rule_id": { "type": "string", "minLength": 1 },
+          "count": { "type": "integer", "minimum": 1 }
+        }
+      }
+    }
+  }
+}

package/docs/schemas/verifier-assessment.schema.json

@@ -0,0 +1,77 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "title": "Hitch artifact-only verifier assessment v1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "kind", "candidate_executes", "eval_id", "task_id", "attempt", "rerun_id", "source", "controller_runtime_id", "backend", "observation", "evidence_digest", "completed_at"],
+  "properties": {
+    "schema_version": { "const": "1" },
+    "kind": { "const": "verifier-only-assessment" },
+    "candidate_executes": { "const": false },
+    "eval_id": { "type": "string", "pattern": "^eval_[a-f0-9]{32}$" },
+    "task_id": { "type": "string", "minLength": 1 },
+    "attempt": { "type": "integer", "minimum": 1 },
+    "rerun_id": { "type": "string", "pattern": "^rerun_[a-f0-9]{32}$" },
+    "controller_runtime_id": { "$ref": "#/$defs/digest" },
+    "runtime_repair": {
+      "type": "object", "additionalProperties": false,
+      "required": ["schema_version", "kind", "source_runtime_id", "replacement_runtime_id", "path", "source_sha256", "replacement_sha256", "unchanged_file_count"],
+      "properties": {
+        "schema_version": { "const": "1" }, "kind": { "const": "verifier-environment-runtime-repair" },
+        "source_runtime_id": { "$ref": "#/$defs/digest" }, "replacement_runtime_id": { "$ref": "#/$defs/digest" },
+        "path": { "const": "integrations/harbor/hitch_harbor_environment.py" },
+        "source_sha256": { "$ref": "#/$defs/digest" }, "replacement_sha256": { "$ref": "#/$defs/digest" },
+        "unchanged_file_count": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "evidence_digest": { "$ref": "#/$defs/digest" },
+    "completed_at": { "type": "string", "format": "date-time" },
+    "source": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["trial_id", "run_id", "work_id", "backend_directory", "bundle_index_digest", "trial_tree_digest", "artifacts_digest", "task_tree_digest", "artifact_capture_at"],
+      "properties": {
+        "trial_id": { "type": "string", "minLength": 1 },
+        "run_id": { "type": "string", "pattern": "^run_[a-f0-9]{32}$" },
+        "controller_runtime_id": { "$ref": "#/$defs/digest" },
+        "work_id": { "type": "string", "pattern": "^work_[a-f0-9]{32}$" },
+        "backend_directory": { "type": "string", "minLength": 1 },
+        "bundle_index_digest": { "$ref": "#/$defs/digest" },
+        "trial_tree_digest": { "$ref": "#/$defs/digest" },
+        "artifacts_digest": { "$ref": "#/$defs/digest" },
+        "task_tree_digest": { "$ref": "#/$defs/digest" },
+        "artifact_capture_at": { "type": "string", "format": "date-time" }
+      }
+    },
+    "backend": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "version", "identity", "process_exit_code", "signal"],
+      "properties": {
+        "name": { "const": "harbor" },
+        "version": { "const": "0.21.0" },
+        "identity": { "type": "string" },
+        "process_exit_code": { "type": ["integer", "null"] },
+        "signal": { "type": ["string", "null"] }
+      }
+    },
+    "observation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["status"],
+      "properties": {
+        "status": { "enum": ["valid", "invalid"] },
+        "reward": { "type": "number" },
+        "invalid_reason": { "type": "string", "minLength": 1 },
+        "verifier_result_ref": { "type": "string", "minLength": 1 }
+      },
+      "allOf": [{
+        "if": { "properties": { "status": { "const": "valid" } } },
+        "then": { "required": ["reward", "verifier_result_ref"], "not": { "required": ["invalid_reason"] } },
+        "else": { "required": ["invalid_reason"] }
+      }]
+    }
+  },
+  "allOf": [{ "if": { "required": ["runtime_repair"] }, "then": { "properties": { "source": { "required": ["controller_runtime_id"] } } } }],
+  "$defs": { "digest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" } }
+}