agent-hitch 0.2.7 → 0.2.8

package/docs/schemas/benchmark-profile-v1.schema.json

@@ -0,0 +1,141 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://hitch.dev/schemas/benchmark-profile-v1.schema.json",
+  "type": "object",
+  "properties": {
+    "schema_version": {
+      "const": "1"
+    },
+    "id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "track": {
+      "enum": [
+        "custom",
+        "public-subset"
+      ]
+    },
+    "input_mode": {
+      "const": "instruction"
+    },
+    "tool_policy": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "allowed": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "minLength": 1
+          },
+          "uniqueItems": true
+        },
+        "network": {
+          "const": "open"
+        },
+        "enforcement": {
+          "const": "required"
+        }
+      },
+      "required": [
+        "id",
+        "allowed",
+        "network",
+        "enforcement"
+      ],
+      "additionalProperties": false
+    },
+    "budget": {
+      "type": "object",
+      "properties": {
+        "agent_timeout": {
+          "type": "object",
+          "properties": {
+            "source": {
+              "const": "task"
+            }
+          },
+          "required": [
+            "source"
+          ],
+          "additionalProperties": false
+        },
+        "setup_timeout_ms": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "collection_timeout_ms": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "cleanup_grace_ms": {
+          "type": "integer",
+          "minimum": 1
+        }
+      },
+      "required": [
+        "agent_timeout",
+        "setup_timeout_ms",
+        "collection_timeout_ms",
+        "cleanup_grace_ms"
+      ],
+      "additionalProperties": false
+    },
+    "sampling": {
+      "type": "object",
+      "properties": {
+        "attempts_per_task": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "seed": {
+          "type": "integer"
+        }
+      },
+      "required": [
+        "attempts_per_task",
+        "seed"
+      ],
+      "additionalProperties": false
+    },
+    "grading": {
+      "type": "object",
+      "properties": {
+        "on_agent_budget_exhausted": {
+          "const": "grade_final_state"
+        },
+        "on_missing_submission": {
+          "const": "error"
+        },
+        "infrastructure_retries": {
+          "type": "integer",
+          "minimum": 0
+        }
+      },
+      "required": [
+        "on_agent_budget_exhausted",
+        "on_missing_submission",
+        "infrastructure_retries"
+      ],
+      "additionalProperties": false
+    },
+    "extensions": {
+      "type": "object"
+    }
+  },
+  "required": [
+    "schema_version",
+    "id",
+    "track",
+    "input_mode",
+    "tool_policy",
+    "budget",
+    "sampling",
+    "grading"
+  ],
+  "additionalProperties": false
+}

package/docs/schemas/benchmark-task-v1.schema.json

@@ -0,0 +1,384 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://hitch.dev/schemas/benchmark-task-v1.schema.json",
+  "type": "object",
+  "properties": {
+    "schema_version": {
+      "const": "1"
+    },
+    "source_task_id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "driver": {
+      "oneOf": [
+        {
+          "type": "object",
+          "properties": {
+            "kind": {
+              "const": "tool-server"
+            },
+            "protocol_version": {
+              "const": "1"
+            },
+            "config": {
+              "type": "object",
+              "properties": {
+                "transport": {
+                  "const": "http-json-cli"
+                },
+                "endpoint": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "schema": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "service": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "native_phases": {
+                  "type": "object",
+                  "additionalProperties": false,
+                  "required": [
+                    "protocol",
+                    "argv",
+                    "audit_path",
+                    "shutdown_timeout_ms"
+                  ],
+                  "properties": {
+                    "protocol": {
+                      "enum": ["hitch-native-phase-control@1", "hitch-native-phase-control@2"]
+                    },
+                    "argv": {
+                      "type": "array",
+                      "minItems": 1,
+                      "maxItems": 32,
+                      "items": {
+                        "type": "string",
+                        "minLength": 1,
+                        "maxLength": 8192
+                      }
+                    },
+                    "audit_path": {
+                      "type": "string",
+                      "pattern": "^/"
+                    },
+                    "shutdown_timeout_ms": {
+                      "type": "integer",
+                      "minimum": 1,
+                      "maximum": 600000
+                    },
+                    "finalization_timeout_ms": {
+                      "type": "integer",
+                      "minimum": 1
+                    }
+                  },
+                  "if": { "properties": { "protocol": { "const": "hitch-native-phase-control@2" } } },
+                  "then": { "required": ["finalization_timeout_ms"] },
+                  "else": { "not": { "required": ["finalization_timeout_ms"] } }
+                }
+              },
+              "required": [
+                "transport",
+                "endpoint",
+                "schema",
+                "service"
+              ],
+              "additionalProperties": false
+            }
+          },
+          "required": [
+            "kind",
+            "protocol_version",
+            "config"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "kind": {
+              "const": "terminal"
+            },
+            "protocol_version": {
+              "const": "1"
+            },
+            "config": {
+              "type": "object",
+              "maxProperties": 0
+            }
+          },
+          "required": [
+            "kind",
+            "protocol_version",
+            "config"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "kind": {
+              "const": "model-call"
+            },
+            "protocol_version": {
+              "const": "1"
+            },
+            "config": {
+              "type": "object",
+              "properties": {
+                "input": {
+                  "type": "string",
+                  "minLength": 1
+                }
+              },
+              "required": [
+                "input"
+              ],
+              "additionalProperties": false
+            }
+          },
+          "required": [
+            "kind",
+            "protocol_version",
+            "config"
+          ],
+          "additionalProperties": false
+        }
+      ]
+    },
+    "requirements": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1
+      },
+      "uniqueItems": true
+    },
+    "lifecycle": {
+      "type": "object",
+      "properties": {
+        "prepare": {
+          "$ref": "benchmark-hook-v1.schema.json"
+        },
+        "quiesce": {
+          "$ref": "benchmark-hook-v1.schema.json"
+        },
+        "snapshot": {
+          "$ref": "benchmark-hook-v1.schema.json"
+        },
+        "cleanup": {
+          "$ref": "benchmark-hook-v1.schema.json"
+        }
+      },
+      "additionalProperties": false
+    },
+    "submission": {
+      "type": "object",
+      "properties": {
+        "kind": {
+          "enum": [
+            "artifacts",
+            "environment"
+          ]
+        },
+        "paths": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "minLength": 1
+          },
+          "minItems": 0,
+          "uniqueItems": true
+        },
+        "max_bytes": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "final_response": {
+          "const": "/hitch-evidence/final-response.json"
+        }
+      },
+      "required": [
+        "kind",
+        "paths",
+        "max_bytes"
+      ],
+      "additionalProperties": false
+    },
+    "grading": {
+      "type": "object",
+      "properties": {
+        "kind": {
+          "enum": [
+            "command",
+            "harbor"
+          ]
+        },
+        "entrypoint": {
+          "const": [
+            "bash",
+            "/tests/test.sh"
+          ]
+        },
+        "metric_map": {
+          "type": "object",
+          "minProperties": 1,
+          "additionalProperties": {
+            "type": "string",
+            "minLength": 1
+          }
+        }
+      },
+      "required": [
+        "kind",
+        "entrypoint",
+        "metric_map"
+      ],
+      "additionalProperties": false
+    },
+    "extensions": {
+      "type": "object"
+    }
+  },
+  "required": [
+    "schema_version",
+    "source_task_id",
+    "driver",
+    "requirements",
+    "lifecycle",
+    "submission",
+    "grading"
+  ],
+  "additionalProperties": false,
+  "allOf": [
+    {
+      "if": {
+        "properties": {
+          "driver": {
+            "properties": {
+              "kind": {
+                "const": "tool-server"
+              }
+            }
+          }
+        }
+      },
+      "then": {
+        "properties": {
+          "lifecycle": {
+            "required": [
+              "prepare",
+              "quiesce",
+              "snapshot",
+              "cleanup"
+            ]
+          },
+          "submission": {
+            "properties": {
+              "kind": {
+                "const": "artifacts"
+              },
+              "paths": {
+                "minItems": 1
+              }
+            }
+          },
+          "grading": {
+            "properties": {
+              "kind": {
+                "const": "command"
+              }
+            }
+          }
+        }
+      },
+      "else": {
+        "properties": {
+          "lifecycle": {
+            "maxProperties": 0
+          }
+        }
+      }
+    },
+    {
+      "if": {
+        "properties": {
+          "driver": {
+            "properties": {
+              "kind": {
+                "const": "model-call"
+              }
+            }
+          }
+        }
+      },
+      "then": {
+        "properties": {
+          "submission": {
+            "required": [
+              "final_response"
+            ],
+            "properties": {
+              "kind": {
+                "const": "artifacts"
+              }
+            }
+          },
+          "grading": {
+            "properties": {
+              "kind": {
+                "const": "command"
+              }
+            }
+          }
+        }
+      }
+    },
+    {
+      "if": {
+        "properties": {
+          "submission": {
+            "properties": {
+              "kind": {
+                "const": "artifacts"
+              }
+            }
+          }
+        }
+      },
+      "then": {
+        "properties": {
+          "submission": {
+            "properties": {
+              "paths": {
+                "minItems": 1
+              }
+            }
+          }
+        }
+      },
+      "else": {
+        "properties": {
+          "driver": {
+            "properties": {
+              "kind": {
+                "const": "terminal"
+              }
+            }
+          },
+          "submission": {
+            "properties": {
+              "paths": {
+                "maxItems": 0
+              }
+            }
+          }
+        }
+      }
+    }
+  ]
+}

package/docs/schemas/benchmark-tool-result-v1.schema.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/rsi-gear/agent-hitch/schemas/benchmark-tool-result-v1.schema.json",
+  "title": "Native image tool result envelope",
+  "type": "object",
+  "required": ["protocol", "content"],
+  "properties": {
+    "protocol": {"const": "hitch-tool-result@1"},
+    "content": {
+      "type": "array",
+      "maxItems": 64,
+      "contains": {"type": "object", "required": ["type"], "properties": {"type": {"const": "image"}}},
+      "minContains": 0,
+      "maxContains": 8,
+      "items": {
+        "oneOf": [
+          {"type": "object", "required": ["type", "text"], "properties": {
+            "type": {"const": "text"}, "text": {"type": "string"}
+          }},
+          {"type": "object", "required": ["type", "mimeType", "data"], "properties": {
+            "type": {"const": "image"},
+            "mimeType": {"enum": ["image/png", "image/jpeg", "image/webp"]},
+            "data": {"type": "string", "minLength": 4, "maxLength": 5592408, "contentEncoding": "base64"}
+          }}
+        ]
+      }
+    }
+  },
+  "$comment": "Wire payload is limited to 16 MiB. Runtime additionally verifies canonical base64, MIME signatures and a 4 MiB decoded image limit. The CLI replaces each image data field with a generated absolute path, byte count and SHA256; this schema describes the server's wire response, not that materialized CLI output."
+}

package/docs/schemas/error.schema.json

@@ -0,0 +1,22 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-hitch.local/schemas/error.schema.json",
+  "title": "Hitch CLI error envelope",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "kind", "error"],
+  "properties": {
+    "schema_version": { "const": "1" },
+    "kind": { "const": "error" },
+    "error": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["code", "message", "exit_code"],
+      "properties": {
+        "code": { "type": "string", "pattern": "^[a-z][a-z0-9_-]*$", "maxLength": 128 },
+        "message": { "type": "string" },
+        "exit_code": { "type": "integer", "minimum": 1, "maximum": 255 }
+      }
+    }
+  }
+}

package/docs/schemas/eval-progress.schema.json

@@ -29,21 +29,7 @@
     "planned_trials": { "type": ["integer", "null"], "minimum": 0 },
     "trials": {
       "type": "array",
-      "items": {
-        "type": "object",
-        "additionalProperties": false,
-        "required": ["trial_id", "run_id", "task_id", "attempt", "observation_status"],
-        "properties": {
-          "trial_id": { "type": "string", "minLength": 1 },
-          "run_id": { "type": "string", "pattern": "^run_[a-f0-9]{32}$" },
-          "task_id": { "type": "string", "minLength": 1 },
-          "attempt": { "type": "integer", "minimum": 1 },
-          "observation_status": { "enum": ["valid", "invalid"] },
-          "reward": { "type": "number" },
-          "verifier_result_ref": { "type": "string", "minLength": 1 },
-          "invalid_reason": { "type": "string", "minLength": 1 }
-        }
-      }
+      "items": { "$ref": "eval-trial-reference.schema.json" }
     },
     "summary": {
       "type": "object",

package/docs/schemas/eval-rerun-result.schema.json

@@ -41,10 +41,15 @@
     "source": {
       "type": "object",
       "additionalProperties": false,
-      "required": ["source_trial_id", "source_run_id", "source_work_id", "source_backend_directory"],
+      "required": ["source_trial_id", "source_work_id", "source_backend_directory"],
+      "oneOf": [
+        { "required": ["source_run_id"], "not": { "required": ["source_run_group_id"] } },
+        { "required": ["source_run_group_id"], "not": { "required": ["source_run_id"] } }
+      ],
       "properties": {
         "source_trial_id": { "type": "string", "minLength": 1 },
         "source_run_id": { "type": "string", "pattern": "^run_[a-f0-9]{32}$" },
+        "source_run_group_id": { "type": "string", "pattern": "^run_group_[a-f0-9]{32}$" },
         "source_work_id": { "type": "string", "pattern": "^work_[a-f0-9]{32}$" },
         "source_backend_directory": { "type": "string", "pattern": "^harbor/work-items/work_[a-f0-9]{32}/epoch-[0-9]{6}$" }
       }

package/docs/schemas/eval-rerun-submission.schema.json

@@ -10,10 +10,12 @@
     "rerun_id": { "type": "string", "pattern": "^rerun_[a-f0-9]{32}$" },
     "eval_id": { "type": "string", "pattern": "^eval_[a-f0-9]{32}$" },
     "rerun_type": { "$ref": "#/$defs/rerunType" },
+    "verifier_runtime_id": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" },
     "semantics": { "$ref": "#/$defs/semantics" },
     "selector": { "$ref": "#/$defs/selector" },
     "submitted_at": { "type": "string", "format": "date-time" }
   },
+  "allOf": [{ "if": { "required": ["verifier_runtime_id"] }, "then": { "properties": { "rerun_type": { "const": "verifier-only" } } } }],
   "$defs": {
     "rerunType": {
       "enum": ["candidate-restart", "candidate-resume", "trajectory-replay", "verifier-only", "collect-only"]
@@ -25,7 +27,7 @@
       "properties": {
         "candidate_action": { "enum": ["restart", "resume", "replay", "none"] },
         "conversation_source": { "enum": ["original-instruction", "native-session", "canonical-trajectory", "none"] },
-        "sandbox_source": { "enum": ["clean", "checkpoint", "retained", "none"] },
+        "sandbox_source": { "enum": ["clean", "checkpoint", "retained", "recorded-artifacts", "none"] },
         "candidate_executes": { "type": "boolean" }
       }
     },

package/docs/schemas/eval-result.schema.json

@@ -76,21 +76,7 @@
     },
     "trials": {
       "type": "array",
-      "items": {
-        "type": "object",
-        "additionalProperties": false,
-        "required": ["trial_id", "run_id", "task_id", "attempt", "observation_status"],
-        "properties": {
-          "trial_id": { "type": "string", "minLength": 1 },
-          "run_id": { "type": "string", "pattern": "^run_[a-f0-9]{32}$" },
-          "task_id": { "type": "string", "minLength": 1 },
-          "attempt": { "type": "integer", "minimum": 1 },
-          "observation_status": { "enum": ["valid", "invalid"] },
-          "reward": { "type": "number" },
-          "verifier_result_ref": { "type": "string", "minLength": 1 },
-          "invalid_reason": { "type": "string", "minLength": 1 }
-        }
-      }
+      "items": { "$ref": "eval-trial-reference.schema.json" }
     },
     "local_source_transport": {
       "type": "object",