agent-hitch 0.2.16 → 0.2.17

package/docs/schemas/eval-progress.schema.json

@@ -23,7 +23,7 @@
     "eval_id": { "type": "string", "pattern": "^eval_[a-f0-9]{32}$" },
     "benchmark_id": { "type": "string", "minLength": 1 },
     "benchmark_revision": { "type": "string", "minLength": 1 },
-    "status": { "const": "running" },
+    "status": { "enum": ["running", "succeeded", "failed", "cancelled"] },
     "generation": { "type": "integer", "minimum": 0 },
     "planned_tasks": { "type": ["integer", "null"], "minimum": 0 },
     "planned_trials": { "type": ["integer", "null"], "minimum": 0 },

package/docs/schemas/eval-request.schema.json

@@ -4,24 +4,155 @@
   "title": "Hitch eval request",
   "type": "object",
   "additionalProperties": false,
-  "required": ["backend", "dataset", "harness_ref"],
+  "required": [
+    "backend",
+    "dataset",
+    "harness_ref"
+  ],
   "properties": {
-    "schema_version": { "const": "1" },
-    "backend": { "const": "harbor" },
-    "dataset": { "type": "string", "minLength": 1 },
-    "benchmark_id": { "type": "string", "minLength": 1 },
-    "benchmark_revision": { "type": "string", "minLength": 1 },
-    "harness_ref": { "type": "string", "minLength": 1 },
-    "model": { "type": "string" },
-    "attempts": { "type": "integer", "minimum": 1 },
-    "max_concurrent": { "type": "integer", "minimum": 1 },
-    "infrastructure_retries": { "type": "integer", "minimum": 0 },
-    "infrastructure_retry_backoff_ms": { "type": "number", "minimum": 0 },
-    "timeout_ms": { "type": "number", "minimum": 0 },
-    "setup_timeout_ms": { "type": "number", "minimum": 0 },
-    "agent_args": { "type": "array", "items": { "type": "string" } },
-    "pass_env": { "type": "array", "items": { "type": "string" }, "uniqueItems": true },
-    "local_inference": { "$ref": "local-inference-selection.schema.json" },
-    "training_binding": { "$ref": "training-external-binding.schema.json" }
+    "schema_version": {
+      "const": "1"
+    },
+    "backend": {
+      "const": "harbor"
+    },
+    "dataset": {
+      "type": "string",
+      "minLength": 1
+    },
+    "benchmark_id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "benchmark_revision": {
+      "type": "string",
+      "minLength": 1
+    },
+    "harness_ref": {
+      "type": "string",
+      "minLength": 1
+    },
+    "model": {
+      "type": "string"
+    },
+    "attempts": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "max_concurrent": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "infrastructure_retries": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "infrastructure_retry_backoff_ms": {
+      "type": "number",
+      "minimum": 0
+    },
+    "timeout_ms": {
+      "type": "number",
+      "minimum": 0
+    },
+    "setup_timeout_ms": {
+      "type": "number",
+      "minimum": 0
+    },
+    "agent_args": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
+    "pass_env": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      },
+      "uniqueItems": true
+    },
+    "training_binding": {
+      "$ref": "training-external-binding.schema.json"
+    },
+    "environment_binding": {
+      "type": "string",
+      "pattern": "^hmac-sha256:[a-f0-9]{64}$"
+    },
+    "artifact_pins": {
+      "type": "array",
+      "minItems": 1,
+      "maxItems": 2,
+      "uniqueItems": true,
+      "items": {
+        "$ref": "eval-artifact-pin.schema.json"
+      },
+      "allOf": [
+        {
+          "contains": {
+            "type": "object",
+            "required": [
+              "platform"
+            ],
+            "properties": {
+              "platform": {
+                "const": "linux-x64"
+              }
+            }
+          },
+          "minContains": 0,
+          "maxContains": 1
+        },
+        {
+          "contains": {
+            "type": "object",
+            "required": [
+              "platform"
+            ],
+            "properties": {
+              "platform": {
+                "const": "linux-arm64"
+              }
+            }
+          },
+          "minContains": 0,
+          "maxContains": 1
+        }
+      ]
+    },
+    "endpoint_binding": {
+      "$ref": "external-endpoint-binding.schema.json"
+    },
+    "model_binding": {
+      "$ref": "ready-endpoint-binding.schema.json"
+    },
+    "generation": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "max_output_tokens": {
+          "type": "integer",
+          "minimum": 1
+        },
+        "temperature": {
+          "type": "number"
+        },
+        "top_p": {
+          "type": "number"
+        },
+        "top_k": {
+          "type": "number"
+        },
+        "min_p": {
+          "type": "number"
+        },
+        "repetition_penalty": {
+          "type": "number"
+        },
+        "seed": {
+          "type": "number"
+        }
+      }
+    }
   }
 }

package/docs/schemas/eval-rerun-submission.schema.json

@@ -10,12 +10,19 @@
     "rerun_id": { "type": "string", "pattern": "^rerun_[a-f0-9]{32}$" },
     "eval_id": { "type": "string", "pattern": "^eval_[a-f0-9]{32}$" },
     "rerun_type": { "$ref": "#/$defs/rerunType" },
+    "endpoint_attempt": { "$ref": "endpoint-evaluation-attempt.schema.json" },
     "verifier_runtime_id": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" },
     "semantics": { "$ref": "#/$defs/semantics" },
     "selector": { "$ref": "#/$defs/selector" },
     "submitted_at": { "type": "string", "format": "date-time" }
   },
-  "allOf": [{ "if": { "required": ["verifier_runtime_id"] }, "then": { "properties": { "rerun_type": { "const": "verifier-only" } } } }],
+  "allOf": [
+    { "if": { "required": ["verifier_runtime_id"] }, "then": { "properties": { "rerun_type": { "const": "verifier-only" } } } },
+    { "if": { "required": ["endpoint_attempt"] }, "then": { "properties": {
+      "rerun_type": { "const": "candidate-restart" },
+      "selector": { "type": "object", "additionalProperties": false, "required": ["mode"], "properties": { "mode": { "const": "invalid" } } }
+    } } }
+  ],
   "$defs": {
     "rerunType": {
       "enum": ["candidate-restart", "candidate-resume", "trajectory-replay", "verifier-only", "collect-only"]

package/docs/schemas/eval-result.schema.json

@@ -4,42 +4,148 @@
   "title": "Hitch eval result",
   "type": "object",
   "additionalProperties": true,
-  "required": ["schema_version", "eval_id", "benchmark_id", "benchmark_revision", "status", "exit_code", "trials", "started_at", "completed_at"],
+  "required": [
+    "schema_version",
+    "eval_id",
+    "benchmark_id",
+    "benchmark_revision",
+    "status",
+    "exit_code",
+    "trials",
+    "started_at",
+    "completed_at"
+  ],
   "properties": {
-    "schema_version": { "const": "1" },
-    "eval_id": { "type": "string", "pattern": "^eval_[a-f0-9]{32}$" },
-    "status": { "enum": ["succeeded", "failed", "cancelled"] },
-    "exit_code": { "type": "integer", "minimum": 0 },
+    "schema_version": {
+      "const": "1"
+    },
+    "eval_id": {
+      "type": "string",
+      "pattern": "^eval_[a-f0-9]{32}$"
+    },
+    "status": {
+      "enum": [
+        "succeeded",
+        "failed",
+        "cancelled"
+      ]
+    },
+    "exit_code": {
+      "type": "integer",
+      "minimum": 0
+    },
     "backend": {
       "type": "object",
-      "required": ["name", "executable", "identity", "config_path", "process_exit_code"],
+      "required": [
+        "name",
+        "executable",
+        "identity",
+        "config_path",
+        "process_exit_code"
+      ],
       "properties": {
-        "name": { "const": "harbor" },
-        "executable": { "type": "string", "minLength": 1 },
-        "version": { "type": ["string", "null"] },
-        "identity": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" },
-        "config_path": { "type": "string", "minLength": 1 },
-        "result_path": { "type": ["string", "null"] },
-        "process_exit_code": { "type": ["integer", "null"] }
+        "name": {
+          "const": "harbor"
+        },
+        "executable": {
+          "type": "string",
+          "minLength": 1
+        },
+        "version": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "identity": {
+          "type": "string",
+          "pattern": "^sha256:[a-f0-9]{64}$"
+        },
+        "config_path": {
+          "type": "string",
+          "minLength": 1
+        },
+        "result_path": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "process_exit_code": {
+          "type": [
+            "integer",
+            "null"
+          ]
+        }
       },
       "additionalProperties": true
     },
-    "summary": { "type": ["object", "null"] },
-    "benchmark_id": { "type": "string", "minLength": 1 },
-    "benchmark_revision": { "type": "string", "minLength": 1 },
+    "summary": {
+      "type": [
+        "object",
+        "null"
+      ]
+    },
+    "benchmark_id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "benchmark_revision": {
+      "type": "string",
+      "minLength": 1
+    },
     "prepared_artifact": {
       "type": "object",
-      "required": ["artifact_id", "artifact_integrity", "entrypoint_integrity", "harness_id", "revision_identity", "platform", "source_type"],
+      "required": [
+        "artifact_id",
+        "artifact_integrity",
+        "entrypoint_integrity",
+        "harness_id",
+        "revision_identity",
+        "platform",
+        "source_type"
+      ],
       "properties": {
-        "artifact_id": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
-        "artifact_integrity": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
-        "entrypoint_integrity": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
-        "harness_id": { "type": "string", "pattern": "^[a-z][a-z0-9-]*$" },
-        "revision_identity": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
-        "platform": { "type": "string", "pattern": "^(?:linux|darwin|win32)-[a-z0-9_]+$" },
-        "node_version": { "type": "string", "pattern": "^v[0-9]+\\.[0-9]+\\.[0-9]+$" },
-        "source_type": { "enum": ["npm", "git"] },
-        "storage": { "enum": ["host-artifact-store-v1", "harbor-artifact-cache-v2"] }
+        "artifact_id": {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$"
+        },
+        "artifact_integrity": {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$"
+        },
+        "entrypoint_integrity": {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$"
+        },
+        "harness_id": {
+          "type": "string",
+          "pattern": "^[a-z][a-z0-9-]*$"
+        },
+        "revision_identity": {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$"
+        },
+        "platform": {
+          "type": "string",
+          "pattern": "^(?:linux|darwin|win32)-[a-z0-9_]+$"
+        },
+        "node_version": {
+          "type": "string",
+          "pattern": "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
+        },
+        "source_type": {
+          "enum": [
+            "npm",
+            "git"
+          ]
+        },
+        "storage": {
+          "enum": [
+            "host-artifact-store-v1",
+            "harbor-artifact-cache-v2"
+          ]
+        }
       },
       "additionalProperties": false
     },
@@ -48,25 +154,91 @@
       "minItems": 1,
       "items": {
         "type": "object",
-        "required": ["artifact_id", "artifact_integrity", "entrypoint_integrity", "harness_id", "revision_identity", "platform", "node_version", "source_type", "task_ids", "runtime_contract"],
+        "required": [
+          "artifact_id",
+          "artifact_integrity",
+          "entrypoint_integrity",
+          "harness_id",
+          "revision_identity",
+          "platform",
+          "node_version",
+          "source_type",
+          "task_ids",
+          "runtime_contract"
+        ],
         "properties": {
-          "artifact_id": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
-          "artifact_integrity": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
-          "entrypoint_integrity": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
-          "harness_id": { "type": "string", "pattern": "^[a-z][a-z0-9-]*$" },
-          "revision_identity": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
-          "platform": { "type": "string", "pattern": "^(?:linux|darwin|win32)-[a-z0-9_]+$" },
-          "node_version": { "type": "string", "pattern": "^v[0-9]+\\.[0-9]+\\.[0-9]+$" },
-          "source_type": { "enum": ["npm", "git"] },
-          "storage": { "enum": ["host-artifact-store-v1", "harbor-artifact-cache-v2"] },
-          "task_ids": { "type": "array", "items": { "type": "string", "minLength": 1 }, "uniqueItems": true },
+          "artifact_id": {
+            "type": "string",
+            "pattern": "^sha256:[0-9a-f]{64}$"
+          },
+          "artifact_integrity": {
+            "type": "string",
+            "pattern": "^sha256:[0-9a-f]{64}$"
+          },
+          "entrypoint_integrity": {
+            "type": "string",
+            "pattern": "^sha256:[0-9a-f]{64}$"
+          },
+          "harness_id": {
+            "type": "string",
+            "pattern": "^[a-z][a-z0-9-]*$"
+          },
+          "revision_identity": {
+            "type": "string",
+            "pattern": "^sha256:[0-9a-f]{64}$"
+          },
+          "platform": {
+            "type": "string",
+            "pattern": "^(?:linux|darwin|win32)-[a-z0-9_]+$"
+          },
+          "node_version": {
+            "type": "string",
+            "pattern": "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
+          },
+          "source_type": {
+            "enum": [
+              "npm",
+              "git"
+            ]
+          },
+          "storage": {
+            "enum": [
+              "host-artifact-store-v1",
+              "harbor-artifact-cache-v2"
+            ]
+          },
+          "task_ids": {
+            "type": "array",
+            "items": {
+              "type": "string",
+              "minLength": 1
+            },
+            "uniqueItems": true
+          },
           "runtime_contract": {
             "type": "object",
-            "required": ["docker_platform", "artifact_platform", "node_version"],
+            "required": [
+              "docker_platform",
+              "artifact_platform",
+              "node_version"
+            ],
             "properties": {
-              "docker_platform": { "enum": ["linux/amd64", "linux/arm64"] },
-              "artifact_platform": { "enum": ["linux-x64", "linux-arm64"] },
-              "node_version": { "type": "string", "pattern": "^v[0-9]+\\.[0-9]+\\.[0-9]+$" }
+              "docker_platform": {
+                "enum": [
+                  "linux/amd64",
+                  "linux/arm64"
+                ]
+              },
+              "artifact_platform": {
+                "enum": [
+                  "linux-x64",
+                  "linux-arm64"
+                ]
+              },
+              "node_version": {
+                "type": "string",
+                "pattern": "^v[0-9]+\\.[0-9]+\\.[0-9]+$"
+              }
             },
             "additionalProperties": false
           }
@@ -76,33 +248,115 @@
     },
     "trials": {
       "type": "array",
-      "items": { "$ref": "eval-trial-reference.schema.json" }
+      "items": {
+        "$ref": "eval-trial-reference.schema.json"
+      }
     },
     "local_source_transport": {
       "type": "object",
-      "required": ["kind", "resolution_identity", "commit", "tree", "payload_sha256", "payload_bytes"],
+      "required": [
+        "kind",
+        "resolution_identity",
+        "commit",
+        "tree",
+        "payload_sha256",
+        "payload_bytes"
+      ],
       "properties": {
-        "kind": { "const": "local-git-commit" },
-        "resolution_identity": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
-        "commit": { "type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$" },
-        "tree": { "type": "string", "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$" },
-        "payload_sha256": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" },
-        "payload_bytes": { "type": "integer", "minimum": 0 },
-        "object_count": { "type": "integer", "minimum": 0 },
-        "file_count": { "type": "integer", "minimum": 0 }
+        "kind": {
+          "const": "local-git-commit"
+        },
+        "resolution_identity": {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$"
+        },
+        "commit": {
+          "type": "string",
+          "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"
+        },
+        "tree": {
+          "type": "string",
+          "pattern": "^(?:[0-9a-f]{40}|[0-9a-f]{64})$"
+        },
+        "payload_sha256": {
+          "type": "string",
+          "pattern": "^sha256:[0-9a-f]{64}$"
+        },
+        "payload_bytes": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "object_count": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "file_count": {
+          "type": "integer",
+          "minimum": 0
+        }
       },
       "additionalProperties": false
     },
     "error": {
       "type": "object",
-      "required": ["code", "message"],
+      "required": [
+        "code",
+        "message"
+      ],
       "properties": {
-        "code": { "type": "string", "minLength": 1 },
-        "message": { "type": "string" }
+        "code": {
+          "type": "string",
+          "minLength": 1
+        },
+        "message": {
+          "type": "string"
+        }
       },
       "additionalProperties": true
     },
-    "started_at": { "type": "string", "format": "date-time" },
-    "completed_at": { "type": "string", "format": "date-time" }
+    "started_at": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "completed_at": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "endpoint_binding": {
+      "$ref": "external-endpoint-binding.schema.json"
+    },
+    "usage": {
+      "type": "object",
+      "properties": {
+        "requests": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "input_tokens": {
+          "type": "number",
+          "minimum": 0
+        },
+        "output_tokens": {
+          "type": "number",
+          "minimum": 0
+        },
+        "total_tokens": {
+          "type": "number",
+          "minimum": 0
+        },
+        "source": {
+          "const": "model-interactions-v1"
+        },
+        "completeness": {
+          "enum": [
+            "complete",
+            "partial"
+          ]
+        }
+      }
+    },
+    "task_completed": {
+      "type": "boolean"
+    }
   }
 }

package/docs/schemas/evaluation-binding.schema.json

@@ -0,0 +1,15 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-hitch.local/schemas/evaluation-binding.schema.json",
+  "title": "Hitch resident evaluation environment binding",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "binding_id", "model", "pass_env", "expires_at_ms"],
+  "properties": {
+    "schema_version": { "const": "1" },
+    "binding_id": { "type": "string", "pattern": "^hmac-sha256:[a-f0-9]{64}$" },
+    "model": { "type": "string", "minLength": 1, "maxLength": 1024, "pattern": "^[^\\u0000\\r\\n]+$" },
+    "pass_env": { "type": "array", "maxItems": 128, "uniqueItems": true, "items": { "type": "string", "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" } },
+    "expires_at_ms": { "type": ["integer", "null"], "minimum": 0 }
+  }
+}