agent-hitch 0.2.6 → 0.2.8

package/docs/schemas/benchmark-lock-v1.schema.json

@@ -0,0 +1,351 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://hitch.dev/schemas/benchmark-lock-v1.schema.json",
+  "type": "object",
+  "properties": {
+    "schema_version": {
+      "const": "1"
+    },
+    "protocol": {
+      "const": "hitch-benchmark@1"
+    },
+    "benchmark_id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "release": {
+      "type": "string",
+      "minLength": 1
+    },
+    "package_digest": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "source": {
+      "type": "object",
+      "properties": {
+        "kind": {
+          "enum": [
+            "local",
+            "git"
+          ]
+        },
+        "uri": {
+          "type": "string",
+          "minLength": 1
+        },
+        "resolved_revision": {
+          "type": "string",
+          "minLength": 1
+        },
+        "access": {
+          "enum": [
+            "public",
+            "private",
+            "gated"
+          ]
+        },
+        "manifest_digest": {
+          "type": "string",
+          "pattern": "^sha256:[a-f0-9]{64}$"
+        }
+      },
+      "required": [
+        "kind",
+        "uri",
+        "resolved_revision",
+        "access",
+        "manifest_digest"
+      ],
+      "additionalProperties": false
+    },
+    "components": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "role": {
+            "type": "string",
+            "minLength": 1
+          },
+          "uri": {
+            "type": "string",
+            "minLength": 1
+          },
+          "resolved_revision": {
+            "type": "string",
+            "minLength": 1
+          },
+          "digest": {
+            "type": "string",
+            "pattern": "^sha256:[a-f0-9]{64}$"
+          }
+        },
+        "required": [
+          "role",
+          "uri",
+          "resolved_revision",
+          "digest"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "resolver": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "version": {
+          "type": "string",
+          "minLength": 1
+        },
+        "code_digest": {
+          "type": "string",
+          "pattern": "^sha256:[a-f0-9]{64}$"
+        }
+      },
+      "required": [
+        "id",
+        "version",
+        "code_digest"
+      ],
+      "additionalProperties": false
+    },
+    "task_dialect": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "const": "harbor"
+        },
+        "schema_version": {
+          "const": "1.4"
+        }
+      },
+      "required": [
+        "name",
+        "schema_version"
+      ],
+      "additionalProperties": false
+    },
+    "runtime_components": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string",
+            "minLength": 1
+          },
+          "protocol": {
+            "type": "string",
+            "minLength": 1
+          },
+          "content_digest": {
+            "type": "string",
+            "pattern": "^sha256:[a-f0-9]{64}$"
+          }
+        },
+        "required": [
+          "id",
+          "protocol",
+          "content_digest"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "tasks": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "task_id": {
+            "type": "string",
+            "minLength": 1
+          },
+          "source_task_id": {
+            "type": "string",
+            "minLength": 1
+          },
+          "path": {
+            "type": "string",
+            "minLength": 1
+          },
+          "task_digest": {
+            "type": "string",
+            "pattern": "^sha256:[a-f0-9]{64}$"
+          },
+          "input_digest": {
+            "type": "string",
+            "pattern": "^sha256:[a-f0-9]{64}$"
+          },
+          "grader_digest": {
+            "type": "string",
+            "pattern": "^sha256:[a-f0-9]{64}$"
+          },
+          "environment_refs": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "role": {
+                  "type": "string",
+                  "minLength": 1
+                },
+                "kind": {
+                  "const": "build"
+                },
+                "context_digest": {
+                  "type": "string",
+                  "pattern": "^sha256:[a-f0-9]{64}$"
+                },
+                "base_image_digests": {
+                  "type": "array",
+                  "items": {
+                    "type": "string",
+                    "pattern": "^sha256:[a-f0-9]{64}$"
+                  }
+                },
+                "platform": {
+                  "const": "linux/amd64"
+                }
+              },
+              "required": [
+                "role",
+                "kind",
+                "context_digest",
+                "base_image_digests",
+                "platform"
+              ],
+              "additionalProperties": false
+            }
+          }
+        },
+        "required": [
+          "task_id",
+          "source_task_id",
+          "path",
+          "task_digest",
+          "input_digest",
+          "grader_digest",
+          "environment_refs"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "profile_digest": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "required_capabilities": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1
+      }
+    },
+    "metric_spec_digest": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "files": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "path": {
+            "type": "string",
+            "minLength": 1
+          },
+          "digest": {
+            "type": "string",
+            "pattern": "^sha256:[a-f0-9]{64}$"
+          },
+          "bytes": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "mode": {
+            "type": "integer",
+            "minimum": 0,
+            "maximum": 511
+          }
+        },
+        "required": [
+          "path",
+          "digest",
+          "bytes",
+          "mode"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "transformations": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "kind": {
+            "type": "string",
+            "minLength": 1
+          },
+          "before": {
+            "type": "string",
+            "pattern": "^sha256:[a-f0-9]{64}$"
+          },
+          "after": {
+            "type": "string",
+            "pattern": "^sha256:[a-f0-9]{64}$"
+          }
+        },
+        "required": [
+          "kind",
+          "before",
+          "after"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "source_adapter": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "type": "string"
+        },
+        "version": {
+          "type": "string"
+        },
+        "code_digest": {
+          "type": "string",
+          "pattern": "^sha256:[a-f0-9]{64}$"
+        }
+      },
+      "required": [
+        "id",
+        "version",
+        "code_digest"
+      ],
+      "additionalProperties": false
+    }
+  },
+  "required": [
+    "schema_version",
+    "protocol",
+    "benchmark_id",
+    "release",
+    "package_digest",
+    "source",
+    "components",
+    "resolver",
+    "task_dialect",
+    "runtime_components",
+    "tasks",
+    "profile_digest",
+    "required_capabilities",
+    "metric_spec_digest",
+    "files",
+    "transformations"
+  ],
+  "additionalProperties": false
+}

package/docs/schemas/benchmark-metric-v1.schema.json

@@ -0,0 +1,37 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://hitch.dev/schemas/benchmark-metric-v1.schema.json",
+  "type": "object",
+  "properties": {
+    "type": {
+      "enum": [
+        "binary",
+        "scalar"
+      ]
+    },
+    "direction": {
+      "enum": [
+        "maximize",
+        "minimize"
+      ]
+    },
+    "range": {
+      "type": "array",
+      "items": {
+        "type": "number"
+      },
+      "minItems": 2,
+      "maxItems": 2
+    },
+    "reducer": {
+      "const": "task_macro_mean"
+    }
+  },
+  "required": [
+    "type",
+    "direction",
+    "range",
+    "reducer"
+  ],
+  "additionalProperties": false
+}

package/docs/schemas/benchmark-package-v1.schema.json

@@ -0,0 +1,167 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://hitch.dev/schemas/benchmark-package-v1.schema.json",
+  "type": "object",
+  "properties": {
+    "schema_version": {
+      "const": "1"
+    },
+    "protocol": {
+      "const": "hitch-benchmark@1"
+    },
+    "id": {
+      "type": "string",
+      "minLength": 1
+    },
+    "release": {
+      "type": "string",
+      "minLength": 1
+    },
+    "task_root": {
+      "type": "string",
+      "minLength": 1
+    },
+    "task_ids": {
+      "type": "array",
+      "items": {
+        "type": "string",
+        "minLength": 1
+      },
+      "minItems": 1,
+      "uniqueItems": true
+    },
+    "default_profile": {
+      "type": "string",
+      "minLength": 1
+    },
+    "primary_metric": {
+      "type": "string",
+      "minLength": 1
+    },
+    "task_format": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "const": "harbor"
+        },
+        "schema_version": {
+          "const": "1.4"
+        }
+      },
+      "required": [
+        "name",
+        "schema_version"
+      ],
+      "additionalProperties": false
+    },
+    "source": {
+      "type": "object",
+      "properties": {
+        "kind": {
+          "enum": [
+            "local",
+            "git"
+          ]
+        },
+        "path": {
+          "type": "string",
+          "minLength": 1
+        },
+        "uri": {
+          "type": "string",
+          "minLength": 1
+        },
+        "resolved_revision": {
+          "type": "string",
+          "minLength": 1
+        },
+        "license": {
+          "type": "string",
+          "minLength": 1
+        },
+        "access": {
+          "enum": [
+            "public",
+            "private",
+            "gated"
+          ]
+        }
+      },
+      "required": [
+        "kind",
+        "license"
+      ],
+      "additionalProperties": false
+    },
+    "metrics": {
+      "type": "object",
+      "minProperties": 1,
+      "additionalProperties": {
+        "$ref": "benchmark-metric-v1.schema.json"
+      }
+    },
+    "publication": {
+      "type": "object",
+      "properties": {
+        "track": {
+          "enum": [
+            "custom",
+            "public-subset"
+          ]
+        },
+        "training_eligible": {
+          "const": false
+        }
+      },
+      "required": [
+        "track",
+        "training_eligible"
+      ],
+      "additionalProperties": false
+    },
+    "runtime_components": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "id": {
+            "type": "string",
+            "minLength": 1
+          },
+          "protocol": {
+            "type": "string",
+            "minLength": 1
+          },
+          "path": {
+            "type": "string",
+            "minLength": 1
+          }
+        },
+        "required": [
+          "id",
+          "protocol",
+          "path"
+        ],
+        "additionalProperties": false
+      }
+    },
+    "extensions": {
+      "type": "object"
+    }
+  },
+  "required": [
+    "schema_version",
+    "protocol",
+    "id",
+    "release",
+    "task_root",
+    "task_ids",
+    "default_profile",
+    "primary_metric",
+    "task_format",
+    "source",
+    "metrics",
+    "publication"
+  ],
+  "additionalProperties": false
+}

package/docs/schemas/benchmark-phase-group.schema.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://agent-hitch.local/schemas/benchmark-phase-group.schema.json",
+  "title": "Hitch benchmark phase candidate evidence group V1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "kind", "scope", "run_group_id", "eval_id", "trial_id", "attempt", "benchmark_id", "benchmark_revision", "task_id", "task_digest", "verifier_identity", "harness", "model", "phases", "created_at"],
+  "properties": {
+    "schema_version": { "const": "1" },
+    "kind": { "const": "benchmark-phase-group" },
+    "scope": { "const": "candidate-evidence-only" },
+    "run_group_id": { "type": "string", "pattern": "^run_group_[a-f0-9]{32}$" },
+    "eval_id": { "type": "string", "pattern": "^eval_[a-f0-9]{32}$" },
+    "trial_id": { "type": "string", "minLength": 1 },
+    "attempt": { "type": "integer", "minimum": 1 },
+    "benchmark_id": { "type": "string", "minLength": 1 },
+    "benchmark_revision": { "type": "string", "minLength": 1 },
+    "task_id": { "type": "string", "minLength": 1 },
+    "task_digest": { "$ref": "#/$defs/digest" },
+    "verifier_identity": { "$ref": "#/$defs/digest" },
+    "harness": { "$ref": "run-manifest.schema.json#/properties/harness" },
+    "model": { "$ref": "run-manifest.schema.json#/properties/model" },
+    "phases": {
+      "type": "array", "minItems": 1, "maxItems": 10000,
+      "items": {
+        "type": "object", "additionalProperties": false,
+        "required": ["phase_index", "run_id", "process_status", "provider_session_id", "bundle_digest", "bundle_index_digest"],
+        "properties": {
+          "phase_index": { "type": "integer", "minimum": 1 },
+          "run_id": { "type": "string", "pattern": "^run_[a-f0-9]{32}$" },
+          "process_status": { "enum": ["succeeded", "failed", "timed_out", "cancelled"] },
+          "provider_session_id": { "type": "string", "minLength": 1 },
+          "bundle_digest": { "$ref": "#/$defs/digest" },
+          "bundle_index_digest": { "$ref": "#/$defs/digest" }
+        }
+      }
+    },
+    "created_at": { "type": "string", "format": "date-time" }
+  },
+  "$defs": { "digest": { "type": "string", "pattern": "^sha256:[a-f0-9]{64}$" } }
+}