agent-hitch 0.2.7 → 0.2.8

package/docs/schemas/verifier-evidence.schema.json

@@ -0,0 +1,276 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://github.com/rsi-gear/agent-hitch/blob/main/docs/schemas/verifier-evidence.schema.json",
+  "title": "Hitch Verifier Evidence V1",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "kind", "run_id", "verifier"],
+  "properties": {
+    "schema_version": { "const": "1" },
+    "kind": { "const": "verifier-evidence" },
+    "run_id": { "type": "string", "pattern": "^run_[a-f0-9]{32}$" },
+    "parent": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["eval_id", "trial_id", "attempt"],
+      "properties": {
+        "eval_id": { "type": "string", "pattern": "^eval_[a-f0-9]{32}$" },
+        "trial_id": { "type": "string", "minLength": 1 },
+        "attempt": { "type": "integer", "minimum": 1 }
+      }
+    },
+    "observation": { "$ref": "#/$defs/observation" },
+    "verifier": { "$ref": "#/$defs/verifier" },
+    "redactions": { "type": "array", "items": { "$ref": "#/$defs/redaction" } }
+  },
+  "$defs": {
+    "observation": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["status", "reward", "verifier_result_ref"],
+          "properties": {
+            "status": { "const": "valid" },
+            "reward": { "type": "number" },
+            "verifier_result_ref": { "$ref": "#/$defs/relativePath" }
+          }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["status", "invalid_reason"],
+          "properties": {
+            "status": { "const": "invalid" },
+            "invalid_reason": { "type": "string", "minLength": 1 },
+            "verifier_result_ref": { "$ref": "#/$defs/relativePath" }
+          }
+        }
+      ]
+    },
+    "verifier": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["status"],
+      "properties": {
+        "status": { "enum": ["complete", "result_only", "missing", "corrupt"] },
+        "result": {},
+        "result_sha256": { "$ref": "#/$defs/sha256" },
+        "scores": { "$ref": "#/$defs/scores" },
+        "process": { "$ref": "#/$defs/process" },
+        "feedback": { "$ref": "#/$defs/feedback" },
+        "structured_artifacts": {
+          "type": "object",
+          "additionalProperties": false,
+          "minProperties": 1,
+          "properties": {
+            "process": { "$ref": "#/$defs/structuredArtifact" },
+            "feedback": { "$ref": "#/$defs/structuredArtifact" }
+          }
+        },
+        "diagnostics": { "$ref": "#/$defs/diagnostics" },
+        "issues": { "type": "array", "maxItems": 16, "items": { "type": "string", "minLength": 1, "maxLength": 1024 } }
+      },
+      "allOf": [
+        {
+          "if": { "properties": { "status": { "const": "complete" } }, "required": ["status"] },
+          "then": {
+            "required": ["result", "result_sha256", "diagnostics"],
+            "properties": {
+              "diagnostics": {
+                "allOf": [
+                  { "$ref": "#/$defs/diagnostics" },
+                  { "anyOf": [{ "required": ["ctrf"] }, { "required": ["stdout"] }, { "required": ["stderr"] }] }
+                ]
+              }
+            }
+          }
+        },
+        {
+          "if": { "properties": { "status": { "const": "result_only" } }, "required": ["status"] },
+          "then": {
+            "required": ["result", "result_sha256"],
+            "properties": {
+              "diagnostics": {
+                "allOf": [
+                  { "$ref": "#/$defs/diagnostics" },
+                  { "not": { "anyOf": [{ "required": ["ctrf"] }, { "required": ["stdout"] }, { "required": ["stderr"] }] } }
+                ]
+              }
+            }
+          }
+        },
+        {
+          "if": { "properties": { "status": { "const": "missing" } }, "required": ["status"] },
+          "then": { "not": { "required": ["result"] } }
+        }
+      ]
+    },
+    "scores": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["total_score", "normalization"],
+      "properties": {
+        "total_score": { "type": "number" },
+        "process_score": { "type": "number" },
+        "normalization": { "enum": ["standard", "legacy-reward"] }
+      },
+      "allOf": [
+        {
+          "if": { "properties": { "normalization": { "const": "legacy-reward" } }, "required": ["normalization"] },
+          "then": { "not": { "required": ["process_score"] } }
+        }
+      ]
+    },
+    "process": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["schema_version", "metric", "score", "detail_status"],
+      "properties": {
+        "schema_version": { "const": "1" },
+        "metric": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$" },
+        "score": { "type": "number" },
+        "detail_status": { "enum": ["components", "aggregate-only"] },
+        "passed": { "type": "integer", "minimum": 0 },
+        "total": { "type": "integer", "minimum": 0 },
+        "excluded": { "type": "integer", "minimum": 0 },
+        "components": { "type": "array", "items": { "$ref": "#/$defs/processComponent" } }
+      },
+      "oneOf": [
+        {
+          "properties": { "detail_status": { "const": "components" } },
+          "required": ["detail_status", "passed", "total", "excluded", "components"]
+        },
+        {
+          "properties": { "detail_status": { "const": "aggregate-only" } },
+          "required": ["detail_status"],
+          "not": { "anyOf": [{ "required": ["passed"] }, { "required": ["total"] }, { "required": ["excluded"] }, { "required": ["components"] }] }
+        }
+      ]
+    },
+    "processComponent": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "category", "status", "weight"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$" },
+        "category": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$" },
+        "status": { "enum": ["passed", "failed", "excluded"] },
+        "weight": { "type": "number", "exclusiveMinimum": 0 },
+        "code": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$" },
+        "public_details": { "type": "object" },
+        "private_details_ref": { "$ref": "#/$defs/relativePath" },
+        "trajectory_refs": { "type": "array", "items": { "$ref": "#/$defs/trajectoryRef" } }
+      }
+    },
+    "feedback": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["schema_version", "items"],
+      "properties": {
+        "schema_version": { "const": "1" },
+        "items": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["code", "severity", "message"],
+            "properties": {
+              "code": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$" },
+              "severity": { "enum": ["info", "warning", "error"] },
+              "message": { "type": "string", "minLength": 1, "maxLength": 16384 },
+              "component_ids": { "type": "array", "uniqueItems": true, "items": { "type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9._:-]{0,255}$" } },
+              "trajectory_refs": { "type": "array", "items": { "$ref": "#/$defs/trajectoryRef" } }
+            }
+          }
+        }
+      }
+    },
+    "trajectoryRef": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["run_id"],
+      "properties": {
+        "run_id": { "type": "string", "pattern": "^run_[a-f0-9]{32}$" },
+        "seq_start": { "type": "integer", "minimum": 0 },
+        "seq_end": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "structuredArtifact": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ref", "bytes", "sha256"],
+      "properties": {
+        "ref": { "enum": ["verifier/process.json", "verifier/feedback.json"] },
+        "bytes": { "type": "integer", "minimum": 0 },
+        "sha256": { "$ref": "#/$defs/sha256" }
+      }
+    },
+    "diagnostics": {
+      "type": "object",
+      "additionalProperties": false,
+      "minProperties": 1,
+      "properties": {
+        "ctrf": { "$ref": "#/$defs/artifact" },
+        "stdout": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/artifact" } },
+        "stderr": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/artifact" } },
+        "infrastructure_error": {},
+        "retry_history": { "type": "array", "minItems": 1, "items": {} }
+      }
+    },
+    "artifact": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "media_type", "bytes", "sha256", "truncated"],
+      "properties": {
+        "name": { "enum": ["ctrf.json", "test-stdout.txt", "test-stderr.txt", "stdout.txt", "stderr.txt"] },
+        "media_type": { "enum": ["application/json", "text/plain"] },
+        "bytes": { "type": "integer", "minimum": 0 },
+        "sha256": { "$ref": "#/$defs/sha256" },
+        "truncated": { "type": "boolean" },
+        "json": {},
+        "text": { "type": "string" }
+      },
+      "oneOf": [
+        { "required": ["json"], "not": { "required": ["text"] } },
+        { "required": ["text"], "not": { "required": ["json"] } }
+      ],
+      "allOf": [
+        {
+          "if": { "properties": { "name": { "const": "ctrf.json" } }, "required": ["name"] },
+          "then": { "properties": { "media_type": { "const": "application/json" } } },
+          "else": { "properties": { "media_type": { "const": "text/plain" } } }
+        },
+        {
+          "if": { "properties": { "truncated": { "const": true } }, "required": ["truncated"] },
+          "then": { "required": ["text"], "not": { "required": ["json"] } }
+        },
+        {
+          "if": { "required": ["json"] },
+          "then": {
+            "properties": {
+              "name": { "const": "ctrf.json" },
+              "media_type": { "const": "application/json" },
+              "truncated": { "const": false }
+            }
+          }
+        }
+      ]
+    },
+    "redaction": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["rule_id", "count"],
+      "properties": {
+        "rule_id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9._-]{0,127}$" },
+        "count": { "type": "integer", "minimum": 1 }
+      }
+    },
+    "relativePath": {
+      "type": "string",
+      "minLength": 1,
+      "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.?(?:/|$))(?!.*\\\\).+$"
+    },
+    "sha256": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }
+  }
+}

package/integrations/harbor/hitch_benchmark.py

@@ -0,0 +1,252 @@
+"""Generic benchmark package hooks, tool binding and metric normalization.
+
+Runs in the trusted Harbor worker. Package business code is never imported here.
+"""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import math
+from pathlib import Path
+import shlex
+import tempfile
+import uuid
+
+
+def descriptor(environment_dir):
+    path = Path(environment_dir).parent / ".hitch-benchmark.json"
+    return json.loads(path.read_text()) if path.is_file() else None
+
+
+class BenchmarkSession:
+    def __init__(self, env, config):
+        self.env, self.config = env, config
+        self.responses = {}
+        self.started = False
+        self.stopped = False
+        self.failure = None
+        self.journal = env.trial_paths.trial_dir / "benchmark-lifecycle.json"
+
+    async def phase(self, phase):
+        if phase in self.responses:
+            return self.responses[phase]
+        hook = self.config["task"]["lifecycle"][phase]
+        labels = self.env._hitch_ownership_labels
+        request = {
+            "schema_version": "1",
+            "request_id": str(uuid.uuid5(uuid.NAMESPACE_URL, self.env.session_id + ":" + phase)),
+            "phase": phase, "task_id": self.config["task_id"],
+            "logical_trial_id": self.env.session_id, "execution_index": 0,
+            "lease_id": labels.get("io.hitch.lease-id", self.env.session_id),
+            "epoch": int(labels.get("io.hitch.lease-epoch", "1")),
+            "profile_digest": self.config["profile_digest"], "input_refs": [],
+        }
+        # Harbor's exec API accepts shell text. Quote each argv and stdin byte;
+        # no task text is interpolated as shell syntax.
+        command = "printf %s " + shlex.quote(json.dumps(request)) + " | " + shlex.join(hook["argv"])
+        try:
+            result = await self.env.service_exec(command, service=hook["target"].split(":", 1)[1], timeout_sec=math.ceil(hook["timeout_ms"] / 1000))
+            if result.return_code != 0:
+                raise RuntimeError(f"hook {phase} exited {result.return_code}: {result.stderr}")
+            if len(result.stdout or "") > 1024 * 1024:
+                raise RuntimeError("oversized hook response")
+            response = json.loads(result.stdout)
+            if set(response) != {"schema_version", "request_id", "status", "output"}:
+                raise RuntimeError(f"invalid hook response fields: {phase}")
+            if response.get("schema_version") != "1" or response.get("request_id") != request["request_id"] or response.get("status") != "ok":
+                raise RuntimeError(f"invalid/failed hook response: {phase}")
+            output = response["output"]
+            required = {"prepare": "ready", "quiesce": "quiesced", "cleanup": "cleaned"}.get(phase)
+            if required and output.get(required) is not True:
+                raise RuntimeError(f"hook {phase} did not confirm {required}")
+            if phase == "snapshot":
+                paths = self.config["task"]["submission"]["paths"]
+                artifacts = output.get("artifacts", [])
+                if not isinstance(artifacts, list) or any(type(a.get("bytes")) is not int or a["bytes"] < 0 for a in artifacts):
+                    raise RuntimeError("invalid snapshot size metadata")
+                if sorted(a.get("path", "") for a in artifacts) != sorted(paths):
+                    raise RuntimeError("snapshot artifact membership mismatch")
+                size = sum(a["bytes"] for a in artifacts)
+                if size > self.config["task"]["submission"]["max_bytes"] or size < 1:
+                    raise RuntimeError("snapshot size outside package limit")
+            self.responses[phase] = response
+            self.write_journal()
+            return response
+        except BaseException as error:
+            self.failure = {"phase": phase, "error": type(error).__name__}
+            self.write_journal()
+            raise
+
+    def write_journal(self):
+        # Never persist tool tokens or opaque management handles.
+        phases = {p: {"request_id": r["request_id"], "status": r["status"]} for p, r in self.responses.items()}
+        self.journal.parent.mkdir(parents=True, exist_ok=True)
+        self.journal.write_text(json.dumps({"schema_version": "1", "phases": phases, "failure": self.failure}, indent=2))
+
+    async def prepare(self):
+        self.started = True
+        if self.config["task"]["driver"]["kind"] != "tool-server":
+            self.write_journal()
+            return
+        response = await self.phase("prepare")
+        if self.config["task"]["driver"]["config"].get("native_phases"):
+            # There is no candidate identity yet. Only the phase supervisor may
+            # bind and upload a phase token after prepare/setup have completed.
+            phases = self.config["task"]["driver"]["config"]["native_phases"]
+            if (response["output"].get("native_phases_ready") is not True or response["output"].get("tool_bindings", []) != []
+                    or phases["protocol"] == "hitch-native-phase-control@2" and response["output"].get("native_deadline_ready") is not True):
+                self.failure = {"phase": "prepare", "error": "NativePhasePrepareInvalid"}
+                self.write_journal()
+                raise RuntimeError("native phase prepare must confirm readiness without a static binding")
+            return
+        bindings = response["output"]["tool_bindings"]
+        expected = self.config["task"]["driver"]["config"]
+        if len(bindings) != 1 or bindings[0]["endpoint"] != expected["endpoint"] or bindings[0]["tools"] != self.config["tools"]:
+            raise RuntimeError("prepared tool binding differs from locked definition")
+        binding = bindings[0]
+        if not isinstance(binding.get("token"), str) or len(binding["token"]) < 32:
+            raise RuntimeError("missing per-trial tool authorization")
+        with tempfile.TemporaryDirectory(prefix="hitch-binding-") as temp:
+            file = Path(temp) / "binding.json"
+            file.write_text(json.dumps(binding)); file.chmod(0o600)
+            await self.env.upload_file(file, "/tmp/hitch-tool-binding.json")
+        await self.env.upload_file(Path(__file__).with_name("hitch_tool_client.mjs"), "/tmp/hitch-tools.mjs")
+
+    async def snapshot(self):
+        if self.stopped:
+            return
+        if self.config["task"]["driver"]["kind"] != "tool-server":
+            # Harbor collects the native task's declared artifacts. Shared
+            # verifiers retain upstream live-workspace semantics.
+            self.stopped = True
+            return
+        async def collect():
+            await self.phase("quiesce")
+            await self.phase("snapshot")
+        budget = self.config["profile"]["budget"]["collection_timeout_ms"]
+        await asyncio.wait_for(collect(), budget / 1000)
+        self.stopped = True
+
+    async def cleanup(self):
+        if self.started and self.config["task"]["driver"]["kind"] == "tool-server":
+            await self.phase("cleanup")
+
+
+def candidate_instruction(instruction, environment):
+    config = descriptor(environment.environment_dir)
+    if config is None:
+        return instruction, None
+    if config["task"]["driver"]["kind"] != "tool-server":
+        return (json.dumps(config["candidate_input"]) if "candidate_input" in config else instruction), int(config["agent_timeout_sec"] * 1000)
+    instruction += "\n\nTools are available through the locked tool-server bridge. Run `node /tmp/hitch-tools.mjs list` to read tool descriptions and JSON schemas. Invoke a tool using `node /tmp/hitch-tools.mjs TOOL_NAME 'JSON_ARGUMENTS'` (or pass - and JSON via stdin). Complete the requested workflow with these simulated service tools.\n"
+    if "tool-result-images@1" in config["task"]["requirements"]:
+        instruction += "Image tool results contain absolute local paths. Open each image with your native image viewing tool to inspect the observation. The files contain the original image bytes; the JSON metadata is not a visual observation.\n"
+    return instruction, int(config["agent_timeout_sec"] * 1000)
+
+
+async def export_final_response(environment, result):
+    config = descriptor(environment.environment_dir)
+    target = config["task"]["submission"].get("final_response") if config else None
+    if not target:
+        return
+    if not isinstance(result.get("output"), str):
+        raise RuntimeError("canonical candidate final response is unavailable")
+    with tempfile.TemporaryDirectory(prefix="hitch-response-") as temp:
+        source = Path(temp) / "response.json"
+        encoded = json.dumps({"schema_version": "1", "run_id": result["run_id"],
+            "response": result["output"], "termination": result["status"], "source": "hitch-run-result"})
+        if len(encoded.encode()) > config["task"]["submission"]["max_bytes"]:
+            raise RuntimeError("canonical response exceeds submission limit")
+        source.write_text(encoded)
+        # Keep an authoritative copy outside the candidate environment. A
+        # background process cannot replace the response used for grading.
+        (environment.trial_paths.trial_dir / "hitch-final-response.json").write_text(encoded)
+        prepared = await environment.exec("mkdir -p /hitch-evidence")
+        if prepared.return_code != 0:
+            raise RuntimeError("cannot prepare candidate response export")
+        await environment.upload_file(source, target)
+
+
+async def restore_final_response(verifier):
+    config = descriptor(verifier.task.paths.environment_dir)
+    target = config["task"]["submission"].get("final_response") if config else None
+    if not target:
+        return
+    source = verifier.trial_paths.trial_dir / "hitch-final-response.json"
+    if not source.is_file():
+        raise RuntimeError("trusted candidate response evidence is missing")
+    # Called after Harbor uploads the frozen artifacts, in the separate grader.
+    prepared = await verifier.environment.exec("rm -rf -- /hitch-evidence && mkdir -p /hitch-evidence")
+    if prepared.return_code != 0:
+        raise RuntimeError("cannot restore trusted candidate response")
+    await verifier.environment.upload_file(source, target)
+
+
+def validate_collected_submission(verifier):
+    config = descriptor(verifier.task.paths.environment_dir)
+    if not config or config["task"]["driver"]["kind"] == "tool-server":
+        return
+    task = config["task"]
+    if "separate-verifier" not in task["requirements"]:
+        return
+    # Harbor 0.21 mirrors absolute sources directly under artifacts/.
+    root = verifier.trial_paths.trial_dir / "artifacts"
+    seen, size = set(), 0
+    for source in task["submission"]["paths"]:
+        file = root / source.lstrip("/")
+        for ancestor in [file, *file.parents]:
+            if ancestor == root.parent:
+                break
+            if ancestor.is_symlink():
+                raise RuntimeError("collected submission contains a symlink")
+        if not file.exists():
+            raise RuntimeError(f"submission_missing: {source}")
+        for entry in [file, *(file.rglob("*") if file.is_dir() else [])]:
+            if entry.is_symlink():
+                raise RuntimeError("collected submission contains a symlink")
+            if entry.is_file() and entry not in seen:
+                seen.add(entry)
+                size += entry.stat().st_size
+                if size > task["submission"]["max_bytes"]:
+                    raise RuntimeError("collected submission exceeds package limit")
+
+
+def normalize_rewards(verifier, result):
+    config = descriptor(verifier.task.paths.environment_dir)
+    if config is None:
+        return result
+    journal = json.loads((verifier.trial_paths.trial_dir / "benchmark-lifecycle.json").read_text())
+    native = config["task"]["driver"]["kind"] != "tool-server"
+    if journal["failure"] or (not native and not {"prepare", "quiesce", "snapshot"} <= journal["phases"].keys()):
+        raise RuntimeError("benchmark lifecycle did not produce a valid snapshot")
+    directory = verifier.trial_paths.verifier_dir
+    upstream = config["task"]["grading"]["kind"] == "harbor"
+    if (directory / "reward.txt").exists() and not upstream:
+        raise RuntimeError("standard packages require only reward.json")
+    # Read the original JSON before Harbor/Pydantic numeric coercion: bools and
+    # numeric strings must not become valid binary scores via the result model.
+    if (directory / "reward.json").is_file():
+        raw = json.loads((directory / "reward.json").read_text())
+    elif upstream:
+        raw = {"reward": float((directory / "reward.txt").read_text().strip())}
+    else:
+        raise RuntimeError("metric_missing: reward.json")
+    if not isinstance(raw, dict):
+        raise RuntimeError("invalid grader metric object")
+    mapped = {}
+    for name, metric in config["metrics"].items():
+        field = config["task"]["grading"]["metric_map"][name]
+        if field not in raw:
+            raise RuntimeError(f"metric_missing: {field}")
+        value = raw[field]
+        if type(value) not in (float, int) or not math.isfinite(value) or not metric["range"][0] <= value <= metric["range"][1] or (metric["type"] == "binary" and value not in (0, 1)):
+            raise RuntimeError(f"metric_invalid: {field}")
+        mapped[name] = value
+    (directory / "benchmark-rewards.json").write_text(json.dumps({"raw": raw, "metrics": mapped, "primary_metric": config["primary_metric"], "source_task_id": config["task"]["source_task_id"], "task_digest": config["task_digest"]}, indent=2))
+    contract = config.get("score_contract", {"total_score": config["primary_metric"]})
+    if not isinstance(contract, dict) or set(contract) != {"total_score"} or contract["total_score"] not in mapped:
+        raise RuntimeError("invalid standardized score contract")
+    total = mapped[contract["total_score"]]
+    auxiliary = {name: value for name, value in raw.items() if name not in {"reward", "total_score", "process_score"}}
+    return result.model_copy(update={"rewards": {**auxiliary, "reward": total, "total_score": total}})