omnius 1.0.842 → 1.0.845

package/dist/scripts/rapid-computer-use/point_benchmark.py

@@ -0,0 +1,278 @@
+#!/usr/bin/env python3
+"""Evaluate twenty local PNG captures against an already resident /v1/point service.
+
+Does not import a model SDK, load weights, capture a live desktop, or click. The
+caller supplies frozen images and annotations; every request uses model_gate's
+fresh hardware and short-lived statement gate. HTTP shape follows Moondream's
+published point API; a compatible local Photon service must be provisioned
+separately with its exact pinned runtime identity and broker lease.
+"""
+from __future__ import annotations
+
+import argparse
+import base64
+import hashlib
+import math
+from pathlib import Path
+import struct
+import sys
+import time
+
+import model_gate as gate
+
+MAX_IMAGE_BYTES = 20 * 1024 * 1024
+
+
+def number(value):
+    return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value)
+
+
+def rectangle(value, name):
+    gate.require(isinstance(value, dict) and all(number(value.get(key)) for key in ("x", "y", "width", "height"))
+                 and value["width"] > 0 and value["height"] > 0, f"invalid {name} rectangle")
+    return value
+
+
+def inside(point, bounds):
+    return bounds["x"] <= point["x"] < bounds["x"] + bounds["width"] and \
+        bounds["y"] <= point["y"] < bounds["y"] + bounds["height"]
+
+
+def contains(outer, inner):
+    return inner["x"] >= outer["x"] and inner["y"] >= outer["y"] and \
+        inner["x"] + inner["width"] <= outer["x"] + outer["width"] and \
+        inner["y"] + inner["height"] <= outer["y"] + outer["height"]
+
+
+def png_dimensions(raw):
+    gate.require(len(raw) >= 33 and raw[:8] == b"\x89PNG\r\n\x1a\n" and
+                 raw[8:12] == b"\x00\x00\x00\r" and raw[12:16] == b"IHDR", "capture is not a PNG")
+    width, height = struct.unpack(">II", raw[16:24])
+    gate.require(0 < width <= 16384 and 0 < height <= 16384, "PNG dimensions exceed bounds")
+    return width, height
+
+
+def capture_bytes(corpus_path, capture):
+    root = Path(corpus_path).resolve().parent
+    relative = Path(capture.get("file", ""))
+    gate.require(not relative.is_absolute() and relative.suffix.lower() == ".png", "relative PNG path required")
+    path = (root / relative).resolve()
+    gate.require(path.is_relative_to(root), "capture path escapes corpus directory")
+    with path.open("rb") as stream:
+        raw = stream.read(MAX_IMAGE_BYTES + 1)
+    gate.require(0 < len(raw) <= MAX_IMAGE_BYTES, "capture exceeds bounded image size")
+    gate.require(hashlib.sha256(raw).hexdigest() == capture.get("sha256"), "capture bytes changed since annotation")
+    width, height = png_dimensions(raw)
+    transform = capture.get("transform", {})
+    gate.require(transform.get("image", {}).get("width") == width and
+                 transform.get("image", {}).get("height") == height, "PNG dimensions differ from image transform")
+    return raw
+
+
+def verify_annotation(corpus_path, capture):
+    root = Path(corpus_path).resolve().parent
+    relative = Path(capture.get("annotation_receipt", ""))
+    gate.require(not relative.is_absolute() and relative.suffix == ".json", "relative annotation receipt required")
+    path = (root / relative).resolve()
+    gate.require(path.is_relative_to(root), "annotation path escapes corpus directory")
+    with path.open("rb") as stream:
+        raw = stream.read(gate.MAX_JSON_BYTES + 1)
+    gate.require(len(raw) <= gate.MAX_JSON_BYTES and
+                 hashlib.sha256(raw).hexdigest() == capture.get("annotation_receipt_sha256"),
+                 "host annotation receipt hash mismatch")
+    receipt = gate.strict_json(raw)
+    gate.require(receipt.get("schema") == "omnius.computer-use.capture-annotation.v1" and
+                 receipt.get("source") in ("playwright-dom", "native-accessibility"),
+                 "annotation must come from a host semantic observer")
+    for key, source in (("capture_id", "id"), ("capture_sha256", "sha256"),
+                        ("capture_revision", "capture_revision"), ("window_id", "window_id"),
+                        ("target_bounds", "target_bounds"), ("transform", "transform")):
+        gate.require(receipt.get(key) == capture.get(source), f"annotation does not match capture: {key}")
+
+
+def validate_transform(capture):
+    transform = capture.get("transform", {})
+    gate.require(bool(capture.get("capture_revision")) and
+                 transform.get("revision") == capture["capture_revision"], "capture revision mismatch")
+    gate.require(bool(capture.get("window_id")) and transform.get("windowId") == capture["window_id"],
+                 "capture window identity mismatch")
+    dimensions = transform.get("capture", {})
+    gate.require(dimensions.get("x", 0) == 0 and dimensions.get("y", 0) == 0,
+                 "capture origin must be zero; use crop/input offsets for translation")
+    full = rectangle({"x": 0, "y": 0, **dimensions}, "capture")
+    image = rectangle({"x": 0, "y": 0, "width": transform.get("image", {}).get("width"),
+                       "height": transform.get("image", {}).get("height")}, "image")
+    crop = rectangle(transform.get("crop", full), "crop")
+    content = rectangle(transform.get("image", {}).get("content", image), "image content")
+    output = rectangle(transform.get("input"), "input")
+    target = rectangle(capture.get("target_bounds"), "annotated target")
+    gate.require(contains(full, crop) and contains(image, content) and contains(output, target),
+                 "crop, content, or target exceeds its containing rectangle")
+    gate.require(output.get("space") in ("device-pixel", "screen-logical", "css-viewport"),
+                 "input coordinate space must be explicit")
+    return full, image, crop, content, output, target
+
+
+def map_point(point, capture):
+    full, image, crop, content, output, _ = validate_transform(capture)
+    gate.require(isinstance(point, dict) and number(point.get("x")) and number(point.get("y"))
+                 and 0 <= point["x"] < 1 and 0 <= point["y"] < 1, "point must be normalized in [0,1)")
+    pixel = {"x": point["x"] * image["width"], "y": point["y"] * image["height"]}
+    gate.require(inside(pixel, content), "point lies in letterbox padding")
+    original_x = crop["x"] + (pixel["x"] - content["x"]) * crop["width"] / content["width"]
+    original_y = crop["y"] + (pixel["y"] - content["y"]) * crop["height"] / content["height"]
+    mapped = {"x": output["x"] + original_x * output["width"] / full["width"],
+              "y": output["y"] + original_y * output["height"] / full["height"]}
+    gate.require(inside(mapped, output), "mapped point is outside the input surface")
+    return mapped
+
+
+def load_capture_corpus_hashed(path):
+    corpus, sha = gate.read_json_hashed(path)
+    gate.require(corpus.get("schema") == "omnius.computer-use.capture-corpus.v1", "unsupported capture corpus")
+    gate.require(corpus.get("source") in ("local_capture", "synthetic_fixture"), "capture provenance required")
+    captures = corpus.get("captures")
+    gate.require(isinstance(captures, list) and 20 <= len(captures) <= 200,
+                 "twenty to two hundred annotated local captures are required")
+    gate.require(len({item.get("id") for item in captures}) == len(captures), "duplicate capture ID")
+    gate.require(len({item.get("sha256") for item in captures}) == len(captures), "twenty distinct image hashes required")
+    for capture in captures:
+        gate.require(isinstance(capture.get("id"), str) and bool(capture["id"]), "capture ID required")
+        gate.require(isinstance(capture.get("query"), str) and 0 < len(capture["query"]) <= 1000,
+                     "bounded target query required")
+        gate.require(capture.get("annotation_source") in ("human", "fixture_expected_bounds", "host_semantic_receipt"),
+                     "target bounds require independent annotation")
+        if corpus["source"] == "local_capture":
+            gate.require(capture["annotation_source"] in ("human", "host_semantic_receipt"),
+                         "local captures require independent human or host semantic bounds")
+        if capture["annotation_source"] == "host_semantic_receipt":
+            gate.require(gate.re.fullmatch(r"[a-f0-9]{64}", capture.get("annotation_receipt_sha256", "")) is not None,
+                         "semantic bounds require their host receipt hash")
+            verify_annotation(path, capture)
+        validate_transform(capture)
+        capture_bytes(path, capture)
+    return corpus, sha
+
+
+def load_capture_corpus(path):
+    return load_capture_corpus_hashed(path)[0]
+
+
+def point_request(selection, corpus_path, capture):
+    raw = capture_bytes(corpus_path, capture)
+    return {"model": selection["model"], "object": capture["query"],
+            "image_url": "data:image/png;base64," + base64.b64encode(raw).decode("ascii")}
+
+
+def point_answer(response, capture):
+    gate.require(isinstance(response, dict) and isinstance(response.get("points"), list)
+                 and len(response["points"]) <= 100, "invalid bounded point response")
+    points = response["points"]
+    for point in points:
+        gate.require(isinstance(point, dict) and number(point.get("x")) and number(point.get("y"))
+                     and 0 <= point["x"] < 1 and 0 <= point["y"] < 1, "non-normalized point response")
+    if len(points) != 1:
+        return {"status": "abstained", "hit": False, "reason": "missing or ambiguous target points"}
+    mapped = map_point(points[0], capture)
+    return {"status": "ok", "point": points[0], "mapped_point": mapped,
+            "hit": inside(mapped, capture["target_bounds"])}
+
+
+def point_metrics(rows, source):
+    gate.require(source in ("live_model", "synthetic_fixture_replay"), "measurement provenance required")
+    quality = {"target_hit_rate_including_errors": sum(row.get("hit") is True for row in rows) / len(rows)
+               if rows else None, "ambiguous_or_missing_count": sum(row["status"] == "abstained" for row in rows),
+               "error_count": sum(row["status"] == "error" for row in rows)}
+    return {"measurement_kind": source, "count": len(rows),
+            "model_grounding_metrics": quality if source == "live_model" else None,
+            "fixture_contract_metrics": quality if source == "synthetic_fixture_replay" else None,
+            "stage_latency_ms": {"ground": gate.quantiles([row["latency_ms"] for row in rows])},
+            "clicks_performed": 0, "task_completion_claimed": False, "peak_vram_mib": None,
+            "cold_start_measured": False}
+
+
+def run_capture_benchmark(selection, lock, receipt, approval, corpus_path, corpus,
+                          collector=gate.collect_snapshot, sender=gate.local_json,
+                          clock=time.time, timer=time.perf_counter, checkpoint=None):
+    gate.require(selection.get("protocol") == "moondream-point", "point protocol required")
+    gate.require(corpus.get("source") in ("local_capture", "synthetic_fixture"),
+                 "live grounding requires declared local capture provenance")
+    rows = []
+    for capture in corpus["captures"]:
+        request = point_request(selection, corpus_path, capture)
+        try:
+            preflight = gate.verify_preflight(selection, lock, receipt, collector(selection, receipt), clock())
+            gate.verify_approval(approval, preflight, clock())
+        except gate.GateError as error:
+            error.partial_rows = rows
+            raise
+        started = timer()
+        row = {"capture_id": capture["id"], "capture_sha256": capture["sha256"]}
+        try:
+            row.update(point_answer(sender(selection["endpoint"], request, timeout=60), capture))
+        except gate.GateError as error:
+            row.update(status="error", hit=False, reason=str(error))
+        row["latency_ms"] = (timer() - started) * 1000
+        rows.append(row)
+        if checkpoint:
+            checkpoint(rows)
+    return rows
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("command", choices=("check-corpus", "benchmark"))
+    parser.add_argument("--corpus", required=True, type=Path)
+    parser.add_argument("--output", type=Path)
+    parser.add_argument("--selection", type=Path)
+    parser.add_argument("--runtime-receipt", type=Path)
+    parser.add_argument("--approval", type=Path)
+    parser.add_argument("--lock", type=Path, default=gate.DEFAULT_LOCK)
+    args = parser.parse_args(argv)
+    attempted = False
+    try:
+        corpus, corpus_sha = load_capture_corpus_hashed(args.corpus)
+        if args.command == "check-corpus":
+            print(gate.json.dumps({"status": "valid", "count": len(corpus["captures"]),
+                                   "corpus_sha256": corpus_sha, "model_inference_performed": False}))
+            return 0
+        gate.require(all((args.output, args.selection, args.runtime_receipt, args.approval)),
+                     "benchmark requires selection, runtime receipt, approval, and output")
+        selection = gate.read_json(args.selection)
+        lock, lock_sha = gate.read_json_hashed(args.lock)
+        gate.verify_corpus_lock(lock, corpus_sha, point=True)
+        receipt, approval = gate.read_json(args.runtime_receipt), gate.read_json(args.approval)
+        gate.validate_selection(selection, lock)
+        gate.write_json(args.output, {"schema": "omnius.computer-use.point-report.v1", "status": "in_progress",
+                                      "selection": selection, "corpus_sha256": corpus_sha, "rows": []})
+        def checkpoint(rows):
+            gate.write_json(args.output, {"schema": "omnius.computer-use.point-report.v1",
+                                          "status": "in_progress", "selection": selection,
+                                          "corpus_sha256": corpus_sha, "rows": rows})
+        attempted = True
+        rows = run_capture_benchmark(selection, lock, receipt, approval, args.corpus, corpus,
+                                     checkpoint=checkpoint)
+        result = {"schema": "omnius.computer-use.point-report.v1", "status": "completed",
+                  "selection": selection, "corpus_sha256": corpus_sha, "rows": rows,
+                  "corpus_source": corpus["source"],
+                  "corpus_kind": corpus.get("corpus_kind", "operator_supplied_local_captures"),
+                  "limitations": ["Actual model predictions on the declared capture corpus only; synthetic UI inputs do not establish real-application accuracy."],
+                  "models_lock_sha256": lock_sha,
+                  **point_metrics(rows, "live_model")}
+        if any(row["status"] == "error" for row in rows):
+            result["status"] = "completed_with_errors"
+        gate.write_json(args.output, result)
+        print(gate.json.dumps({"status": result["status"], "artifact": str(args.output)}))
+        return 3 if result["status"] == "completed_with_errors" else 0
+    except (gate.GateError, OSError, KeyError, TypeError) as error:
+        failure = {"schema": "omnius.computer-use.gate-failure.v1", "status": "blocked", "reason": str(error),
+                   "inference_status": "possibly_started" if attempted else "not_attempted",
+                   "partial_rows": getattr(error, "partial_rows", [])}
+        if args.output:
+            gate.write_json(args.output, failure)
+        print(gate.json.dumps(failure), file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())