@moleculeagora/cli 0.1.0

package/dist/python-v1/agora_runtime.py

@@ -0,0 +1,282 @@
+import json
+import math
+import os
+import tempfile
+import zipfile
+from pathlib import Path
+from typing import Any, Callable
+
+from runtime_manifest import (
+    load_runtime_manifest,
+    resolve_artifact_by_role,
+    resolve_program_scoring_asset,
+    resolve_scoring_asset_by_role,
+)
+
+INPUT_ROOT = Path(os.environ.get("AGORA_RUNTIME_INPUT_ROOT", "/input"))
+OUTPUT_ROOT = Path(os.environ.get("AGORA_RUNTIME_OUTPUT_ROOT", "/output"))
+OUTPUT_PATH = OUTPUT_ROOT / "score.json"
+MAX_ZIP_MEMBER_COUNT = 1024
+MAX_ZIP_MEMBER_BYTES = 50 * 1024 * 1024
+MAX_ZIP_TOTAL_BYTES = 50 * 1024 * 1024
+_ZIP_COPY_CHUNK_BYTES = 1024 * 1024
+
+
+def _serialize_payload(payload: dict[str, Any]) -> str:
+    return json.dumps(payload, sort_keys=True, separators=(",", ":"))
+
+
+def write_score(
+    *,
+    score: float,
+    details: dict[str, Any] | None = None,
+    ok: bool = True,
+    error: str | None = None,
+) -> None:
+    OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)
+    payload = {
+        "ok": bool(ok),
+        "score": float(score),
+        "details": details or {},
+    }
+    if error is not None:
+        payload["error"] = error
+    OUTPUT_PATH.write_text(_serialize_payload(payload), encoding="utf-8")
+
+
+def fail_runtime(message: str) -> None:
+    write_score(score=0.0, details={}, ok=False, error=message)
+    raise SystemExit(1)
+
+
+def reject_submission(
+    message: str,
+    *,
+    details: dict[str, Any] | None = None,
+) -> None:
+    write_score(score=0.0, details=details or {}, ok=False, error=message)
+    raise SystemExit(0)
+
+
+def load_runtime_context(
+    *,
+    input_dir: Path | None = None,
+    fail_runtime_handler: Callable[[str], None] | None = None,
+) -> dict[str, Any]:
+    fail_handler = fail_runtime_handler or fail_runtime
+    runtime_manifest = load_runtime_manifest(
+        input_dir=input_dir or INPUT_ROOT,
+        fail_runtime=fail_handler,
+    )
+    runtime_manifest["program_asset"] = resolve_program_scoring_asset(
+        runtime_manifest,
+        fail_runtime=fail_handler,
+        supported_abi_versions={"python-v1"},
+    )
+    return runtime_manifest
+
+
+def resolve_evaluation_artifact(
+    runtime_context: dict[str, Any],
+    role: str,
+    *,
+    fail_runtime_handler: Callable[[str], None] | None = None,
+) -> Path:
+    fail_handler = fail_runtime_handler or fail_runtime
+    artifact = resolve_artifact_by_role(
+        runtime_context,
+        lane="evaluation",
+        role=role,
+        fail_runtime=fail_handler,
+    )
+    if artifact["path"] is None:
+        fail_handler(f"Missing required evaluation artifact for role {role}.")
+    return artifact["path"]
+
+
+def resolve_submission_artifact(
+    runtime_context: dict[str, Any],
+    role: str,
+    *,
+    fail_runtime_handler: Callable[[str], None] | None = None,
+) -> Path:
+    fail_handler = fail_runtime_handler or fail_runtime
+    artifact = resolve_artifact_by_role(
+        runtime_context,
+        lane="submission",
+        role=role,
+        fail_runtime=fail_handler,
+    )
+    if artifact["path"] is None:
+        reject_submission(f"Missing required submission artifact for role {role}.")
+    return artifact["path"]
+
+
+def resolve_scoring_asset(
+    runtime_context: dict[str, Any],
+    role: str,
+    *,
+    kind: str | None = None,
+    fail_runtime_handler: Callable[[str], None] | None = None,
+) -> Path:
+    fail_handler = fail_runtime_handler or fail_runtime
+    asset = resolve_scoring_asset_by_role(
+        runtime_context,
+        role=role,
+        kind=kind,
+        fail_runtime=fail_handler,
+    )
+    return asset["path"]
+
+
+def load_json_file(path: Path, *, label: str | None = None) -> Any:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError as error:
+        human_label = label or str(path)
+        raise RuntimeError(f"{human_label} is missing") from error
+    except json.JSONDecodeError as error:
+        human_label = label or str(path)
+        raise RuntimeError(f"{human_label} is not valid JSON: {error.msg}") from error
+
+
+def _require_finite_score(value: Any, *, label: str) -> float:
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        raise RuntimeError(f"{label}.score must be a finite number.")
+    score = float(value)
+    if not math.isfinite(score):
+        raise RuntimeError(f"{label}.score must be a finite number.")
+    return score
+
+
+def load_score_output(path: Path, *, label: str) -> dict[str, Any]:
+    payload = load_json_file(path, label=label)
+    if not isinstance(payload, dict):
+        raise RuntimeError(f"{label} must be a JSON object.")
+
+    ok = payload.get("ok")
+    if not isinstance(ok, bool):
+        raise RuntimeError(f"{label}.ok must be a boolean.")
+
+    details = payload.get("details", {})
+    if not isinstance(details, dict):
+        raise RuntimeError(f"{label}.details must be an object when present.")
+
+    if "score" not in payload:
+        raise RuntimeError(f"{label}.score is missing.")
+    score = _require_finite_score(payload.get("score"), label=label)
+
+    error = payload.get("error")
+    if error is not None and not isinstance(error, str):
+        raise RuntimeError(f"{label}.error must be a string when present.")
+    if ok is False and (error is None or not error.strip()):
+        raise RuntimeError(f"{label}.error must explain ok=false outputs.")
+
+    normalized = dict(payload)
+    normalized["ok"] = ok
+    normalized["score"] = score
+    normalized["details"] = details
+    return normalized
+
+
+def load_text_file(path: Path) -> str:
+    return path.read_text(encoding="utf-8")
+
+
+def safe_extract_zip(
+    archive_path: Path,
+    destination: Path,
+    *,
+    label: str,
+    invalid_handler: Callable[[str], None] | None = None,
+) -> None:
+    handler = invalid_handler or fail_runtime
+    root = destination.resolve()
+    root.mkdir(parents=True, exist_ok=True)
+    try:
+        archive = zipfile.ZipFile(archive_path, "r")
+    except zipfile.BadZipFile:
+        handler(f"{label} is not a valid ZIP archive.")
+        return
+
+    with archive:
+        members = archive.infolist()
+        if len(members) > MAX_ZIP_MEMBER_COUNT:
+            handler(
+                f"{label} contains {len(members)} archive entries, exceeding the "
+                f"maximum ZIP member count of {MAX_ZIP_MEMBER_COUNT}. Next step: "
+                "remove unnecessary files and upload a smaller ZIP archive."
+            )
+            return
+
+        extraction_plan = []
+        for member in members:
+            name = member.filename
+            member_path = Path(name)
+            if (
+                not name
+                or member_path.is_absolute()
+                or any(part in {"", ".", ".."} for part in member_path.parts)
+            ):
+                handler(f"{label} contains unsafe archive path {name!r}.")
+                return
+            target = (root / member_path).resolve()
+            try:
+                target.relative_to(root)
+            except ValueError:
+                handler(f"{label} contains archive path outside extraction root: {name!r}.")
+                return
+            extraction_plan.append((member, target))
+
+        total_bytes = 0
+        for member, target in extraction_plan:
+            if member.is_dir():
+                target.mkdir(parents=True, exist_ok=True)
+                continue
+
+            name = member.filename
+            target.parent.mkdir(parents=True, exist_ok=True)
+            member_bytes = 0
+            temp_path = None
+            try:
+                with archive.open(member, "r") as source:
+                    with tempfile.NamedTemporaryFile(
+                        "wb",
+                        dir=target.parent,
+                        prefix=".agora-zip-",
+                        delete=False,
+                    ) as output:
+                        temp_path = Path(output.name)
+                        while True:
+                            chunk = source.read(_ZIP_COPY_CHUNK_BYTES)
+                            if not chunk:
+                                break
+                            member_bytes += len(chunk)
+                            total_bytes += len(chunk)
+                            if member_bytes > MAX_ZIP_MEMBER_BYTES:
+                                output.close()
+                                temp_path.unlink(missing_ok=True)
+                                handler(
+                                    f"{label} member {name!r} exceeds the maximum ZIP "
+                                    f"member size of {MAX_ZIP_MEMBER_BYTES} bytes. "
+                                    "Next step: reduce that file and upload a ZIP "
+                                    "within the Python-v1 artifact limits."
+                                )
+                                return
+                            if total_bytes > MAX_ZIP_TOTAL_BYTES:
+                                output.close()
+                                temp_path.unlink(missing_ok=True)
+                                handler(
+                                    f"{label} exceeds the maximum ZIP extracted size "
+                                    f"of {MAX_ZIP_TOTAL_BYTES} bytes. Next step: "
+                                    "remove unnecessary files or upload a smaller "
+                                    "ZIP archive."
+                                )
+                                return
+                            output.write(chunk)
+                temp_path.replace(target)
+            except zipfile.BadZipFile:
+                if temp_path is not None:
+                    temp_path.unlink(missing_ok=True)
+                handler(f"{label} is not a valid ZIP archive.")
+                return

package/dist/python-v1/answer-set-metric.py

@@ -0,0 +1,264 @@
+import math
+import re
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    reject_submission,
+    resolve_evaluation_artifact,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    write_score,
+)
+
+SUPPORTED_METRICS = ("precision", "recall", "f1", "iou")
+SUPPORTED_ID_SPACES = ("opaque", "doi", "pmid", "arxiv")
+DOI_PATTERN = re.compile(r"^10\.[0-9]{4,9}/[!-~]+$")
+PMID_PATTERN = re.compile(r"^[0-9]+$")
+ARXIV_PATTERN = re.compile(
+    r"^(?:[a-z-]+(?:\.[a-z]{2})?/[0-9]{7}|[0-9]{4}\.[0-9]{4,5})(?:v[0-9]+)?$"
+)
+
+
+def format_value_list(values):
+    ordered = list(values)
+    if len(ordered) == 1:
+        return ordered[0]
+    if len(ordered) == 2:
+        return f"{ordered[0]} or {ordered[1]}"
+    return f"{', '.join(ordered[:-1])}, or {ordered[-1]}"
+
+
+def require_string(value, label):
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_config_value(config, key, supported_values):
+    value = require_string(config.get(key), f"compiled_config.{key}").lower()
+    if value not in supported_values:
+        fail_runtime(
+            f"compiled_config.{key} must be one of {format_value_list(supported_values)}."
+        )
+    return value
+
+
+def require_json_slot(runtime_context, lane, role):
+    artifact_contract = runtime_context.get("artifact_contract")
+    if not isinstance(artifact_contract, dict):
+        fail_runtime("Runtime context is missing artifact_contract.")
+    slots = artifact_contract.get(lane)
+    if not isinstance(slots, list):
+        fail_runtime(f"Runtime context is missing artifact_contract.{lane}.")
+    for slot in slots:
+        if not isinstance(slot, dict) or slot.get("role") != role:
+            continue
+        validator = slot.get("validator")
+        if not isinstance(validator, dict) or validator.get("kind") not in {
+            "json_document",
+            "json_schema",
+        }:
+            fail_runtime(
+                f"{lane} role {role} must use validator.kind=json_document or json_schema for answer_set_metric."
+            )
+        return
+    fail_runtime(f"Runtime context is missing {lane} slot for role {role}.")
+
+
+def load_json_payload(path, label, invalid_handler):
+    try:
+        return load_json_file(path, label=label)
+    except RuntimeError as error:
+        invalid_handler(str(error))
+
+
+def require_ascii(value, label, invalid_handler):
+    try:
+        value.encode("ascii")
+    except UnicodeEncodeError:
+        invalid_handler(f"{label} must contain ASCII characters only.")
+    return value
+
+
+def normalize_answer_id(raw_value, id_space, label, invalid_handler):
+    if not isinstance(raw_value, str):
+        invalid_handler(f"{label} must be a string.")
+    value = raw_value.strip()
+    if not value:
+        invalid_handler(f"{label} must be a non-empty string after trimming.")
+
+    if id_space == "opaque":
+        return value
+
+    ascii_value = require_ascii(value, label, invalid_handler)
+    if id_space == "pmid":
+        if not PMID_PATTERN.fullmatch(ascii_value):
+            invalid_handler(f"{label} must be a bare PMID containing ASCII digits only.")
+        return ascii_value
+
+    normalized = ascii_value.lower()
+    if id_space == "doi":
+        if not DOI_PATTERN.fullmatch(normalized):
+            invalid_handler(
+                f"{label} must be a bare DOI such as 10.1234/example, not a URL or alias."
+            )
+        return normalized
+
+    if not ARXIV_PATTERN.fullmatch(normalized):
+        invalid_handler(
+            f"{label} must be a canonical arXiv id string, not a URL or prefixed alias."
+        )
+    return normalized
+
+
+def load_answer_set(path, role, id_space, *, lane, invalid_handler):
+    payload = load_json_payload(path, f"{lane} artifact {role}", invalid_handler)
+    if not isinstance(payload, dict):
+        invalid_handler(f"{lane} artifact {role} must be a JSON object.")
+    items = payload.get("items")
+    if not isinstance(items, list):
+        invalid_handler(f"{lane} artifact {role}.items must be an array.")
+
+    answer_ids = set()
+    for index, item in enumerate(items):
+        normalized = normalize_answer_id(
+            item,
+            id_space,
+            f"{lane} artifact {role}.items[{index}]",
+            invalid_handler,
+        )
+        if normalized in answer_ids:
+            invalid_handler(
+                f"{lane} artifact {role} contains duplicate normalized id {normalized!r}."
+            )
+        answer_ids.add(normalized)
+
+    if lane == "evaluation" and not answer_ids:
+        invalid_handler(f"evaluation artifact {role}.items must contain at least one id.")
+    return answer_ids
+
+
+def compute_counts(reference_ids, candidate_ids):
+    true_positive_count = len(reference_ids & candidate_ids)
+    false_positive_count = len(candidate_ids - reference_ids)
+    false_negative_count = len(reference_ids - candidate_ids)
+    return true_positive_count, false_positive_count, false_negative_count
+
+
+def compute_metric(metric, true_positive_count, false_positive_count, false_negative_count):
+    candidate_count = true_positive_count + false_positive_count
+    reference_count = true_positive_count + false_negative_count
+    if metric == "precision":
+        if candidate_count == 0:
+            return 0.0
+        return true_positive_count / candidate_count
+    if metric == "recall":
+        return true_positive_count / reference_count
+    if metric == "f1":
+        precision = 0.0 if candidate_count == 0 else true_positive_count / candidate_count
+        recall = true_positive_count / reference_count
+        if precision + recall == 0.0:
+            return 0.0
+        return (2.0 * precision * recall) / (precision + recall)
+
+    union_count = true_positive_count + false_positive_count + false_negative_count
+    return true_positive_count / union_count
+
+
+def require_finite_unit_interval(value, label):
+    if not math.isfinite(value):
+        fail_runtime(f"{label} must be finite.")
+    if value < 0.0:
+        return 0.0
+    if value > 1.0:
+        return 1.0
+    return value
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    config = load_json_payload(
+        config_path,
+        "compiled_config",
+        fail_runtime,
+    )
+    if not isinstance(config, dict):
+        fail_runtime("compiled_config must be a JSON object.")
+
+    metric = require_config_value(config, "metric", SUPPORTED_METRICS)
+    id_space = require_config_value(config, "id_space", SUPPORTED_ID_SPACES)
+    reference_role = require_string(
+        config.get("reference_role"),
+        "compiled_config.reference_role",
+    )
+    candidate_role = require_string(
+        config.get("candidate_role"),
+        "compiled_config.candidate_role",
+    )
+    final_score_key = require_string(
+        config.get("final_score_key"),
+        "compiled_config.final_score_key",
+    )
+
+    objective = require_string(
+        runtime_context.get("objective"),
+        "runtime_context.objective",
+    )
+    if objective != "maximize":
+        fail_runtime(f"answer_set_metric metric {metric} requires objective=maximize.")
+
+    require_json_slot(runtime_context, "evaluation", reference_role)
+    require_json_slot(runtime_context, "submission", candidate_role)
+    reference_path = resolve_evaluation_artifact(runtime_context, reference_role)
+    candidate_path = resolve_submission_artifact(runtime_context, candidate_role)
+    reference_ids = load_answer_set(
+        reference_path,
+        reference_role,
+        id_space,
+        lane="evaluation",
+        invalid_handler=fail_runtime,
+    )
+    candidate_ids = load_answer_set(
+        candidate_path,
+        candidate_role,
+        id_space,
+        lane="submission",
+        invalid_handler=reject_submission,
+    )
+
+    true_positive_count, false_positive_count, false_negative_count = compute_counts(
+        reference_ids,
+        candidate_ids,
+    )
+    raw_metric = compute_metric(
+        metric,
+        true_positive_count,
+        false_positive_count,
+        false_negative_count,
+    )
+    score = require_finite_unit_interval(raw_metric, "answer_set_metric score")
+    write_score(
+        score=score,
+        details={
+            final_score_key: score,
+            "selected_metric": metric,
+            "selected_id_space": id_space,
+            "reference_count": len(reference_ids),
+            "candidate_count": len(candidate_ids),
+            "true_positive_count": true_positive_count,
+            "false_positive_count": false_positive_count,
+            "false_negative_count": false_negative_count,
+            "selected_metric_value": raw_metric,
+        },
+    )
+
+
+if __name__ == "__main__":
+    main()