@moleculeagora/cli 0.1.0

package/dist/python-v1/project-test.py

@@ -0,0 +1,256 @@
+import json
+import math
+import os
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    reject_submission,
+    resolve_evaluation_artifact,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    safe_extract_zip,
+    write_score,
+)
+
+
+def require_string(value, label):
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_positive_int(value, label):
+    if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+        fail_runtime(f"{label} must be a positive integer.")
+    return value
+
+
+def require_json_object(path, label):
+    try:
+        data = load_json_file(path, label=label)
+    except FileNotFoundError:
+        fail_runtime(f"{label} does not exist.")
+    except OSError as error:
+        fail_runtime(f"Unable to read {label}: {error}.")
+    except RuntimeError as error:
+        fail_runtime(str(error))
+    if not isinstance(data, dict):
+        fail_runtime(f"{label} must be a JSON object.")
+    return data
+
+
+def resolve_profile_determinism_env(runtime_context):
+    profile = runtime_context.get("runtime_profile")
+    if not isinstance(profile, dict):
+        fail_runtime("runtime_profile must be an object.")
+    determinism_env = profile.get("determinism_env")
+    if not isinstance(determinism_env, dict):
+        fail_runtime("runtime_profile.determinism_env must be an object.")
+    resolved = {}
+    for name, value in determinism_env.items():
+        key = require_string(name, "runtime_profile.determinism_env key")
+        resolved[key] = require_string(
+            value,
+            f"runtime_profile.determinism_env.{key}",
+        )
+    return resolved
+
+
+def require_relative_path(value, label):
+    raw_path = require_string(value, label)
+    relative_path = Path(raw_path)
+    if relative_path.is_absolute() or any(
+        part in {"", ".", ".."} for part in relative_path.parts
+    ):
+        fail_runtime(f"{label} must be a safe relative path inside the criterion bundle.")
+    return relative_path
+
+
+def resolve_inside(root, relative_path, label):
+    resolved_root = root.resolve()
+    resolved_path = (resolved_root / relative_path).resolve()
+    try:
+        resolved_path.relative_to(resolved_root)
+    except ValueError:
+        fail_runtime(f"{label} must stay inside the criterion bundle.")
+    return resolved_path
+
+
+def read_output_json(path):
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except FileNotFoundError:
+        fail_runtime("Criterion entrypoint did not write AGORA_SCORE_OUTPUT_PATH.")
+    except json.JSONDecodeError as error:
+        fail_runtime(f"Criterion score output is not valid JSON: {error.msg}.")
+    except OSError as error:
+        fail_runtime(f"Unable to read criterion score output: {error}.")
+    if not isinstance(data, dict):
+        fail_runtime("Criterion score output must be a JSON object.")
+    return data
+
+
+def require_score(value):
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        fail_runtime("Criterion score output field score must be a number.")
+    score = float(value)
+    if not math.isfinite(score):
+        fail_runtime("Criterion score output field score must be finite.")
+    if score < 0 or score > 1:
+        fail_runtime("Criterion score output field score must be in [0, 1].")
+    return score
+
+
+def truncate_text(value):
+    limit = 4000
+    if len(value) <= limit:
+        return value
+    return value[:limit] + "...<truncated>"
+
+
+def run_criterion(entrypoint_path, criterion_dir, env, timeout_ms):
+    try:
+        return subprocess.run(
+            [sys.executable, str(entrypoint_path)],
+            cwd=str(criterion_dir),
+            env=env,
+            capture_output=True,
+            text=True,
+            timeout=timeout_ms / 1000.0,
+            check=False,
+        )
+    except subprocess.TimeoutExpired:
+        reject_submission(
+            f"Criterion entrypoint exceeded timeout_ms={timeout_ms}.",
+            details={"timeout_ms": timeout_ms},
+        )
+
+
+def build_criterion_env(runtime_context, criterion_dir, submission_dir, output_path):
+    env = {
+        **resolve_profile_determinism_env(runtime_context),
+        "AGORA_CRITERION_DIR": str(criterion_dir),
+        "AGORA_SUBMISSION_DIR": str(submission_dir),
+        "AGORA_SCORE_OUTPUT_PATH": str(output_path),
+        "AGORA_RANDOM_SEED": os.environ.get("AGORA_RANDOM_SEED", "0"),
+    }
+    return env
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    config = require_json_object(config_path, "compiled_config")
+    criterion_role = require_string(
+        config.get("criterion_role"),
+        "compiled_config.criterion_role",
+    )
+    submission_role = require_string(
+        config.get("submission_role"),
+        "compiled_config.submission_role",
+    )
+    manifest_file = require_string(
+        config.get("criterion_manifest_file"),
+        "compiled_config.criterion_manifest_file",
+    )
+    timeout_ms = require_positive_int(
+        config.get("timeout_ms"),
+        "compiled_config.timeout_ms",
+    )
+    final_score_key = require_string(
+        runtime_context.get("final_score_key"),
+        "runtime_context.final_score_key",
+    )
+    criterion_archive_path = resolve_evaluation_artifact(runtime_context, criterion_role)
+    submission_archive_path = resolve_submission_artifact(runtime_context, submission_role)
+
+    with tempfile.TemporaryDirectory(prefix="agora-project-test-") as working_root:
+        working_dir = Path(working_root)
+        criterion_dir = working_dir / "criterion"
+        submission_dir = working_dir / "candidate"
+        score_output_path = working_dir / "criterion-score.json"
+
+        safe_extract_zip(
+            criterion_archive_path,
+            criterion_dir,
+            label=f"evaluation artifact {criterion_role}",
+        )
+        safe_extract_zip(
+            submission_archive_path,
+            submission_dir,
+            label=f"submission artifact {submission_role}",
+            invalid_handler=reject_submission,
+        )
+
+        manifest_path = criterion_dir / manifest_file
+        manifest = require_json_object(manifest_path, manifest_file)
+        version = require_string(
+            manifest.get("version"),
+            f"{manifest_file}.version",
+        )
+        if version != "project_test_v1":
+            fail_runtime(f"Unsupported project-test manifest version {version}.")
+
+        entrypoint_relative_path = require_relative_path(
+            manifest.get("entrypoint"),
+            f"{manifest_file}.entrypoint",
+        )
+        entrypoint_path = resolve_inside(
+            criterion_dir,
+            entrypoint_relative_path,
+            f"{manifest_file}.entrypoint",
+        )
+        if not entrypoint_path.is_file():
+            fail_runtime(f"Criterion entrypoint does not exist: {entrypoint_relative_path}.")
+
+        manifest_timeout_ms = manifest.get("timeout_ms", timeout_ms)
+        manifest_timeout_ms = require_positive_int(
+            manifest_timeout_ms,
+            f"{manifest_file}.timeout_ms",
+        )
+        if manifest_timeout_ms > timeout_ms:
+            fail_runtime(
+                f"{manifest_file}.timeout_ms must be less than or equal to compiled_config.timeout_ms."
+            )
+
+        run = run_criterion(
+            entrypoint_path,
+            criterion_dir,
+            build_criterion_env(
+                runtime_context,
+                criterion_dir,
+                submission_dir,
+                score_output_path,
+            ),
+            manifest_timeout_ms,
+        )
+        if run.returncode != 0:
+            reject_submission(
+                f"Criterion entrypoint exited with code {run.returncode}.",
+                details={
+                    "returncode": run.returncode,
+                    "stdout": truncate_text(run.stdout),
+                    "stderr": truncate_text(run.stderr),
+                },
+            )
+
+        output = read_output_json(score_output_path)
+        score = require_score(output.get("score"))
+        details = output.get("details", {})
+        if not isinstance(details, dict):
+            fail_runtime("Criterion score output field details must be a JSON object.")
+        write_score(score=score, details={**details, final_score_key: score})
+
+
+if __name__ == "__main__":
+    main()