@moleculeagora/cli 0.1.0

package/dist/python-v1/rubric-validation.py

@@ -0,0 +1,246 @@
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    reject_submission,
+    resolve_evaluation_artifact,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    write_score,
+)
+
+
+def require_string(value, label):
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_string_list(value, label):
+    if value is None:
+        return []
+    if not isinstance(value, list):
+        fail_runtime(f"{label} must be an array of strings.")
+    normalized = []
+    for index, item in enumerate(value):
+        if not isinstance(item, str) or not item.strip():
+            fail_runtime(f"{label}[{index}] must be a non-empty string.")
+        normalized.append(item.strip())
+    return normalized
+
+
+def require_number(value, label):
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        fail_runtime(f"{label} must be a number.")
+    return float(value)
+
+
+def require_positive_number(value, label):
+    number = require_number(value, label)
+    if number <= 0:
+        fail_runtime(f"{label} must be positive.")
+    return number
+
+
+def require_positive_int(value, label):
+    if isinstance(value, bool) or not isinstance(value, int):
+        fail_runtime(f"{label} must be a positive integer.")
+    if value <= 0:
+        fail_runtime(f"{label} must be a positive integer.")
+    return value
+
+
+def require_object(value, label):
+    if not isinstance(value, dict):
+        fail_runtime(f"{label} must be a JSON object.")
+    return value
+
+
+def normalize_check(value, index):
+    check = require_object(value, f"rubric.checks[{index}]")
+    rule = require_string(check.get("rule"), f"rubric.checks[{index}].rule")
+    normalized = {
+        "id": require_string(check.get("id"), f"rubric.checks[{index}].id"),
+        "weight": require_positive_number(check.get("weight"), f"rubric.checks[{index}].weight"),
+        "rule": rule,
+    }
+    if rule == "required_path":
+        normalized["path"] = require_string(check.get("path"), f"rubric.checks[{index}].path")
+        return normalized
+    if rule == "string_min_length":
+        normalized["path"] = require_string(check.get("path"), f"rubric.checks[{index}].path")
+        normalized["min"] = require_positive_int(check.get("min"), f"rubric.checks[{index}].min")
+        return normalized
+    if rule == "array_min_length":
+        normalized["path"] = require_string(check.get("path"), f"rubric.checks[{index}].path")
+        normalized["min"] = require_positive_int(check.get("min"), f"rubric.checks[{index}].min")
+        return normalized
+    if rule == "forbidden_terms_absent":
+        normalized["paths"] = require_string_list(
+            check.get("paths"),
+            f"rubric.checks[{index}].paths",
+        )
+        normalized["terms"] = require_string_list(
+            check.get("terms"),
+            f"rubric.checks[{index}].terms",
+        )
+        if not normalized["paths"] or not normalized["terms"]:
+            fail_runtime(f"rubric.checks[{index}] must declare non-empty paths and terms.")
+        return normalized
+    if rule == "enum":
+        normalized["path"] = require_string(check.get("path"), f"rubric.checks[{index}].path")
+        normalized["allowed"] = require_string_list(
+            check.get("allowed"),
+            f"rubric.checks[{index}].allowed",
+        )
+        if not normalized["allowed"]:
+            fail_runtime(f"rubric.checks[{index}].allowed must be non-empty.")
+        return normalized
+    fail_runtime(
+        f"rubric.checks[{index}].rule={rule} is not supported. Supported rules are required_path, string_min_length, array_min_length, forbidden_terms_absent, enum."
+    )
+
+
+def normalize_aggregation(rubric):
+    aggregation = rubric.get("aggregation")
+    aggregation = require_object(aggregation, "rubric.aggregation")
+    strategy = require_string(aggregation.get("strategy"), "rubric.aggregation.strategy")
+    if strategy != "weighted_average":
+        fail_runtime(
+            f"rubric.aggregation.strategy={strategy} is not supported. Supported strategy is weighted_average."
+        )
+    return strategy
+
+
+def normalize_checks(rubric):
+    checks_value = rubric.get("checks")
+    if not isinstance(checks_value, list) or not checks_value:
+        fail_runtime("rubric.checks must be a non-empty array.")
+    checks = [normalize_check(check, index) for index, check in enumerate(checks_value)]
+    seen_ids = set()
+    for check in checks:
+        if check["id"] in seen_ids:
+            fail_runtime(f"rubric.checks id {check['id']} is duplicated.")
+        seen_ids.add(check["id"])
+    return checks
+
+
+def has_required_content(value):
+    if value is None:
+        return False
+    if isinstance(value, str):
+        return len(value.strip()) > 0
+    if isinstance(value, (list, dict)):
+        return len(value) > 0
+    return True
+
+
+def evaluate_check(check, record):
+    rule = check["rule"]
+    if rule == "required_path":
+        return check["path"] in record and has_required_content(record.get(check["path"]))
+    if rule == "string_min_length":
+        value = record.get(check["path"])
+        return isinstance(value, str) and len(value.strip()) >= check["min"]
+    if rule == "array_min_length":
+        value = record.get(check["path"])
+        return isinstance(value, list) and len(value) >= check["min"]
+    if rule == "forbidden_terms_absent":
+        lowered_terms = [term.lower() for term in check["terms"]]
+        for path in check["paths"]:
+            value = record.get(path)
+            if not isinstance(value, str):
+                continue
+            lowered_value = value.lower()
+            if any(term in lowered_value for term in lowered_terms):
+                return False
+        return True
+    if rule == "enum":
+        value = record.get(check["path"])
+        return isinstance(value, str) and value in check["allowed"]
+    fail_runtime(f"rubric.checks rule {rule} is not supported.")
+
+
+def load_json_object(path, label, *, reject_invalid):
+    try:
+        data = load_json_file(path, label=label)
+    except RuntimeError as error:
+        if reject_invalid:
+            reject_submission(str(error))
+        fail_runtime(str(error))
+    if not isinstance(data, dict):
+        message = f"{label} must be a JSON object."
+        if reject_invalid:
+            reject_submission(message)
+        fail_runtime(message)
+    return data
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    try:
+        config = load_json_file(config_path, label="compiled_config")
+    except RuntimeError as error:
+        fail_runtime(str(error))
+    rubric_role = require_string(config.get("evaluation_role"), "compiled_config.evaluation_role")
+    submission_role = require_string(config.get("submission_role"), "compiled_config.submission_role")
+    final_score_key = require_string(
+        runtime_context.get("final_score_key"),
+        "runtime_context.final_score_key",
+    )
+    rubric_path = resolve_evaluation_artifact(runtime_context, rubric_role)
+    submission_path = resolve_submission_artifact(runtime_context, submission_role)
+    rubric = load_json_object(
+        rubric_path,
+        f"evaluation artifact {rubric_role}",
+        reject_invalid=False,
+    )
+    record = load_json_object(
+        submission_path,
+        f"submission artifact {submission_role}",
+        reject_invalid=True,
+    )
+    aggregation_strategy = normalize_aggregation(rubric)
+    checks = normalize_checks(rubric)
+    total_weight = sum(check["weight"] for check in checks)
+    if total_weight <= 0:
+        fail_runtime("rubric.checks must declare a positive total weight.")
+    check_results = []
+    earned_weight = 0.0
+    for check in checks:
+        passed = evaluate_check(check, record)
+        rule_score = 1.0 if passed else 0.0
+        earned_weight += check["weight"] * rule_score
+        check_results.append(
+            {
+                "id": check["id"],
+                "rule": check["rule"],
+                "weight": check["weight"],
+                "score": rule_score,
+                "passed": passed,
+            }
+        )
+    score = earned_weight / total_weight
+    passed_checks = sum(1 for result in check_results if result["passed"])
+    write_score(
+        score=score,
+        details={
+            final_score_key: score,
+            "validated_fields": passed_checks,
+            "passed_checks": passed_checks,
+            "total_checks": len(checks),
+            "earned_weight": earned_weight,
+            "total_weight": total_weight,
+            "aggregation_strategy": aggregation_strategy,
+            "check_results": check_results,
+        },
+    )
+
+
+if __name__ == "__main__":
+    main()

package/dist/python-v1/solver-python-stdio-test.py

@@ -0,0 +1,160 @@
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    resolve_evaluation_artifact,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    safe_extract_zip,
+    write_score,
+)
+
+
+def require_string(value, label):
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_int(value, label):
+    if not isinstance(value, int) or value <= 0:
+        fail_runtime(f"{label} must be a positive integer.")
+    return value
+
+
+def load_json_object(path, label):
+    try:
+        data = load_json_file(path, label=label)
+    except RuntimeError as error:
+        fail_runtime(str(error))
+    if not isinstance(data, dict):
+        fail_runtime(f"{label} must be a JSON object.")
+    return data
+
+
+def read_required_text(path, label):
+    try:
+        return path.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        fail_runtime(f"Missing {label} at {path}.")
+    except OSError as error:
+        fail_runtime(f"Unable to read {label}: {error}.")
+
+
+def normalize_output(text, strip_trailing_whitespace):
+    if strip_trailing_whitespace:
+        return text.rstrip()
+    return text
+
+
+def run_python_solution(solution_path, stdin_text, timeout_ms, working_dir):
+    try:
+        return subprocess.run(
+            [sys.executable, str(solution_path)],
+            input=stdin_text,
+            capture_output=True,
+            text=True,
+            timeout=timeout_ms / 1000.0,
+            cwd=str(working_dir),
+            check=False,
+        )
+    except subprocess.TimeoutExpired:
+        return None
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    config = load_json_object(config_path, "compiled_config")
+    harness_role = require_string(config.get("evaluation_role"), "compiled_config.evaluation_role")
+    solution_role = require_string(config.get("submission_role"), "compiled_config.submission_role")
+    final_score_key = require_string(
+        runtime_context.get("final_score_key"),
+        "runtime_context.final_score_key",
+    )
+    harness_path = resolve_evaluation_artifact(runtime_context, harness_role)
+    solution_path = resolve_submission_artifact(runtime_context, solution_role)
+    with tempfile.TemporaryDirectory(prefix="agora-harness-") as working_root:
+        working_dir = Path(working_root)
+        safe_extract_zip(
+            harness_path,
+            working_dir,
+            label=f"evaluation artifact {harness_role}",
+        )
+        manifest_path = working_dir / "agora-harness.json"
+        harness_manifest = load_json_object(manifest_path, "agora-harness.json")
+        version = require_string(harness_manifest.get("version"), "agora-harness.version")
+        language = require_string(harness_manifest.get("language"), "agora-harness.language")
+        if version != "v1":
+            fail_runtime(f"Unsupported harness version {version}.")
+        if language != "python":
+            fail_runtime(f"Unsupported harness language {language}.")
+        timeout_ms = require_int(
+            harness_manifest.get("timeout_ms", 30000),
+            "agora-harness.timeout_ms",
+        )
+        strip_trailing_whitespace = bool(
+            harness_manifest.get("strip_trailing_whitespace", False)
+        )
+        tests = harness_manifest.get("tests")
+        if not isinstance(tests, list) or len(tests) == 0:
+            fail_runtime("agora-harness.tests must declare at least one test case.")
+        passed_tests = 0
+        for index, test_case in enumerate(tests):
+            if not isinstance(test_case, dict):
+                fail_runtime(f"agora-harness.tests[{index}] must be an object.")
+            stdin_path = working_dir / require_string(
+                test_case.get("stdin_path"),
+                f"agora-harness.tests[{index}].stdin_path",
+            )
+            expected_stdout_path = working_dir / require_string(
+                test_case.get("expected_stdout_path"),
+                f"agora-harness.tests[{index}].expected_stdout_path",
+            )
+            stdin_text = read_required_text(
+                stdin_path,
+                f"test stdin {stdin_path.name}",
+            )
+            expected_stdout = normalize_output(
+                read_required_text(
+                    expected_stdout_path,
+                    f"test expected stdout {expected_stdout_path.name}",
+                ),
+                strip_trailing_whitespace,
+            )
+            run = run_python_solution(
+                solution_path,
+                stdin_text,
+                timeout_ms,
+                working_dir,
+            )
+            if run is None:
+                continue
+            if run.returncode != 0:
+                continue
+            actual_stdout = normalize_output(run.stdout, strip_trailing_whitespace)
+            if actual_stdout == expected_stdout:
+                passed_tests += 1
+        total_tests = len(tests)
+        score = passed_tests / total_tests
+        write_score(
+            score=score,
+            details={
+                final_score_key: score,
+                "tests_passed": passed_tests,
+                "total_tests": total_tests,
+            },
+        )
+
+
+if __name__ == "__main__":
+    main()