@moleculeagora/cli 0.1.0

package/dist/python-v1/reference-artifact-assertion.py

@@ -0,0 +1,532 @@
+import csv
+import json
+import math
+import re
+import tempfile
+from pathlib import Path
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    reject_submission,
+    resolve_evaluation_artifact,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    safe_extract_zip,
+    write_score,
+)
+
+FIELD_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
+
+
+def require_string(value, label):
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_object(value, label):
+    if not isinstance(value, dict):
+        fail_runtime(f"{label} must be a JSON object.")
+    return value
+
+
+def require_list(value, label):
+    if not isinstance(value, list):
+        fail_runtime(f"{label} must be an array.")
+    return value
+
+
+def require_number(value, label):
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        fail_runtime(f"{label} must be a finite number.")
+    number = float(value)
+    if not math.isfinite(number):
+        fail_runtime(f"{label} must be a finite number.")
+    return number
+
+
+def require_positive_number(value, label):
+    number = require_number(value, label)
+    if number <= 0:
+        fail_runtime(f"{label} must be positive.")
+    return number
+
+
+def require_nonnegative_number(value, label):
+    number = require_number(value, label)
+    if number < 0:
+        fail_runtime(f"{label} must be nonnegative.")
+    return number
+
+
+def require_positive_int(value, label):
+    if isinstance(value, bool) or not isinstance(value, int) or value <= 0:
+        fail_runtime(f"{label} must be a positive integer.")
+    return value
+
+
+def require_json_scalar(value, label):
+    if (
+        value is None
+        or isinstance(value, str)
+        or isinstance(value, bool)
+        or (
+            isinstance(value, (int, float))
+            and not isinstance(value, bool)
+            and math.isfinite(float(value))
+        )
+    ):
+        return value
+    fail_runtime(f"{label} must be a JSON scalar.")
+
+
+def normalize_member_path(value, label):
+    path_value = require_string(value, label)
+    path_parts = path_value.split("/")
+    member_path = Path(path_value)
+    if member_path.is_absolute() or any(
+        part in {"", ".", ".."} for part in path_parts
+    ):
+        fail_runtime(f"{label} must be a safe relative archive path.")
+    return path_value
+
+
+def resolve_member(root, path_value, label):
+    relative_path = normalize_member_path(path_value, label)
+    root_path = root.resolve()
+    target = (root_path / relative_path).resolve()
+    try:
+        target.relative_to(root_path)
+    except ValueError:
+        fail_runtime(f"{label} must stay inside the extracted archive root.")
+    return target
+
+
+def load_compiled_config(path):
+    try:
+        return load_json_file(path, label="compiled_config")
+    except RuntimeError as error:
+        fail_runtime(str(error))
+
+
+def read_text(path, label):
+    try:
+        return True, path.read_text(encoding="utf-8"), None
+    except FileNotFoundError:
+        return False, None, f"{label} is missing."
+    except UnicodeDecodeError:
+        return False, None, f"{label} is not valid UTF-8 text."
+    except OSError as error:
+        return False, None, f"{label} could not be read: {error}."
+
+
+def load_json_member(path, label):
+    try:
+        return True, json.loads(path.read_text(encoding="utf-8")), None
+    except FileNotFoundError:
+        return False, None, f"{label} is missing."
+    except UnicodeDecodeError:
+        return False, None, f"{label} is not valid UTF-8 text."
+    except json.JSONDecodeError as error:
+        return False, None, f"{label} is not valid JSON: {error.msg}."
+    except OSError as error:
+        return False, None, f"{label} could not be read: {error}."
+
+
+def parse_json_path(path_value, label):
+    path = require_string(path_value, label)
+    if path == "$":
+        return []
+    if not path.startswith("$"):
+        fail_runtime(f"{label} must start with $.")
+    tokens = []
+    index = 1
+    while index < len(path):
+        if path[index] == ".":
+            index += 1
+            match = FIELD_RE.match(path, index)
+            if not match:
+                fail_runtime(f"{label} contains an invalid field segment.")
+            tokens.append(("field", match.group(0)))
+            index = match.end()
+            continue
+        if path[index] == "[":
+            end = path.find("]", index)
+            if end == -1:
+                fail_runtime(f"{label} contains an unterminated array index.")
+            raw_index = path[index + 1 : end]
+            if not raw_index.isdigit() or (
+                len(raw_index) > 1 and raw_index.startswith("0")
+            ):
+                fail_runtime(f"{label} contains an invalid array index.")
+            tokens.append(("index", int(raw_index)))
+            index = end + 1
+            continue
+        fail_runtime(f"{label} contains an unsupported path segment.")
+    return tokens
+
+
+def extract_json_path(data, tokens):
+    current = data
+    for kind, token in tokens:
+        if kind == "field":
+            if not isinstance(current, dict) or token not in current:
+                return False, None, f"JSON field {token} is missing."
+            current = current[token]
+            continue
+        if not isinstance(current, list) or token >= len(current):
+            return False, None, f"JSON array index {token} is missing."
+        current = current[token]
+    return True, current, None
+
+
+def json_scalar_equal(candidate_value, reference_value):
+    if reference_value is None or isinstance(reference_value, bool):
+        return candidate_value is reference_value
+    if isinstance(reference_value, str):
+        return isinstance(candidate_value, str) and candidate_value == reference_value
+    if isinstance(reference_value, (int, float)) and not isinstance(
+        reference_value,
+        bool,
+    ):
+        return (
+            isinstance(candidate_value, (int, float))
+            and not isinstance(candidate_value, bool)
+            and float(candidate_value) == float(reference_value)
+        )
+    return False
+
+
+def read_csv_cell(path, row_number, column, label):
+    try:
+        with path.open("r", encoding="utf-8", newline="") as handle:
+            reader = csv.DictReader(handle)
+            if reader.fieldnames is None:
+                return False, None, f"{label} is missing a header row."
+            if column not in reader.fieldnames:
+                return False, None, f"{label} is missing CSV column {column}."
+            for current_row, row in enumerate(reader, start=1):
+                if current_row == row_number:
+                    return True, row.get(column, ""), None
+    except FileNotFoundError:
+        return False, None, f"{label} is missing."
+    except UnicodeDecodeError:
+        return False, None, f"{label} is not valid UTF-8 text."
+    except csv.Error as error:
+        return False, None, f"{label} is not valid CSV: {error}."
+    except OSError as error:
+        return False, None, f"{label} could not be read: {error}."
+    return False, None, f"{label} is missing CSV row {row_number}."
+
+
+def to_finite_number(value, label):
+    if isinstance(value, bool) or not isinstance(value, (int, float, str)):
+        return False, None, f"{label} is not numeric."
+    try:
+        number = float(value)
+    except ValueError:
+        return False, None, f"{label} is not numeric."
+    if not math.isfinite(number):
+        return False, None, f"{label} is not finite."
+    return True, number, None
+
+
+def extract_regex_capture(root, extractor, label):
+    path = resolve_member(root, extractor.get("path"), f"{label}.path")
+    ok, text, reason = read_text(path, f"{label}.path")
+    if not ok:
+        return False, None, reason
+    pattern = require_string(extractor.get("pattern"), f"{label}.pattern")
+    group = require_positive_int(extractor.get("group"), f"{label}.group")
+    try:
+        match = re.search(pattern, text)
+    except re.error as error:
+        fail_runtime(f"{label}.pattern is not a valid regex: {error}.")
+    if not match:
+        return False, None, f"{label}.pattern did not match."
+    try:
+        return True, match.group(group), None
+    except IndexError:
+        fail_runtime(f"{label}.group references a missing capture group.")
+
+
+def extract_scalar(root, extractor, label):
+    source = require_object(extractor, label)
+    extractor_kind = require_string(source.get("kind"), f"{label}.kind")
+    if extractor_kind == "json_path":
+        path = resolve_member(root, source.get("path"), f"{label}.path")
+        ok, data, reason = load_json_member(path, f"{label}.path")
+        if not ok:
+            return False, None, reason
+        tokens = parse_json_path(source.get("json_path"), f"{label}.json_path")
+        ok, value, reason = extract_json_path(data, tokens)
+        if not ok:
+            return False, None, reason
+        return True, require_json_scalar(value, label), None
+    if extractor_kind == "csv_cell":
+        path = resolve_member(root, source.get("path"), f"{label}.path")
+        row = require_positive_int(source.get("row"), f"{label}.row")
+        column = require_string(source.get("column"), f"{label}.column")
+        return read_csv_cell(path, row, column, f"{label}.path")
+    if extractor_kind == "text_regex_capture":
+        return extract_regex_capture(root, source, label)
+    fail_runtime(f"{label}.kind={extractor_kind} is unsupported.")
+
+
+def extract_numeric(root, extractor, label):
+    source = require_object(extractor, label)
+    extractor_kind = require_string(source.get("kind"), f"{label}.kind")
+    if extractor_kind == "json_path_number":
+        path = resolve_member(root, source.get("path"), f"{label}.path")
+        ok, data, reason = load_json_member(path, f"{label}.path")
+        if not ok:
+            return False, None, reason
+        tokens = parse_json_path(source.get("json_path"), f"{label}.json_path")
+        ok, value, reason = extract_json_path(data, tokens)
+        if not ok:
+            return False, None, reason
+        return to_finite_number(value, label)
+    if extractor_kind == "csv_cell_number":
+        path = resolve_member(root, source.get("path"), f"{label}.path")
+        row = require_positive_int(source.get("row"), f"{label}.row")
+        column = require_string(source.get("column"), f"{label}.column")
+        ok, value, reason = read_csv_cell(path, row, column, f"{label}.path")
+        if not ok:
+            return False, None, reason
+        return to_finite_number(value, label)
+    if extractor_kind == "text_regex_number":
+        ok, value, reason = extract_regex_capture(root, source, label)
+        if not ok:
+            return False, None, reason
+        return to_finite_number(value, label)
+    fail_runtime(f"{label}.kind={extractor_kind} is unsupported.")
+
+
+def require_reference_scalar(root, extractor, label):
+    ok, value, reason = extract_scalar(root, extractor, label)
+    if not ok:
+        fail_runtime(f"{label} could not be extracted from reference archive: {reason}")
+    return value
+
+
+def require_reference_number(root, extractor, label):
+    ok, value, reason = extract_numeric(root, extractor, label)
+    if not ok:
+        fail_runtime(f"{label} could not be extracted from reference archive: {reason}")
+    return value
+
+
+def resolve_tolerance(reference_root, tolerance, label):
+    policy = require_object(tolerance, label)
+    tolerance_kind = require_string(policy.get("kind"), f"{label}.kind")
+    if tolerance_kind == "absolute":
+        return require_nonnegative_number(policy.get("value"), f"{label}.value")
+    if tolerance_kind == "absolute_from_reference":
+        value = require_reference_number(
+            reference_root,
+            policy.get("source"),
+            f"{label}.source",
+        )
+        if value < 0:
+            fail_runtime(f"{label}.source must extract a nonnegative tolerance.")
+        return value
+    fail_runtime(f"{label}.kind={tolerance_kind} is unsupported.")
+
+
+def normalize_assertion(value, index):
+    assertion = require_object(
+        value,
+        f"reference_artifact_assertion.assertions[{index}]",
+    )
+    assertion_id = require_string(
+        assertion.get("id"),
+        f"reference_artifact_assertion.assertions[{index}].id",
+    )
+    assertion_kind = require_string(
+        assertion.get("kind"),
+        f"reference_artifact_assertion.assertions[{index}].kind",
+    )
+    weight = require_positive_number(
+        assertion.get("weight"),
+        f"reference_artifact_assertion.assertions[{index}].weight",
+    )
+    return assertion_id, assertion_kind, weight, assertion
+
+
+def evaluate_assertion(candidate_root, reference_root, assertion, index):
+    assertion_id, assertion_kind, weight, raw = normalize_assertion(assertion, index)
+    label = f"reference_artifact_assertion.assertions[{index}]"
+
+    if assertion_kind == "scalar_equals_reference":
+        ok, candidate_value, reason = extract_scalar(
+            candidate_root,
+            raw.get("candidate"),
+            f"{label}.candidate",
+        )
+        reference_value = require_reference_scalar(
+            reference_root,
+            raw.get("reference"),
+            f"{label}.reference",
+        )
+        passed = ok and json_scalar_equal(candidate_value, reference_value)
+        if ok and not passed:
+            reason = "candidate scalar did not equal reference scalar."
+    elif assertion_kind == "number_within_reference_tolerance":
+        ok, candidate_value, reason = extract_numeric(
+            candidate_root,
+            raw.get("candidate"),
+            f"{label}.candidate",
+        )
+        reference_value = require_reference_number(
+            reference_root,
+            raw.get("reference"),
+            f"{label}.reference",
+        )
+        tolerance = resolve_tolerance(
+            reference_root,
+            raw.get("tolerance"),
+            f"{label}.tolerance",
+        )
+        if ok:
+            difference = abs(candidate_value - reference_value)
+            passed = difference <= tolerance
+            if not passed:
+                reason = (
+                    f"numeric difference {difference} exceeded absolute "
+                    f"tolerance {tolerance}."
+                )
+        else:
+            passed = False
+    elif assertion_kind == "number_in_reference_interval":
+        ok, candidate_value, reason = extract_numeric(
+            candidate_root,
+            raw.get("candidate"),
+            f"{label}.candidate",
+        )
+        reference = require_object(raw.get("reference"), f"{label}.reference")
+        minimum = require_reference_number(
+            reference_root,
+            reference.get("min"),
+            f"{label}.reference.min",
+        )
+        maximum = require_reference_number(
+            reference_root,
+            reference.get("max"),
+            f"{label}.reference.max",
+        )
+        if minimum > maximum:
+            fail_runtime(f"{label}.reference.min must be less than or equal to max.")
+        passed = ok and minimum <= candidate_value <= maximum
+        if ok and not passed:
+            reason = "candidate number was outside the reference interval."
+    else:
+        fail_runtime(f"{label}.kind={assertion_kind} is unsupported.")
+
+    return {
+        "id": assertion_id,
+        "kind": assertion_kind,
+        "weight": weight,
+        "passed": bool(passed),
+        "score": 1.0 if passed else 0.0,
+        **({} if reason is None else {"reason": reason}),
+    }
+
+
+def normalize_assertions(params):
+    assertions = require_list(
+        params.get("assertions"),
+        "reference_artifact_assertion.assertions",
+    )
+    if not assertions:
+        fail_runtime("reference_artifact_assertion.assertions must be non-empty.")
+    seen = set()
+    for index, assertion in enumerate(assertions):
+        assertion_id, _, _, _ = normalize_assertion(assertion, index)
+        if assertion_id in seen:
+            fail_runtime(
+                f"reference_artifact_assertion.assertions id {assertion_id} is duplicated."
+            )
+        seen.add(assertion_id)
+    return assertions
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    config = require_object(load_compiled_config(config_path), "compiled_config")
+    candidate_role = require_string(
+        config.get("candidate_role"),
+        "compiled_config.candidate_role",
+    )
+    reference_role = require_string(
+        config.get("reference_role"),
+        "compiled_config.reference_role",
+    )
+    config_score_key = require_string(
+        config.get("final_score_key"),
+        "compiled_config.final_score_key",
+    )
+    final_score_key = require_string(
+        runtime_context.get("final_score_key"),
+        "runtime_context.final_score_key",
+    )
+    if config_score_key != final_score_key:
+        fail_runtime("compiled_config.final_score_key must match runtime context.")
+    params = require_object(
+        config.get("reference_artifact_assertion"),
+        "compiled_config.reference_artifact_assertion",
+    )
+    assertions = normalize_assertions(params)
+    candidate_path = resolve_submission_artifact(runtime_context, candidate_role)
+    reference_path = resolve_evaluation_artifact(runtime_context, reference_role)
+
+    with tempfile.TemporaryDirectory(prefix="agora-reference-artifact-") as temp_dir:
+        candidate_root = Path(temp_dir) / "candidate"
+        reference_root = Path(temp_dir) / "reference"
+        safe_extract_zip(
+            candidate_path,
+            candidate_root,
+            label=f"submission artifact {candidate_role}",
+            invalid_handler=reject_submission,
+        )
+        safe_extract_zip(
+            reference_path,
+            reference_root,
+            label=f"evaluation artifact {reference_role}",
+        )
+        results = [
+            evaluate_assertion(candidate_root, reference_root, assertion, index)
+            for index, assertion in enumerate(assertions)
+        ]
+
+    total_weight = sum(result["weight"] for result in results)
+    if not math.isfinite(total_weight) or total_weight <= 0:
+        fail_runtime(
+            "reference_artifact_assertion.assertions must declare finite positive total weight."
+        )
+    earned_weight = sum(result["weight"] for result in results if result["passed"])
+    score = earned_weight / total_weight
+    if not math.isfinite(score):
+        fail_runtime("reference_artifact_assertion score must be finite.")
+    passed_count = sum(1 for result in results if result["passed"])
+    write_score(
+        score=score,
+        details={
+            final_score_key: score,
+            "passed_assertions": passed_count,
+            "total_assertions": len(results),
+            "earned_weight": earned_weight,
+            "total_weight": total_weight,
+            "assertion_results": results,
+        },
+    )
+
+
+if __name__ == "__main__":
+    main()