@moleculeagora/cli 0.1.0

package/dist/python-v1/panel-ranking-metric.py

@@ -0,0 +1,622 @@
+import csv
+import math
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    reject_submission,
+    resolve_evaluation_artifact,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    write_score,
+)
+
+SUPPORTED_METRICS = ("grouped_ndcg", "top_1_accuracy", "pairwise_accuracy")
+SUPPORTED_METRIC_SET = set(SUPPORTED_METRICS)
+SUPPORTED_AGGREGATIONS = {"macro_mean", "weighted_mean"}
+
+
+def format_metric_list(metrics):
+    ordered = list(metrics)
+    if len(ordered) == 1:
+        return ordered[0]
+    if len(ordered) == 2:
+        return f"{ordered[0]} or {ordered[1]}"
+    return f"{', '.join(ordered[:-1])}, or {ordered[-1]}"
+
+
+def require_string(value, label):
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_metric(config):
+    metric = require_string(config.get("metric"), "compiled_config.metric").lower()
+    if metric not in SUPPORTED_METRIC_SET:
+        fail_runtime(
+            f"compiled_config.metric must be one of {format_metric_list(SUPPORTED_METRICS)}."
+        )
+    return metric
+
+
+def require_aggregation(config):
+    aggregation = require_string(
+        config.get("aggregation"),
+        "compiled_config.aggregation",
+    )
+    if aggregation not in SUPPORTED_AGGREGATIONS:
+        fail_runtime(
+            "compiled_config.aggregation must be macro_mean or weighted_mean for panel_ranking_metric."
+        )
+    return aggregation
+
+
+def require_metric_params(config, metric, panel_field, aggregation):
+    metric_params = config.get("metric_params")
+    if not isinstance(metric_params, dict):
+        fail_runtime("compiled_config.metric_params must be an object.")
+
+    param_metric = require_string(
+        metric_params.get("metric"),
+        "compiled_config.metric_params.metric",
+    ).lower()
+    if param_metric != metric:
+        fail_runtime(
+            f"compiled_config.metric_params.metric must equal compiled_config.metric {metric!r}."
+        )
+    param_panel_field = require_string(
+        metric_params.get("panel_field"),
+        "compiled_config.metric_params.panel_field",
+    )
+    if param_panel_field != panel_field:
+        fail_runtime(
+            "compiled_config.metric_params.panel_field must match compiled_config.panel_field."
+        )
+    param_aggregation = require_string(
+        metric_params.get("aggregation"),
+        "compiled_config.metric_params.aggregation",
+    )
+    if param_aggregation != aggregation:
+        fail_runtime(
+            "compiled_config.metric_params.aggregation must match compiled_config.aggregation."
+        )
+    return metric_params
+
+
+def require_policy(policies, key, allowed):
+    value = require_string(policies.get(key), f"policies.{key}")
+    if value not in allowed:
+        fail_runtime(
+            f"policies.{key} must be one of {', '.join(sorted(allowed))}."
+        )
+    return value
+
+
+def find_slot(runtime_context, lane, role):
+    slot_key = f"{lane}_slots"
+    slots = runtime_context.get(slot_key)
+    if not isinstance(slots, list):
+        fail_runtime(f"Runtime context is missing {slot_key}.")
+    for slot in slots:
+        if isinstance(slot, dict) and slot.get("role") == role:
+            return slot
+    fail_runtime(f"Runtime context is missing {lane} slot for role {role}.")
+
+
+def require_csv_slot_columns(runtime_context, lane, role):
+    slot = find_slot(runtime_context, lane, role)
+    validator = slot.get("validator")
+    if not isinstance(validator, dict) or validator.get("kind") != "csv_columns":
+        fail_runtime(
+            f"{lane} role {role} must use validator.kind=csv_columns for panel_ranking_metric."
+        )
+    record_key = require_string(
+        validator.get("record_key"),
+        f"{lane}.{role}.validator.record_key",
+    )
+    value_field = require_string(
+        validator.get("value_field"),
+        f"{lane}.{role}.validator.value_field",
+    )
+    return record_key, value_field
+
+
+def read_csv_rows(path, label, *, invalid_handler):
+    try:
+        with path.open("r", encoding="utf-8", newline="") as handle:
+            reader = csv.DictReader(handle)
+            fieldnames = reader.fieldnames
+            if not fieldnames:
+                invalid_handler(f"{label} must include a CSV header row.")
+            normalized_fieldnames = []
+            for fieldname in fieldnames:
+                if not isinstance(fieldname, str) or not fieldname.strip():
+                    invalid_handler(f"{label} contains an empty CSV column name.")
+                normalized_fieldnames.append(fieldname.strip())
+            rows = list(reader)
+    except FileNotFoundError:
+        invalid_handler(f"Missing {label} at {path}.")
+    except OSError as error:
+        invalid_handler(f"Unable to read {label}: {error}.")
+    return normalized_fieldnames, rows
+
+
+def parse_reference_relevance(raw_value, label):
+    text = raw_value.strip() if isinstance(raw_value, str) else ""
+    if not text:
+        fail_runtime(f"{label} is blank.")
+    try:
+        value = float(text)
+    except ValueError:
+        fail_runtime(f"{label} must be numeric, received {text!r}.")
+    if not math.isfinite(value):
+        fail_runtime(f"{label} must be finite.")
+    if value < 0:
+        fail_runtime(f"{label} must be non-negative for panel_ranking_metric.")
+    return value
+
+
+def parse_submission_score(raw_value, label, invalid_value_policy):
+    text = raw_value.strip() if isinstance(raw_value, str) else ""
+    if not text:
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} is blank.")
+        return None
+    try:
+        value = float(text)
+    except ValueError:
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} must be numeric, received {text!r}.")
+        return None
+    if not math.isfinite(value):
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} must be finite.")
+        return None
+    return value
+
+
+def load_reference_panel_records(path, role, record_key, value_field, panel_field):
+    fieldnames, rows = read_csv_rows(
+        path,
+        f"evaluation artifact {role}",
+        invalid_handler=fail_runtime,
+    )
+    if record_key not in fieldnames:
+        fail_runtime(
+            f"evaluation artifact {role} is missing record key column {record_key}."
+        )
+    if value_field not in fieldnames:
+        fail_runtime(
+            f"evaluation artifact {role} is missing value column {value_field}."
+        )
+    if panel_field not in fieldnames:
+        fail_runtime(
+            f"evaluation artifact {role} is missing panel field column {panel_field}."
+        )
+
+    records = {}
+    for row_index, row in enumerate(rows, start=2):
+        raw_key = row.get(record_key)
+        key = raw_key.strip() if isinstance(raw_key, str) else ""
+        if not key:
+            fail_runtime(
+                f"evaluation artifact {role} row {row_index} is missing {record_key}."
+            )
+        if key in records:
+            fail_runtime(
+                f"evaluation artifact {role} contains duplicate record id {key!r}."
+            )
+
+        raw_panel = row.get(panel_field)
+        panel_value = raw_panel.strip() if isinstance(raw_panel, str) else ""
+        if not panel_value:
+            fail_runtime(
+                f"evaluation artifact {role} row {row_index} is missing panel field column {panel_field}."
+            )
+
+        records[key] = {
+            "relevance": parse_reference_relevance(
+                row.get(value_field),
+                f"evaluation artifact {role} row {row_index} column {value_field}",
+            ),
+            "panel": panel_value,
+        }
+
+    if not records:
+        fail_runtime(
+            f"evaluation artifact {role} must contain at least one ranked row."
+        )
+    return records
+
+
+def load_submission_scores(
+    path,
+    role,
+    record_key,
+    value_field,
+    duplicate_id_policy,
+    invalid_value_policy,
+):
+    fieldnames, rows = read_csv_rows(
+        path,
+        f"submission artifact {role}",
+        invalid_handler=reject_submission,
+    )
+    if record_key not in fieldnames:
+        reject_submission(
+            f"submission artifact {role} is missing record key column {record_key}."
+        )
+    if value_field not in fieldnames:
+        reject_submission(
+            f"submission artifact {role} is missing value column {value_field}."
+        )
+    values = {}
+    for row_index, row in enumerate(rows, start=2):
+        raw_key = row.get(record_key)
+        key = raw_key.strip() if isinstance(raw_key, str) else ""
+        if not key:
+            if invalid_value_policy == "reject":
+                reject_submission(
+                    f"submission artifact {role} row {row_index} is missing {record_key}."
+                )
+            continue
+        if key in values:
+            if duplicate_id_policy == "reject":
+                reject_submission(
+                    f"submission artifact {role} contains duplicate record id {key!r}."
+                )
+            continue
+        parsed_value = parse_submission_score(
+            row.get(value_field),
+            f"submission artifact {role} row {row_index} column {value_field}",
+            invalid_value_policy,
+        )
+        if parsed_value is None:
+            continue
+        values[key] = parsed_value
+    return values
+
+
+def discount(rank_index):
+    return 1.0 / math.log2(rank_index + 2.0)
+
+
+def dcg(labels):
+    return sum(
+        ((2.0 ** label) - 1.0) * discount(index)
+        for index, label in enumerate(labels)
+    )
+
+
+def rank_submission_ids(submission_by_id):
+    return sorted(
+        submission_by_id.keys(),
+        key=lambda record_id: (-submission_by_id[record_id], record_id),
+    )
+
+
+def has_comparable_pairs(relevances):
+    seen = set()
+    for relevance in relevances:
+        seen.add(relevance)
+        if len(seen) > 1:
+            return True
+    return False
+
+
+def validate_reference_panel_preconditions(metric, panel_label, reference_relevances):
+    if metric == "grouped_ndcg":
+        if len(reference_relevances) < 2:
+            fail_runtime(
+                f"panel_ranking_metric metric grouped_ndcg requires at least two reference rows in panel {panel_label}."
+            )
+        if max(reference_relevances) <= 0:
+            fail_runtime(
+                f"panel_ranking_metric metric grouped_ndcg requires at least one positive reference relevance in panel {panel_label}."
+            )
+        return
+
+    if metric == "top_1_accuracy":
+        if max(reference_relevances) <= 0:
+            fail_runtime(
+                f"panel_ranking_metric metric top_1_accuracy requires a positive winning relevance in panel {panel_label}."
+            )
+        return
+
+    if not has_comparable_pairs(reference_relevances):
+        fail_runtime(
+            f"panel_ranking_metric metric pairwise_accuracy requires at least one unequal reference relevance pair in panel {panel_label}."
+        )
+
+
+def validate_scored_panel_preconditions(metric, panel_label, scored_relevances):
+    if metric == "grouped_ndcg":
+        if len(scored_relevances) < 2:
+            reject_submission(
+                f"panel_ranking_metric metric grouped_ndcg requires at least two scored rows in panel {panel_label}."
+            )
+        if max(scored_relevances) <= 0:
+            reject_submission(
+                f"panel_ranking_metric metric grouped_ndcg requires at least one positive relevance among scored rows in panel {panel_label}."
+            )
+        return
+
+    if metric == "pairwise_accuracy" and not has_comparable_pairs(scored_relevances):
+        reject_submission(
+            f"panel_ranking_metric metric pairwise_accuracy requires at least one comparable scored row pair in panel {panel_label}."
+        )
+
+
+def compute_grouped_ndcg(reference_by_id, submission_by_id, scored_ids):
+    scored_submission = {
+        record_id: submission_by_id[record_id] for record_id in scored_ids
+    }
+    ranked_ids = rank_submission_ids(scored_submission)
+    ranked_labels = [
+        reference_by_id[record_id]["relevance"] for record_id in ranked_ids
+    ]
+    ideal_labels = sorted(ranked_labels, reverse=True)
+    ideal_dcg = dcg(ideal_labels)
+    if ideal_dcg == 0.0:
+        reject_submission(
+            "panel_ranking_metric metric grouped_ndcg has no positive scored relevance after applying runtime policies."
+        )
+    return dcg(ranked_labels) / ideal_dcg
+
+
+def compute_top_1_accuracy(reference_by_id, submission_by_id, panel_record_ids, scored_ids):
+    max_relevance = max(
+        reference_by_id[record_id]["relevance"] for record_id in panel_record_ids
+    )
+    winners = {
+        record_id
+        for record_id in panel_record_ids
+        if reference_by_id[record_id]["relevance"] == max_relevance
+    }
+    scored_submission = {
+        record_id: submission_by_id[record_id] for record_id in scored_ids
+    }
+    top_record_id = rank_submission_ids(scored_submission)[0]
+    return 1.0 if top_record_id in winners else 0.0
+
+
+def compute_pairwise_accuracy(reference_by_id, submission_by_id, scored_ids):
+    correct = 0.0
+    total = 0
+    for left_index, left_id in enumerate(scored_ids):
+        for right_id in scored_ids[left_index + 1 :]:
+            left_relevance = reference_by_id[left_id]["relevance"]
+            right_relevance = reference_by_id[right_id]["relevance"]
+            if left_relevance == right_relevance:
+                continue
+
+            total += 1
+            left_score = submission_by_id[left_id]
+            right_score = submission_by_id[right_id]
+            if left_score == right_score:
+                correct += 0.5
+                continue
+            if left_relevance > right_relevance and left_score > right_score:
+                correct += 1.0
+            elif right_relevance > left_relevance and right_score > left_score:
+                correct += 1.0
+
+    if total == 0:
+        reject_submission(
+            "panel_ranking_metric metric pairwise_accuracy has no comparable scored row pairs after applying runtime policies."
+        )
+    return correct / total
+
+
+def compute_panel_metric(metric, reference_by_id, submission_by_id, panel_record_ids, scored_ids):
+    if metric == "grouped_ndcg":
+        return compute_grouped_ndcg(reference_by_id, submission_by_id, scored_ids)
+    if metric == "top_1_accuracy":
+        return compute_top_1_accuracy(
+            reference_by_id,
+            submission_by_id,
+            panel_record_ids,
+            scored_ids,
+        )
+    return compute_pairwise_accuracy(reference_by_id, submission_by_id, scored_ids)
+
+
+def format_panel_label(panel_field, panel_value):
+    return f"{panel_field}={panel_value!r}"
+
+
+def aggregate_panel_scores(panel_scores, aggregation):
+    if not panel_scores:
+        reject_submission(
+            "Submission produced no scoreable panels after applying runtime policies."
+        )
+    if aggregation == "macro_mean":
+        return sum(panel_score["score"] for panel_score in panel_scores) / len(
+            panel_scores
+        )
+
+    total_weight = sum(panel_score["weight"] for panel_score in panel_scores)
+    if total_weight <= 0:
+        reject_submission(
+            "Submission produced no weighted panel rows after applying runtime policies."
+        )
+    return (
+        sum(
+            panel_score["score"] * panel_score["weight"]
+            for panel_score in panel_scores
+        )
+        / total_weight
+    )
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    try:
+        config = load_json_file(config_path, label="compiled_config")
+    except RuntimeError as error:
+        fail_runtime(str(error))
+
+    metric = require_metric(config)
+    panel_field = require_string(
+        config.get("panel_field"),
+        "compiled_config.panel_field",
+    )
+    aggregation = require_aggregation(config)
+    require_metric_params(config, metric, panel_field, aggregation)
+    evaluation_role = require_string(
+        config.get("evaluation_role"),
+        "compiled_config.evaluation_role",
+    )
+    submission_role = require_string(
+        config.get("submission_role"),
+        "compiled_config.submission_role",
+    )
+    objective = require_string(
+        runtime_context.get("objective"),
+        "runtime_context.objective",
+    )
+    if objective != "maximize":
+        fail_runtime(f"panel_ranking_metric metric {metric} requires objective=maximize.")
+    final_score_key = require_string(
+        runtime_context.get("final_score_key"),
+        "runtime_context.final_score_key",
+    )
+    policies = runtime_context.get("policies")
+    if not isinstance(policies, dict):
+        fail_runtime("Runtime context is missing execution policies.")
+    coverage_policy = require_policy(
+        policies,
+        "coverage_policy",
+        {"reject", "ignore", "penalize"},
+    )
+    duplicate_id_policy = require_policy(
+        policies,
+        "duplicate_id_policy",
+        {"reject", "ignore"},
+    )
+    invalid_value_policy = require_policy(
+        policies,
+        "invalid_value_policy",
+        {"reject", "ignore"},
+    )
+    evaluation_record_key, evaluation_value_field = require_csv_slot_columns(
+        runtime_context,
+        "evaluation",
+        evaluation_role,
+    )
+    submission_record_key, submission_value_field = require_csv_slot_columns(
+        runtime_context,
+        "submission",
+        submission_role,
+    )
+    evaluation_path = resolve_evaluation_artifact(runtime_context, evaluation_role)
+    submission_path = resolve_submission_artifact(runtime_context, submission_role)
+
+    reference_by_id = load_reference_panel_records(
+        evaluation_path,
+        evaluation_role,
+        evaluation_record_key,
+        evaluation_value_field,
+        panel_field,
+    )
+    panels = {}
+    for record_id, record in reference_by_id.items():
+        panels.setdefault(record["panel"], []).append(record_id)
+    for panel_value in sorted(panels):
+        panel_label = format_panel_label(panel_field, panel_value)
+        panel_relevances = [
+            reference_by_id[record_id]["relevance"] for record_id in panels[panel_value]
+        ]
+        validate_reference_panel_preconditions(metric, panel_label, panel_relevances)
+
+    submission_by_id = load_submission_scores(
+        submission_path,
+        submission_role,
+        submission_record_key,
+        submission_value_field,
+        duplicate_id_policy,
+        invalid_value_policy,
+    )
+    missing_ids = [
+        record_id
+        for record_id in reference_by_id
+        if record_id not in submission_by_id
+    ]
+    if missing_ids and coverage_policy == "reject":
+        reject_submission(
+            f"Submission is missing rankings for {len(missing_ids)} required rows; first missing id is {missing_ids[0]!r}."
+        )
+
+    panel_scores = []
+    panel_details = []
+    total_scored = 0
+
+    for panel_value in sorted(panels):
+        panel_record_ids = panels[panel_value]
+        panel_label = format_panel_label(panel_field, panel_value)
+        panel_missing_ids = [
+            record_id for record_id in panel_record_ids if record_id not in submission_by_id
+        ]
+        scored_ids = [
+            record_id for record_id in panel_record_ids if record_id in submission_by_id
+        ]
+        if not scored_ids:
+            reject_submission(
+                f"Submission produced no ranked rows for panel {panel_label} after applying runtime policies."
+            )
+
+        scored_relevances = [
+            reference_by_id[record_id]["relevance"] for record_id in scored_ids
+        ]
+        validate_scored_panel_preconditions(metric, panel_label, scored_relevances)
+        raw_metric = compute_panel_metric(
+            metric,
+            reference_by_id,
+            submission_by_id,
+            panel_record_ids,
+            scored_ids,
+        )
+        normalized_score = max(0.0, min(raw_metric, 1.0))
+        if coverage_policy == "penalize":
+            normalized_score *= len(scored_ids) / len(panel_record_ids)
+
+        aggregation_weight = len(scored_ids)
+        total_scored += aggregation_weight
+        panel_scores.append({"score": normalized_score, "weight": aggregation_weight})
+        panel_details.append(
+            {
+                "panel": {panel_field: panel_value},
+                "reference_row_count": len(panel_record_ids),
+                "rows_scored": len(scored_ids),
+                "missing_count": len(panel_missing_ids),
+                "selected_metric_value": raw_metric,
+                "normalized_score": normalized_score,
+                "aggregation_weight": aggregation_weight,
+            }
+        )
+
+    final_score = aggregate_panel_scores(panel_scores, aggregation)
+    write_score(
+        score=final_score,
+        details={
+            final_score_key: final_score,
+            "selected_metric": metric,
+            "aggregation": aggregation,
+            "panel_field": panel_field,
+            "panel_count": len(panel_details),
+            "rows_scored": total_scored,
+            "missing_count": len(missing_ids),
+            "panels": panel_details,
+        },
+    )
+
+
+if __name__ == "__main__":
+    main()