@moleculeagora/cli 0.1.0

package/dist/python-v1/table-metric.py

@@ -0,0 +1,1291 @@
+import csv
+import math
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    reject_submission,
+    resolve_evaluation_artifact,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    write_score,
+)
+
+SUPPORTED_METRICS = (
+    "rmse",
+    "mae",
+    "r2",
+    "pearson",
+    "spearman",
+    "accuracy",
+    "f1",
+    "auroc",
+    "brier",
+    "log_loss",
+    "auprc",
+    "ece",
+    "z_rmse",
+    "within_noise_fraction",
+)
+SUPPORTED_METRIC_SET = set(SUPPORTED_METRICS)
+LOG_LOSS_EPSILON = 1e-15
+PROBABILITY_METRICS = {"brier", "log_loss", "auprc", "ece"}
+CALIBRATED_METRICS = {"z_rmse", "within_noise_fraction"}
+MINIMIZE_METRICS = {"rmse", "mae", "brier", "log_loss", "ece", "z_rmse"}
+CORRELATION_METRICS = {"pearson", "spearman"}
+BINARY_LABEL_METRICS = {"accuracy", "f1", "auroc"} | PROBABILITY_METRICS
+UNIT_INTERVAL_METRICS = {
+    "accuracy",
+    "f1",
+    "auroc",
+    "auprc",
+    "within_noise_fraction",
+}
+MAXIMIZE_METRICS = {"r2"} | CORRELATION_METRICS | UNIT_INTERVAL_METRICS
+
+
+def format_metric_list(metrics):
+    ordered = list(metrics)
+    if len(ordered) == 1:
+        return ordered[0]
+    if len(ordered) == 2:
+        return f"{ordered[0]} or {ordered[1]}"
+    return f"{', '.join(ordered[:-1])}, or {ordered[-1]}"
+
+
+def require_string(value, label):
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_metric(config):
+    metric = require_string(config.get("metric"), "compiled_config.metric").lower()
+    if metric not in SUPPORTED_METRIC_SET:
+        fail_runtime(
+            f"compiled_config.metric must be one of {format_metric_list(SUPPORTED_METRICS)}."
+        )
+    return metric
+
+
+def require_metric_params(config, metric):
+    metric_params = config.get("metric_params")
+    if not isinstance(metric_params, dict):
+        fail_runtime("compiled_config.metric_params must be an object.")
+
+    param_metric = require_string(
+        metric_params.get("metric"),
+        "compiled_config.metric_params.metric",
+    ).lower()
+    if param_metric != metric:
+        fail_runtime(
+            f"compiled_config.metric_params.metric must equal compiled_config.metric {metric!r}."
+        )
+
+    if metric == "ece":
+        bin_count = metric_params.get("bin_count")
+        if isinstance(bin_count, bool) or not isinstance(bin_count, int):
+            fail_runtime(
+                "compiled_config.metric_params.bin_count must be an integer from 2 to 100 for ece."
+            )
+        if bin_count < 2 or bin_count > 100:
+            fail_runtime(
+                "compiled_config.metric_params.bin_count must be an integer from 2 to 100 for ece."
+            )
+        binning = require_string(
+            metric_params.get("binning"),
+            "compiled_config.metric_params.binning",
+        )
+        if binning != "uniform":
+            fail_runtime(
+                "compiled_config.metric_params.binning must be uniform for ece."
+            )
+    if metric == "z_rmse":
+        require_string(
+            metric_params.get("baseline_mean_field"),
+            "compiled_config.metric_params.baseline_mean_field",
+        )
+        require_string(
+            metric_params.get("baseline_sd_field"),
+            "compiled_config.metric_params.baseline_sd_field",
+        )
+    if metric == "within_noise_fraction":
+        require_string(
+            metric_params.get("noise_floor_field"),
+            "compiled_config.metric_params.noise_floor_field",
+        )
+    return metric_params
+
+
+def require_config_field_matches_metric_params(config, metric_params, key):
+    value = require_string(config.get(key), f"compiled_config.{key}")
+    metric_param_value = require_string(
+        metric_params.get(key),
+        f"compiled_config.metric_params.{key}",
+    )
+    if metric_param_value != value:
+        fail_runtime(
+            f"compiled_config.{key} must match compiled_config.metric_params.{key}."
+        )
+    return value
+
+
+def require_grouping_config(config):
+    raw_group_by = config.get("group_by")
+    if raw_group_by is None:
+        if "aggregation" in config:
+            fail_runtime("compiled_config.aggregation requires compiled_config.group_by.")
+        return [], None
+    if not isinstance(raw_group_by, list) or not (1 <= len(raw_group_by) <= 3):
+        fail_runtime("compiled_config.group_by must list one to three CSV columns.")
+
+    group_by = []
+    seen = set()
+    for index, raw_column in enumerate(raw_group_by):
+        column = require_string(
+            raw_column,
+            f"compiled_config.group_by[{index}]",
+        )
+        if column in seen:
+            fail_runtime(
+                f"compiled_config.group_by repeats column {column!r}; grouped metrics require canonical unique columns."
+            )
+        seen.add(column)
+        group_by.append(column)
+
+    aggregation = require_string(
+        config.get("aggregation"),
+        "compiled_config.aggregation",
+    )
+    if aggregation not in {"macro_mean", "weighted_mean"}:
+        fail_runtime(
+            "compiled_config.aggregation must be macro_mean or weighted_mean for grouped table metrics."
+        )
+    return group_by, aggregation
+
+
+def require_policy(policies, key, allowed):
+    value = require_string(policies.get(key), f"policies.{key}")
+    if value not in allowed:
+        fail_runtime(
+            f"policies.{key} must be one of {', '.join(sorted(allowed))}."
+        )
+    return value
+
+
+def find_slot(runtime_context, lane, role):
+    slot_key = f"{lane}_slots"
+    slots = runtime_context.get(slot_key)
+    if not isinstance(slots, list):
+        fail_runtime(f"Runtime context is missing {slot_key}.")
+    for slot in slots:
+        if isinstance(slot, dict) and slot.get("role") == role:
+            return slot
+    fail_runtime(f"Runtime context is missing {lane} slot for role {role}.")
+
+
+def require_csv_slot_columns(runtime_context, lane, role):
+    slot = find_slot(runtime_context, lane, role)
+    validator = slot.get("validator")
+    if not isinstance(validator, dict) or validator.get("kind") != "csv_columns":
+        fail_runtime(
+            f"{lane} role {role} must use validator.kind=csv_columns for table_metric."
+        )
+    record_key = require_string(
+        validator.get("record_key"),
+        f"{lane}.{role}.validator.record_key",
+    )
+    value_field = require_string(
+        validator.get("value_field"),
+        f"{lane}.{role}.validator.value_field",
+    )
+    return record_key, value_field
+
+
+def read_csv_rows(path, label, *, invalid_handler):
+    try:
+        with path.open("r", encoding="utf-8", newline="") as handle:
+            reader = csv.DictReader(handle)
+            fieldnames = reader.fieldnames
+            if not fieldnames:
+                invalid_handler(f"{label} must include a CSV header row.")
+            normalized_fieldnames = []
+            for fieldname in fieldnames:
+                if not isinstance(fieldname, str) or not fieldname.strip():
+                    invalid_handler(f"{label} contains an empty CSV column name.")
+                normalized_fieldnames.append(fieldname.strip())
+            rows = list(reader)
+    except FileNotFoundError:
+        invalid_handler(f"Missing {label} at {path}.")
+    except OSError as error:
+        invalid_handler(f"Unable to read {label}: {error}.")
+    return normalized_fieldnames, rows
+
+
+def parse_reference_value(raw_value, label):
+    text = raw_value.strip() if isinstance(raw_value, str) else ""
+    if not text:
+        fail_runtime(f"{label} is blank.")
+    try:
+        value = float(text)
+    except ValueError:
+        fail_runtime(f"{label} must be numeric, received {text!r}.")
+    if not math.isfinite(value):
+        fail_runtime(f"{label} must be finite.")
+    return value
+
+
+def parse_submission_value(raw_value, label, invalid_value_policy, metric):
+    text = raw_value.strip() if isinstance(raw_value, str) else ""
+    if not text:
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} is blank.")
+        return None
+    try:
+        value = float(text)
+    except ValueError:
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} must be numeric, received {text!r}.")
+        return None
+    if not math.isfinite(value):
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} must be finite.")
+        return None
+    if metric in PROBABILITY_METRICS and not (0.0 <= value <= 1.0):
+        if invalid_value_policy == "reject":
+            reject_submission(
+                f"{label} must be a probability in [0, 1] for table_metric metric {metric}; received {value}."
+            )
+        return None
+    return value
+
+
+def load_reference_values(path, role, record_key, value_field):
+    fieldnames, rows = read_csv_rows(
+        path,
+        f"evaluation artifact {role}",
+        invalid_handler=fail_runtime,
+    )
+    if record_key not in fieldnames:
+        fail_runtime(
+            f"evaluation artifact {role} is missing record key column {record_key}."
+        )
+    if value_field not in fieldnames:
+        fail_runtime(
+            f"evaluation artifact {role} is missing value column {value_field}."
+        )
+    values = {}
+    for row_index, row in enumerate(rows, start=2):
+        raw_key = row.get(record_key)
+        key = raw_key.strip() if isinstance(raw_key, str) else ""
+        if not key:
+            fail_runtime(
+                f"evaluation artifact {role} row {row_index} is missing {record_key}."
+            )
+        if key in values:
+            fail_runtime(
+                f"evaluation artifact {role} contains duplicate record id {key!r}."
+            )
+        values[key] = parse_reference_value(
+            row.get(value_field),
+            f"evaluation artifact {role} row {row_index} column {value_field}",
+        )
+    if not values:
+        fail_runtime(
+            f"evaluation artifact {role} must contain at least one scored row."
+        )
+    return values
+
+
+def load_reference_group_records(path, role, record_key, value_field, group_by):
+    fieldnames, rows = read_csv_rows(
+        path,
+        f"evaluation artifact {role}",
+        invalid_handler=fail_runtime,
+    )
+    if record_key not in fieldnames:
+        fail_runtime(
+            f"evaluation artifact {role} is missing record key column {record_key}."
+        )
+    if value_field not in fieldnames:
+        fail_runtime(
+            f"evaluation artifact {role} is missing value column {value_field}."
+        )
+    for group_column in group_by:
+        if group_column not in fieldnames:
+            fail_runtime(
+                f"evaluation artifact {role} is missing group_by column {group_column}."
+            )
+
+    records = {}
+    for row_index, row in enumerate(rows, start=2):
+        raw_key = row.get(record_key)
+        key = raw_key.strip() if isinstance(raw_key, str) else ""
+        if not key:
+            fail_runtime(
+                f"evaluation artifact {role} row {row_index} is missing {record_key}."
+            )
+        if key in records:
+            fail_runtime(
+                f"evaluation artifact {role} contains duplicate record id {key!r}."
+            )
+
+        group_values = []
+        for group_column in group_by:
+            raw_group = row.get(group_column)
+            group_value = raw_group.strip() if isinstance(raw_group, str) else ""
+            if not group_value:
+                fail_runtime(
+                    f"evaluation artifact {role} row {row_index} is missing group_by column {group_column}."
+                )
+            group_values.append(group_value)
+
+        records[key] = {
+            "value": parse_reference_value(
+                row.get(value_field),
+                f"evaluation artifact {role} row {row_index} column {value_field}",
+            ),
+            "group_key": tuple(group_values),
+        }
+
+    if not records:
+        fail_runtime(
+            f"evaluation artifact {role} must contain at least one scored row."
+        )
+    return records
+
+
+def parse_positive_reference_value(raw_value, label):
+    value = parse_reference_value(raw_value, label)
+    if value <= 0.0:
+        fail_runtime(f"{label} must be greater than 0.")
+    return value
+
+
+def parse_nonnegative_reference_value(raw_value, label):
+    value = parse_reference_value(raw_value, label)
+    if value < 0.0:
+        fail_runtime(f"{label} must be greater than or equal to 0.")
+    return value
+
+
+def load_calibrated_reference_records(
+    path,
+    role,
+    record_key,
+    value_field,
+    metric,
+    metric_params,
+):
+    fieldnames, rows = read_csv_rows(
+        path,
+        f"evaluation artifact {role}",
+        invalid_handler=fail_runtime,
+    )
+    if record_key not in fieldnames:
+        fail_runtime(
+            f"evaluation artifact {role} is missing record key column {record_key}."
+        )
+    if value_field not in fieldnames:
+        fail_runtime(
+            f"evaluation artifact {role} is missing value column {value_field}."
+        )
+
+    required_calibration_fields = []
+    if metric == "z_rmse":
+        required_calibration_fields = [
+            metric_params["baseline_mean_field"],
+            metric_params["baseline_sd_field"],
+        ]
+    elif metric == "within_noise_fraction":
+        required_calibration_fields = [metric_params["noise_floor_field"]]
+
+    for calibration_field in required_calibration_fields:
+        if calibration_field not in fieldnames:
+            fail_runtime(
+                f"evaluation artifact {role} is missing calibration column {calibration_field}."
+            )
+
+    records = {}
+    for row_index, row in enumerate(rows, start=2):
+        raw_key = row.get(record_key)
+        key = raw_key.strip() if isinstance(raw_key, str) else ""
+        if not key:
+            fail_runtime(
+                f"evaluation artifact {role} row {row_index} is missing {record_key}."
+            )
+        if key in records:
+            fail_runtime(
+                f"evaluation artifact {role} contains duplicate record id {key!r}."
+            )
+
+        record = {
+            "value": parse_reference_value(
+                row.get(value_field),
+                f"evaluation artifact {role} row {row_index} column {value_field}",
+            ),
+        }
+        if metric == "z_rmse":
+            baseline_mean_field = metric_params["baseline_mean_field"]
+            baseline_sd_field = metric_params["baseline_sd_field"]
+            record["baseline_mean"] = parse_reference_value(
+                row.get(baseline_mean_field),
+                f"evaluation artifact {role} row {row_index} column {baseline_mean_field}",
+            )
+            record["baseline_sd"] = parse_positive_reference_value(
+                row.get(baseline_sd_field),
+                f"evaluation artifact {role} row {row_index} column {baseline_sd_field}",
+            )
+        elif metric == "within_noise_fraction":
+            noise_floor_field = metric_params["noise_floor_field"]
+            record["noise_floor"] = parse_nonnegative_reference_value(
+                row.get(noise_floor_field),
+                f"evaluation artifact {role} row {row_index} column {noise_floor_field}",
+            )
+        records[key] = record
+
+    if not records:
+        fail_runtime(
+            f"evaluation artifact {role} must contain at least one scored row."
+        )
+    return records
+
+
+def load_submission_values(
+    path,
+    role,
+    record_key,
+    value_field,
+    duplicate_id_policy,
+    invalid_value_policy,
+    metric,
+):
+    fieldnames, rows = read_csv_rows(
+        path,
+        f"submission artifact {role}",
+        invalid_handler=reject_submission,
+    )
+    if record_key not in fieldnames:
+        reject_submission(
+            f"submission artifact {role} is missing record key column {record_key}."
+        )
+    if value_field not in fieldnames:
+        reject_submission(
+            f"submission artifact {role} is missing value column {value_field}."
+        )
+    values = {}
+    for row_index, row in enumerate(rows, start=2):
+        raw_key = row.get(record_key)
+        key = raw_key.strip() if isinstance(raw_key, str) else ""
+        if not key:
+            if invalid_value_policy == "reject":
+                reject_submission(
+                    f"submission artifact {role} row {row_index} is missing {record_key}."
+                )
+            continue
+        if key in values:
+            if duplicate_id_policy == "reject":
+                reject_submission(
+                    f"submission artifact {role} contains duplicate record id {key!r}."
+                )
+            continue
+        parsed_value = parse_submission_value(
+            row.get(value_field),
+            f"submission artifact {role} row {row_index} column {value_field}",
+            invalid_value_policy,
+            metric,
+        )
+        if parsed_value is None:
+            continue
+        values[key] = parsed_value
+    return values
+
+
+def compute_rmse(reference_values, submission_values):
+    squared_error = sum(
+        (submission - reference) ** 2
+        for reference, submission in zip(reference_values, submission_values)
+    )
+    return math.sqrt(squared_error / len(reference_values))
+
+
+def compute_mae(reference_values, submission_values):
+    absolute_error = sum(
+        abs(submission - reference)
+        for reference, submission in zip(reference_values, submission_values)
+    )
+    return absolute_error / len(reference_values)
+
+
+def compute_r2(reference_values, submission_values):
+    mean_reference = sum(reference_values) / len(reference_values)
+    ss_res = sum(
+        (reference - submission) ** 2
+        for reference, submission in zip(reference_values, submission_values)
+    )
+    ss_tot = sum((reference - mean_reference) ** 2 for reference in reference_values)
+    if ss_tot == 0:
+        return 1.0 if ss_res == 0 else 0.0
+    return 1.0 - (ss_res / ss_tot)
+
+
+def compute_pearson(reference_values, submission_values):
+    if len(reference_values) < 2:
+        reject_submission("Pearson correlation requires at least two scored rows.")
+    mean_reference = sum(reference_values) / len(reference_values)
+    mean_submission = sum(submission_values) / len(submission_values)
+    numerator = sum(
+        (reference - mean_reference) * (submission - mean_submission)
+        for reference, submission in zip(reference_values, submission_values)
+    )
+    denominator_left = sum(
+        (reference - mean_reference) ** 2 for reference in reference_values
+    )
+    denominator_right = sum(
+        (submission - mean_submission) ** 2 for submission in submission_values
+    )
+    denominator = math.sqrt(denominator_left * denominator_right)
+    if denominator == 0:
+        return 1.0 if reference_values == submission_values else 0.0
+    return numerator / denominator
+
+
+def average_ranks(values):
+    ordered = sorted(enumerate(values), key=lambda item: item[1])
+    ranks = [0.0] * len(values)
+    start = 0
+    while start < len(ordered):
+        end = start
+        while end + 1 < len(ordered) and ordered[end + 1][1] == ordered[start][1]:
+            end += 1
+        average_rank = (start + end + 2) / 2.0
+        for index in range(start, end + 1):
+            ranks[ordered[index][0]] = average_rank
+        start = end + 1
+    return ranks
+
+
+def compute_spearman(reference_values, submission_values):
+    return compute_pearson(
+        average_ranks(reference_values),
+        average_ranks(submission_values),
+    )
+
+
+def require_binary_reference_labels(metric, reference_values):
+    labels = []
+    for index, value in enumerate(reference_values, start=1):
+        if value == 0.0:
+            labels.append(0)
+            continue
+        if value == 1.0:
+            labels.append(1)
+            continue
+        fail_runtime(
+            f"table_metric metric {metric} requires evaluation labels encoded as 0 or 1; scored row {index} had {value}."
+        )
+    return labels
+
+
+def require_positive_reference_label(metric, reference_labels, label):
+    if sum(reference_labels) == 0:
+        fail_runtime(
+            f"table_metric metric {metric} requires at least one positive label in {label}."
+        )
+
+
+def threshold_submission_labels(submission_values):
+    return [1 if value >= 0.5 else 0 for value in submission_values]
+
+
+def compute_accuracy(reference_values, submission_values):
+    reference_labels = require_binary_reference_labels("accuracy", reference_values)
+    predicted_labels = threshold_submission_labels(submission_values)
+    correct = sum(
+        1
+        for reference, predicted in zip(reference_labels, predicted_labels)
+        if reference == predicted
+    )
+    return correct / len(reference_labels)
+
+
+def compute_f1(reference_values, submission_values):
+    reference_labels = require_binary_reference_labels("f1", reference_values)
+    predicted_labels = threshold_submission_labels(submission_values)
+    true_positive = sum(
+        1
+        for reference, predicted in zip(reference_labels, predicted_labels)
+        if reference == 1 and predicted == 1
+    )
+    false_positive = sum(
+        1
+        for reference, predicted in zip(reference_labels, predicted_labels)
+        if reference == 0 and predicted == 1
+    )
+    false_negative = sum(
+        1
+        for reference, predicted in zip(reference_labels, predicted_labels)
+        if reference == 1 and predicted == 0
+    )
+    if true_positive == 0 and false_positive == 0 and false_negative == 0:
+        return 1.0
+    if true_positive == 0:
+        return 0.0
+    precision = true_positive / (true_positive + false_positive)
+    recall = true_positive / (true_positive + false_negative)
+    if precision + recall == 0.0:
+        return 0.0
+    return (2.0 * precision * recall) / (precision + recall)
+
+
+def compute_auroc(reference_values, submission_values):
+    reference_labels = require_binary_reference_labels("auroc", reference_values)
+    positive_count = sum(reference_labels)
+    negative_count = len(reference_labels) - positive_count
+    if positive_count == 0 or negative_count == 0:
+        fail_runtime(
+            "table_metric metric auroc requires both positive and negative labels in the evaluation panel."
+        )
+    submission_ranks = average_ranks(submission_values)
+    positive_rank_sum = sum(
+        rank
+        for label, rank in zip(reference_labels, submission_ranks)
+        if label == 1
+    )
+    return (
+        positive_rank_sum - (positive_count * (positive_count + 1) / 2.0)
+    ) / (positive_count * negative_count)
+
+
+def compute_brier(reference_values, submission_values):
+    reference_labels = require_binary_reference_labels("brier", reference_values)
+    squared_error = sum(
+        (probability - label) ** 2
+        for label, probability in zip(reference_labels, submission_values)
+    )
+    return squared_error / len(reference_labels)
+
+
+def compute_log_loss(reference_values, submission_values):
+    reference_labels = require_binary_reference_labels("log_loss", reference_values)
+    total = 0.0
+    for label, probability in zip(reference_labels, submission_values):
+        clipped_probability = min(
+            max(probability, LOG_LOSS_EPSILON),
+            1.0 - LOG_LOSS_EPSILON,
+        )
+        total += -(
+            (label * math.log(clipped_probability))
+            + ((1 - label) * math.log(1.0 - clipped_probability))
+        )
+    return total / len(reference_labels)
+
+
+def compute_auprc(reference_values, submission_values, record_ids):
+    reference_labels = require_binary_reference_labels("auprc", reference_values)
+    positive_count = sum(reference_labels)
+    if positive_count == 0:
+        reject_submission(
+            "table_metric metric auprc requires at least one positive label among scored rows."
+        )
+
+    ordered = sorted(
+        zip(reference_labels, submission_values, record_ids),
+        key=lambda item: (-item[1], item[2]),
+    )
+    true_positive_count = 0
+    precision_sum = 0.0
+    for rank, (label, _probability, _record_id) in enumerate(ordered, start=1):
+        if label == 1:
+            true_positive_count += 1
+            precision_sum += true_positive_count / rank
+    return precision_sum / positive_count
+
+
+def compute_ece(reference_values, submission_values, bin_count):
+    reference_labels = require_binary_reference_labels("ece", reference_values)
+    bin_totals = [
+        {"count": 0, "probability_sum": 0.0, "positive_sum": 0}
+        for _index in range(bin_count)
+    ]
+    for label, probability in zip(reference_labels, submission_values):
+        bin_index = min(int(probability * bin_count), bin_count - 1)
+        bin_totals[bin_index]["count"] += 1
+        bin_totals[bin_index]["probability_sum"] += probability
+        bin_totals[bin_index]["positive_sum"] += label
+
+    scored_row_count = len(reference_labels)
+    total_error = 0.0
+    for bin_total in bin_totals:
+        bin_size = bin_total["count"]
+        if bin_size == 0:
+            continue
+        mean_probability = bin_total["probability_sum"] / bin_size
+        empirical_positive_rate = bin_total["positive_sum"] / bin_size
+        total_error += (
+            bin_size
+            / scored_row_count
+            * abs(mean_probability - empirical_positive_rate)
+        )
+    return total_error
+
+
+def compute_metric(
+    metric,
+    reference_values,
+    submission_values,
+    record_ids,
+    metric_params,
+):
+    if metric == "rmse":
+        return compute_rmse(reference_values, submission_values)
+    if metric == "mae":
+        return compute_mae(reference_values, submission_values)
+    if metric == "r2":
+        return compute_r2(reference_values, submission_values)
+    if metric == "pearson":
+        return compute_pearson(reference_values, submission_values)
+    if metric == "spearman":
+        return compute_spearman(reference_values, submission_values)
+    if metric == "accuracy":
+        return compute_accuracy(reference_values, submission_values)
+    if metric == "f1":
+        return compute_f1(reference_values, submission_values)
+    if metric == "auroc":
+        return compute_auroc(reference_values, submission_values)
+    if metric == "brier":
+        return compute_brier(reference_values, submission_values)
+    if metric == "log_loss":
+        return compute_log_loss(reference_values, submission_values)
+    if metric == "auprc":
+        return compute_auprc(reference_values, submission_values, record_ids)
+    if metric == "ece":
+        return compute_ece(
+            reference_values,
+            submission_values,
+            metric_params["bin_count"],
+        )
+    fail_runtime(f"table_metric metric {metric} is not supported by this scorer path.")
+
+
+def compute_calibrated_metric(metric, reference_records, submission_by_id, scored_ids):
+    if metric == "z_rmse":
+        z_errors = []
+        for record_id in scored_ids:
+            record = reference_records[record_id]
+            baseline_mean = record["baseline_mean"]
+            baseline_sd = record["baseline_sd"]
+            candidate_z = (submission_by_id[record_id] - baseline_mean) / baseline_sd
+            reference_z = (record["value"] - baseline_mean) / baseline_sd
+            z_errors.append(candidate_z - reference_z)
+        raw_metric = math.sqrt(sum(error**2 for error in z_errors) / len(z_errors))
+        return raw_metric, {
+            "mean_absolute_z_error": sum(abs(error) for error in z_errors)
+            / len(z_errors),
+            "max_absolute_z_error": max(abs(error) for error in z_errors),
+        }
+
+    passed_count = 0
+    for record_id in scored_ids:
+        record = reference_records[record_id]
+        if abs(submission_by_id[record_id] - record["value"]) <= record["noise_floor"]:
+            passed_count += 1
+    raw_metric = passed_count / len(scored_ids)
+    return raw_metric, {
+        "passed_count": passed_count,
+        "failed_count": len(scored_ids) - passed_count,
+    }
+
+
+def clamp01(value):
+    if value < 0.0:
+        return 0.0
+    if value > 1.0:
+        return 1.0
+    return value
+
+
+def normalize_score(metric, raw_metric):
+    if metric in MINIMIZE_METRICS:
+        return 1.0 / (1.0 + max(raw_metric, 0.0))
+    if metric == "r2":
+        return clamp01(raw_metric)
+    if metric in UNIT_INTERVAL_METRICS:
+        return clamp01(raw_metric)
+    return clamp01((raw_metric + 1.0) / 2.0)
+
+
+def group_object(group_by, group_key):
+    return {column: value for column, value in zip(group_by, group_key)}
+
+
+def format_group_label(group_by, group_key):
+    parts = [f"{column}={value!r}" for column, value in zip(group_by, group_key)]
+    return ", ".join(parts)
+
+
+def validate_reference_group_preconditions(metric, group_label, reference_values):
+    if metric in CORRELATION_METRICS and len(reference_values) < 2:
+        fail_runtime(
+            f"grouped table_metric metric {metric} requires at least two reference rows in group {group_label}."
+        )
+
+    if metric not in BINARY_LABEL_METRICS:
+        return
+
+    reference_labels = require_binary_reference_labels(metric, reference_values)
+    if metric == "auprc":
+        require_positive_reference_label(
+            metric,
+            reference_labels,
+            f"reference group {group_label}",
+        )
+        return
+    if metric != "auroc":
+        return
+
+    positive_count = sum(reference_labels)
+    negative_count = len(reference_labels) - positive_count
+    if positive_count == 0 or negative_count == 0:
+        fail_runtime(
+            f"grouped table_metric metric auroc requires both positive and negative labels in reference group {group_label}."
+        )
+
+
+def validate_scored_group_preconditions(metric, group_label, reference_values):
+    if metric in CORRELATION_METRICS and len(reference_values) < 2:
+        reject_submission(
+            f"grouped table_metric metric {metric} requires at least two scored rows in group {group_label}."
+        )
+
+    if metric == "auprc":
+        reference_labels = require_binary_reference_labels(metric, reference_values)
+        if sum(reference_labels) == 0:
+            reject_submission(
+                f"grouped table_metric metric auprc requires at least one positive label among scored rows in group {group_label}."
+            )
+        return
+
+    if metric != "auroc":
+        return
+
+    reference_labels = require_binary_reference_labels(metric, reference_values)
+    positive_count = sum(reference_labels)
+    negative_count = len(reference_labels) - positive_count
+    if positive_count == 0 or negative_count == 0:
+        reject_submission(
+            f"grouped table_metric metric auroc requires both positive and negative labels among scored rows in group {group_label}."
+        )
+
+
+def validate_reference_panel_preconditions(metric, reference_values):
+    if metric not in BINARY_LABEL_METRICS:
+        return
+
+    reference_labels = require_binary_reference_labels(metric, reference_values)
+    if metric == "auprc":
+        require_positive_reference_label(metric, reference_labels, "the evaluation panel")
+        return
+    if metric != "auroc":
+        return
+
+    positive_count = sum(reference_labels)
+    negative_count = len(reference_labels) - positive_count
+    if positive_count == 0 or negative_count == 0:
+        fail_runtime(
+            "table_metric metric auroc requires both positive and negative labels in the evaluation panel."
+        )
+
+
+def validate_objective(metric, objective):
+    if metric in MINIMIZE_METRICS and objective != "minimize":
+        fail_runtime(
+            f"table_metric metric {metric} requires objective=minimize, received {objective}."
+        )
+    if metric in MAXIMIZE_METRICS and objective != "maximize":
+        fail_runtime(
+            f"table_metric metric {metric} requires objective=maximize, received {objective}."
+        )
+
+
+def score_grouped_table_metric(
+    *,
+    metric,
+    metric_params,
+    group_by,
+    aggregation,
+    evaluation_path,
+    evaluation_role,
+    evaluation_record_key,
+    evaluation_value_field,
+    submission_path,
+    submission_role,
+    submission_record_key,
+    submission_value_field,
+    coverage_policy,
+    duplicate_id_policy,
+    invalid_value_policy,
+    final_score_key,
+):
+    reference_by_id = load_reference_group_records(
+        evaluation_path,
+        evaluation_role,
+        evaluation_record_key,
+        evaluation_value_field,
+        group_by,
+    )
+    groups = {}
+    for record_id, record in reference_by_id.items():
+        groups.setdefault(record["group_key"], []).append(record_id)
+    for group_key in sorted(groups):
+        group_label = format_group_label(group_by, group_key)
+        group_reference_values = [
+            reference_by_id[record_id]["value"] for record_id in groups[group_key]
+        ]
+        validate_reference_group_preconditions(
+            metric,
+            group_label,
+            group_reference_values,
+        )
+
+    submission_by_id = load_submission_values(
+        submission_path,
+        submission_role,
+        submission_record_key,
+        submission_value_field,
+        duplicate_id_policy,
+        invalid_value_policy,
+        metric,
+    )
+    missing_ids = [
+        record_id
+        for record_id in reference_by_id
+        if record_id not in submission_by_id
+    ]
+    if missing_ids and coverage_policy == "reject":
+        reject_submission(
+            f"Submission is missing predictions for {len(missing_ids)} required rows; first missing id is {missing_ids[0]!r}."
+        )
+
+    group_details = []
+    group_scores = []
+    total_scored = 0
+
+    for group_key in sorted(groups):
+        group_record_ids = groups[group_key]
+        group_label = format_group_label(group_by, group_key)
+        group_missing_ids = [
+            record_id for record_id in group_record_ids if record_id not in submission_by_id
+        ]
+        group_scored_ids = [
+            record_id for record_id in group_record_ids if record_id in submission_by_id
+        ]
+        if not group_scored_ids:
+            reject_submission(
+                f"Submission produced no scoreable rows for group {group_label} after applying runtime policies."
+            )
+
+        scored_reference_values = [
+            reference_by_id[record_id]["value"] for record_id in group_scored_ids
+        ]
+        scored_submission_values = [
+            submission_by_id[record_id] for record_id in group_scored_ids
+        ]
+        validate_scored_group_preconditions(
+            metric,
+            group_label,
+            scored_reference_values,
+        )
+        raw_metric = compute_metric(
+            metric,
+            scored_reference_values,
+            scored_submission_values,
+            group_scored_ids,
+            metric_params,
+        )
+        normalized_score = normalize_score(metric, raw_metric)
+        if coverage_policy == "penalize":
+            normalized_score *= len(group_scored_ids) / len(group_record_ids)
+
+        aggregation_weight = len(group_scored_ids)
+        total_scored += aggregation_weight
+        group_scores.append(
+            {
+                "score": normalized_score,
+                "weight": aggregation_weight,
+            }
+        )
+        group_details.append(
+            {
+                "group": group_object(group_by, group_key),
+                "reference_row_count": len(group_record_ids),
+                "rows_scored": len(group_scored_ids),
+                "missing_count": len(group_missing_ids),
+                "selected_metric_value": raw_metric,
+                "normalized_score": normalized_score,
+                "aggregation_weight": aggregation_weight,
+            }
+        )
+
+    if not group_scores:
+        reject_submission(
+            "Submission produced no scoreable groups after applying runtime policies."
+        )
+
+    if aggregation == "macro_mean":
+        final_score = sum(group_score["score"] for group_score in group_scores) / len(
+            group_scores
+        )
+    else:
+        total_weight = sum(group_score["weight"] for group_score in group_scores)
+        if total_weight <= 0:
+            reject_submission(
+                "Submission produced no weighted group rows after applying runtime policies."
+            )
+        final_score = (
+            sum(
+                group_score["score"] * group_score["weight"]
+                for group_score in group_scores
+            )
+            / total_weight
+        )
+
+    write_score(
+        score=final_score,
+        details={
+            final_score_key: final_score,
+            "selected_metric": metric,
+            "aggregation": aggregation,
+            "group_by": group_by,
+            "group_count": len(group_details),
+            "rows_scored": total_scored,
+            "missing_count": len(missing_ids),
+            "groups": group_details,
+        },
+    )
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    try:
+        config = load_json_file(config_path, label="compiled_config")
+    except RuntimeError as error:
+        fail_runtime(str(error))
+    metric = require_metric(config)
+    metric_params = require_metric_params(config, metric)
+    group_by, aggregation = require_grouping_config(config)
+    evaluation_role = require_string(
+        config.get("evaluation_role"),
+        "compiled_config.evaluation_role",
+    )
+    submission_role = require_string(
+        config.get("submission_role"),
+        "compiled_config.submission_role",
+    )
+    final_score_key = require_string(
+        runtime_context.get("final_score_key"),
+        "runtime_context.final_score_key",
+    )
+    objective = require_string(
+        runtime_context.get("objective"),
+        "runtime_context.objective",
+    )
+    validate_objective(metric, objective)
+    policies = runtime_context.get("policies")
+    if not isinstance(policies, dict):
+        fail_runtime("Runtime context is missing execution policies.")
+    coverage_policy = require_policy(
+        policies,
+        "coverage_policy",
+        {"reject", "ignore", "penalize"},
+    )
+    duplicate_id_policy = require_policy(
+        policies,
+        "duplicate_id_policy",
+        {"reject", "ignore"},
+    )
+    invalid_value_policy = require_policy(
+        policies,
+        "invalid_value_policy",
+        {"reject", "ignore"},
+    )
+    evaluation_record_key, evaluation_value_field = require_csv_slot_columns(
+        runtime_context,
+        "evaluation",
+        evaluation_role,
+    )
+    submission_record_key, submission_value_field = require_csv_slot_columns(
+        runtime_context,
+        "submission",
+        submission_role,
+    )
+    evaluation_path = resolve_evaluation_artifact(runtime_context, evaluation_role)
+    submission_path = resolve_submission_artifact(runtime_context, submission_role)
+    if group_by:
+        if metric in CALIBRATED_METRICS:
+            fail_runtime(
+                f"table_metric metric {metric} is not supported with compiled_config.group_by; use calibrated_table_metric@1 without grouped params."
+            )
+        score_grouped_table_metric(
+            metric=metric,
+            metric_params=metric_params,
+            group_by=group_by,
+            aggregation=aggregation,
+            evaluation_path=evaluation_path,
+            evaluation_role=evaluation_role,
+            evaluation_record_key=evaluation_record_key,
+            evaluation_value_field=evaluation_value_field,
+            submission_path=submission_path,
+            submission_role=submission_role,
+            submission_record_key=submission_record_key,
+            submission_value_field=submission_value_field,
+            coverage_policy=coverage_policy,
+            duplicate_id_policy=duplicate_id_policy,
+            invalid_value_policy=invalid_value_policy,
+            final_score_key=final_score_key,
+        )
+        return
+
+    if metric in CALIBRATED_METRICS:
+        if metric == "z_rmse":
+            require_config_field_matches_metric_params(
+                config,
+                metric_params,
+                "baseline_mean_field",
+            )
+            require_config_field_matches_metric_params(
+                config,
+                metric_params,
+                "baseline_sd_field",
+            )
+        elif metric == "within_noise_fraction":
+            require_config_field_matches_metric_params(
+                config,
+                metric_params,
+                "noise_floor_field",
+            )
+
+        reference_by_id = load_calibrated_reference_records(
+            evaluation_path,
+            evaluation_role,
+            evaluation_record_key,
+            evaluation_value_field,
+            metric,
+            metric_params,
+        )
+        submission_by_id = load_submission_values(
+            submission_path,
+            submission_role,
+            submission_record_key,
+            submission_value_field,
+            duplicate_id_policy,
+            invalid_value_policy,
+            metric,
+        )
+        missing_ids = [
+            record_id
+            for record_id in reference_by_id
+            if record_id not in submission_by_id
+        ]
+        if missing_ids and coverage_policy == "reject":
+            reject_submission(
+                f"Submission is missing predictions for {len(missing_ids)} required rows; first missing id is {missing_ids[0]!r}."
+            )
+        scored_ids = [
+            record_id
+            for record_id in reference_by_id
+            if record_id in submission_by_id
+        ]
+        if not scored_ids:
+            reject_submission(
+                "Submission produced no scoreable rows after applying runtime policies."
+            )
+        raw_metric, metric_details = compute_calibrated_metric(
+            metric,
+            reference_by_id,
+            submission_by_id,
+            scored_ids,
+        )
+        normalized_score = normalize_score(metric, raw_metric)
+        if coverage_policy == "penalize":
+            normalized_score *= len(scored_ids) / len(reference_by_id)
+        write_score(
+            score=normalized_score,
+            details={
+                final_score_key: normalized_score,
+                "selected_metric": metric,
+                "selected_metric_value": raw_metric,
+                "rows_scored": len(scored_ids),
+                **metric_details,
+            },
+        )
+        return
+
+    reference_by_id = load_reference_values(
+        evaluation_path,
+        evaluation_role,
+        evaluation_record_key,
+        evaluation_value_field,
+    )
+    validate_reference_panel_preconditions(
+        metric,
+        list(reference_by_id.values()),
+    )
+    submission_by_id = load_submission_values(
+        submission_path,
+        submission_role,
+        submission_record_key,
+        submission_value_field,
+        duplicate_id_policy,
+        invalid_value_policy,
+        metric,
+    )
+    missing_ids = [
+        record_id
+        for record_id in reference_by_id
+        if record_id not in submission_by_id
+    ]
+    if missing_ids and coverage_policy == "reject":
+        reject_submission(
+            f"Submission is missing predictions for {len(missing_ids)} required rows; first missing id is {missing_ids[0]!r}."
+        )
+    scored_ids = [
+        record_id
+        for record_id in reference_by_id
+        if record_id in submission_by_id
+    ]
+    if not scored_ids:
+        reject_submission(
+            "Submission produced no scoreable rows after applying runtime policies."
+        )
+    reference_values = [reference_by_id[record_id] for record_id in scored_ids]
+    submission_values = [submission_by_id[record_id] for record_id in scored_ids]
+    raw_metric = compute_metric(
+        metric,
+        reference_values,
+        submission_values,
+        scored_ids,
+        metric_params,
+    )
+    normalized_score = normalize_score(metric, raw_metric)
+    if coverage_policy == "penalize":
+        normalized_score *= len(scored_ids) / len(reference_by_id)
+    write_score(
+        score=normalized_score,
+        details={
+            final_score_key: normalized_score,
+            "selected_metric": metric,
+            "selected_metric_value": raw_metric,
+            "rows_scored": len(scored_ids),
+        },
+    )
+
+
+if __name__ == "__main__":
+    main()