@moleculeagora/cli 0.1.0

package/dist/python-v1/multi-output-tabular-metric.py

@@ -0,0 +1,392 @@
+import csv
+import math
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    reject_submission,
+    resolve_evaluation_artifact,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    write_score,
+)
+
+SUPPORTED_METRIC = "mcrmse"
+
+
+def require_string(value, label):
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_metric(config):
+    metric = require_string(config.get("metric"), "compiled_config.metric").lower()
+    if metric != SUPPORTED_METRIC:
+        fail_runtime("compiled_config.metric must be mcrmse.")
+    return metric
+
+
+def require_policy(policies, key, allowed):
+    value = require_string(policies.get(key), f"policies.{key}")
+    if value not in allowed:
+        fail_runtime(
+            f"policies.{key} must be one of {', '.join(sorted(allowed))}."
+        )
+    return value
+
+
+def find_slot(runtime_context, lane, role):
+    slot_key = f"{lane}_slots"
+    slots = runtime_context.get(slot_key)
+    if not isinstance(slots, list):
+        fail_runtime(f"Runtime context is missing {slot_key}.")
+    for slot in slots:
+        if isinstance(slot, dict) and slot.get("role") == role:
+            return slot
+    fail_runtime(f"Runtime context is missing {lane} slot for role {role}.")
+
+
+def require_csv_slot_columns(runtime_context, lane, role):
+    slot = find_slot(runtime_context, lane, role)
+    validator = slot.get("validator")
+    if not isinstance(validator, dict) or validator.get("kind") != "csv_columns":
+        fail_runtime(
+            f"{lane} role {role} must use validator.kind=csv_columns for multi_output_tabular_metric."
+        )
+    record_key = require_string(
+        validator.get("record_key"),
+        f"{lane}.{role}.validator.record_key",
+    )
+    value_fields = validator.get("value_fields")
+    if not isinstance(value_fields, list) or not value_fields:
+        fail_runtime(
+            f"{lane}.{role}.validator.value_fields must be a non-empty array."
+        )
+    normalized_value_fields = [
+        require_string(field, f"{lane}.{role}.validator.value_fields[{index}]")
+        for index, field in enumerate(value_fields)
+    ]
+    if len(set(normalized_value_fields)) != len(normalized_value_fields):
+        fail_runtime(f"{lane}.{role}.validator.value_fields must be unique.")
+    return record_key, normalized_value_fields
+
+
+def read_csv_rows(path, label, *, invalid_handler):
+    try:
+        with path.open("r", encoding="utf-8", newline="") as handle:
+            reader = csv.DictReader(handle)
+            fieldnames = reader.fieldnames
+            if not fieldnames:
+                invalid_handler(f"{label} must include a CSV header row.")
+            normalized_fieldnames = []
+            for fieldname in fieldnames:
+                if not isinstance(fieldname, str) or not fieldname.strip():
+                    invalid_handler(f"{label} contains an empty CSV column name.")
+                normalized_fieldnames.append(fieldname.strip())
+            rows = list(reader)
+    except FileNotFoundError:
+        invalid_handler(f"Missing {label} at {path}.")
+    except OSError as error:
+        invalid_handler(f"Unable to read {label}: {error}.")
+    return normalized_fieldnames, rows
+
+
+def parse_reference_value(raw_value, label):
+    text = raw_value.strip() if isinstance(raw_value, str) else ""
+    if not text:
+        fail_runtime(f"{label} is blank.")
+    try:
+        value = float(text)
+    except ValueError:
+        fail_runtime(f"{label} must be numeric, received {text!r}.")
+    if not math.isfinite(value):
+        fail_runtime(f"{label} must be finite.")
+    return value
+
+
+def parse_submission_value(raw_value, label, invalid_value_policy):
+    text = raw_value.strip() if isinstance(raw_value, str) else ""
+    if not text:
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} is blank.")
+        return None
+    try:
+        value = float(text)
+    except ValueError:
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} must be numeric, received {text!r}.")
+        return None
+    if not math.isfinite(value):
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} must be finite.")
+        return None
+    return value
+
+
+def require_columns(fieldnames, role, lane, record_key, value_fields, invalid_handler):
+    if record_key not in fieldnames:
+        invalid_handler(f"{lane} artifact {role} is missing record key column {record_key}.")
+    for value_field in value_fields:
+        if value_field not in fieldnames:
+            invalid_handler(
+                f"{lane} artifact {role} is missing target column {value_field}."
+            )
+
+
+def load_reference_values(path, role, record_key, value_fields):
+    fieldnames, rows = read_csv_rows(
+        path,
+        f"evaluation artifact {role}",
+        invalid_handler=fail_runtime,
+    )
+    require_columns(
+        fieldnames,
+        role,
+        "evaluation",
+        record_key,
+        value_fields,
+        fail_runtime,
+    )
+    values = {}
+    for row_index, row in enumerate(rows, start=2):
+        raw_key = row.get(record_key)
+        key = raw_key.strip() if isinstance(raw_key, str) else ""
+        if not key:
+            fail_runtime(
+                f"evaluation artifact {role} row {row_index} is missing {record_key}."
+            )
+        if key in values:
+            fail_runtime(
+                f"evaluation artifact {role} contains duplicate record id {key!r}."
+            )
+        values[key] = {
+            value_field: parse_reference_value(
+                row.get(value_field),
+                f"evaluation artifact {role} row {row_index} column {value_field}",
+            )
+            for value_field in value_fields
+        }
+    if not values:
+        fail_runtime(
+            f"evaluation artifact {role} must contain at least one scored row."
+        )
+    return values
+
+
+def parse_submission_row(row, role, row_index, value_fields, invalid_value_policy):
+    parsed = {}
+    for value_field in value_fields:
+        parsed_value = parse_submission_value(
+            row.get(value_field),
+            f"submission artifact {role} row {row_index} column {value_field}",
+            invalid_value_policy,
+        )
+        if parsed_value is None:
+            return None
+        parsed[value_field] = parsed_value
+    return parsed
+
+
+def load_submission_values(
+    path,
+    role,
+    record_key,
+    value_fields,
+    duplicate_id_policy,
+    invalid_value_policy,
+):
+    fieldnames, rows = read_csv_rows(
+        path,
+        f"submission artifact {role}",
+        invalid_handler=reject_submission,
+    )
+    require_columns(
+        fieldnames,
+        role,
+        "submission",
+        record_key,
+        value_fields,
+        reject_submission,
+    )
+    values = {}
+    for row_index, row in enumerate(rows, start=2):
+        raw_key = row.get(record_key)
+        key = raw_key.strip() if isinstance(raw_key, str) else ""
+        if not key:
+            if invalid_value_policy == "reject":
+                reject_submission(
+                    f"submission artifact {role} row {row_index} is missing {record_key}."
+                )
+            continue
+        if key in values:
+            if duplicate_id_policy == "reject":
+                reject_submission(
+                    f"submission artifact {role} contains duplicate record id {key!r}."
+                )
+            continue
+        parsed_row = parse_submission_row(
+            row,
+            role,
+            row_index,
+            value_fields,
+            invalid_value_policy,
+        )
+        if parsed_row is None:
+            continue
+        values[key] = parsed_row
+    return values
+
+
+def compute_rmse(reference_values, submission_values):
+    squared_error = sum(
+        (submission - reference) ** 2
+        for reference, submission in zip(reference_values, submission_values)
+    )
+    return math.sqrt(squared_error / len(reference_values))
+
+
+def compute_mcrmse(reference_by_id, submission_by_id, scored_ids, value_fields):
+    target_rmse = {}
+    for value_field in value_fields:
+        reference_values = [
+            reference_by_id[record_id][value_field] for record_id in scored_ids
+        ]
+        submission_values = [
+            submission_by_id[record_id][value_field] for record_id in scored_ids
+        ]
+        target_rmse[value_field] = compute_rmse(reference_values, submission_values)
+    return sum(target_rmse.values()) / len(target_rmse), target_rmse
+
+
+def normalize_score(raw_metric):
+    return 1.0 / (1.0 + max(raw_metric, 0.0))
+
+
+def validate_objective(objective):
+    if objective != "minimize":
+        fail_runtime(
+            f"multi_output_tabular_metric metric mcrmse requires objective=minimize, received {objective}."
+        )
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    try:
+        config = load_json_file(config_path, label="compiled_config")
+    except RuntimeError as error:
+        fail_runtime(str(error))
+    metric = require_metric(config)
+    evaluation_role = require_string(
+        config.get("evaluation_role"),
+        "compiled_config.evaluation_role",
+    )
+    submission_role = require_string(
+        config.get("submission_role"),
+        "compiled_config.submission_role",
+    )
+    final_score_key = require_string(
+        runtime_context.get("final_score_key"),
+        "runtime_context.final_score_key",
+    )
+    objective = require_string(
+        runtime_context.get("objective"),
+        "runtime_context.objective",
+    )
+    validate_objective(objective)
+    policies = runtime_context.get("policies")
+    if not isinstance(policies, dict):
+        fail_runtime("Runtime context is missing execution policies.")
+    coverage_policy = require_policy(
+        policies,
+        "coverage_policy",
+        {"reject", "ignore", "penalize"},
+    )
+    duplicate_id_policy = require_policy(
+        policies,
+        "duplicate_id_policy",
+        {"reject", "ignore"},
+    )
+    invalid_value_policy = require_policy(
+        policies,
+        "invalid_value_policy",
+        {"reject", "ignore"},
+    )
+    evaluation_record_key, evaluation_value_fields = require_csv_slot_columns(
+        runtime_context,
+        "evaluation",
+        evaluation_role,
+    )
+    submission_record_key, submission_value_fields = require_csv_slot_columns(
+        runtime_context,
+        "submission",
+        submission_role,
+    )
+    if evaluation_value_fields != submission_value_fields:
+        fail_runtime(
+            "evaluation and submission csv_columns.validator.value_fields must match for multi_output_tabular_metric."
+        )
+    evaluation_path = resolve_evaluation_artifact(runtime_context, evaluation_role)
+    submission_path = resolve_submission_artifact(runtime_context, submission_role)
+    reference_by_id = load_reference_values(
+        evaluation_path,
+        evaluation_role,
+        evaluation_record_key,
+        evaluation_value_fields,
+    )
+    submission_by_id = load_submission_values(
+        submission_path,
+        submission_role,
+        submission_record_key,
+        submission_value_fields,
+        duplicate_id_policy,
+        invalid_value_policy,
+    )
+    missing_ids = [
+        record_id
+        for record_id in reference_by_id
+        if record_id not in submission_by_id
+    ]
+    if missing_ids and coverage_policy == "reject":
+        reject_submission(
+            f"Submission is missing predictions for {len(missing_ids)} required rows; first missing id is {missing_ids[0]!r}."
+        )
+    scored_ids = [
+        record_id
+        for record_id in reference_by_id
+        if record_id in submission_by_id
+    ]
+    if not scored_ids:
+        reject_submission(
+            "Submission produced no scoreable rows after applying runtime policies."
+        )
+    raw_metric, target_rmse = compute_mcrmse(
+        reference_by_id,
+        submission_by_id,
+        scored_ids,
+        evaluation_value_fields,
+    )
+    normalized_score = normalize_score(raw_metric)
+    if coverage_policy == "penalize":
+        normalized_score *= len(scored_ids) / len(reference_by_id)
+    write_score(
+        score=normalized_score,
+        details={
+            final_score_key: normalized_score,
+            "selected_metric": metric,
+            "selected_metric_value": raw_metric,
+            "rows_scored": len(scored_ids),
+            "target_count": len(evaluation_value_fields),
+            "target_rmse": target_rmse,
+        },
+    )
+
+
+if __name__ == "__main__":
+    main()