@moleculeagora/cli 0.1.0

package/dist/python-v1/statistical-endpoint-test-v2.py

@@ -0,0 +1,629 @@
+import csv
+import math
+import re
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    reject_submission,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    write_score,
+)
+
+SUPPORTED_DIRECTIONS = ("treatment_lt_control", "treatment_gt_control")
+SUPPORTED_P_ADJUSTMENTS = ("bonferroni", "holm")
+CONTRAST_ID_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$")
+BETA_CONTINUED_FRACTION_MAX_ITERATIONS = 200
+BETA_CONTINUED_FRACTION_EPSILON = 3e-14
+BETA_CONTINUED_FRACTION_MIN_FLOAT = 1e-300
+
+
+def require_string(value, label, invalid_handler=fail_runtime):
+    if not isinstance(value, str) or not value.strip():
+        invalid_handler(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_config_dict(config, key):
+    value = config.get(key)
+    if not isinstance(value, dict):
+        fail_runtime(f"compiled_config.{key} must be an object.")
+    return value
+
+
+def require_alpha(value):
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        fail_runtime("endpoint_test.alpha must be a finite number.")
+    alpha = float(value)
+    if not math.isfinite(alpha) or alpha <= 0.0 or alpha >= 1.0:
+        fail_runtime("endpoint_test.alpha must be finite, greater than 0, and less than 1.")
+    return alpha
+
+
+def require_minimum_n(value):
+    if isinstance(value, bool) or not isinstance(value, int):
+        fail_runtime("endpoint_test.minimum_n_per_group must be an integer.")
+    if value < 2:
+        fail_runtime("endpoint_test.minimum_n_per_group must be at least 2.")
+    return value
+
+
+def require_string_list(value, label, min_length, max_length):
+    if not isinstance(value, list):
+        fail_runtime(f"{label} must be an array.")
+    if len(value) < min_length or len(value) > max_length:
+        fail_runtime(f"{label} must contain between {min_length} and {max_length} entries.")
+    result = []
+    seen = set()
+    for index, item in enumerate(value):
+        text = require_string(item, f"{label}[{index}]")
+        if text in seen:
+            fail_runtime(f"{label} repeats value {text!r}.")
+        seen.add(text)
+        result.append(text)
+    return result
+
+
+def require_p_adjustment(value):
+    adjustment = require_string(value, "endpoint_test.p_adjustment")
+    if adjustment not in SUPPORTED_P_ADJUSTMENTS:
+        fail_runtime(
+            f"endpoint_test.p_adjustment must be one of {', '.join(SUPPORTED_P_ADJUSTMENTS)}."
+        )
+    return adjustment
+
+
+def load_contrasts(value, included_groups):
+    if not isinstance(value, list):
+        fail_runtime("endpoint_test.contrasts must be an array.")
+    if len(value) < 1 or len(value) > 20:
+        fail_runtime("endpoint_test.contrasts must contain between 1 and 20 entries.")
+    group_set = set(included_groups)
+    contrast_ids = set()
+    contrasts = []
+    for index, item in enumerate(value):
+        if not isinstance(item, dict):
+            fail_runtime(f"endpoint_test.contrasts[{index}] must be an object.")
+        contrast_id = require_string(item.get("id"), f"endpoint_test.contrasts[{index}].id")
+        if not CONTRAST_ID_PATTERN.match(contrast_id):
+            fail_runtime(
+                f"endpoint_test.contrasts[{index}].id must start with a lowercase letter and use only lowercase letters, digits, and underscores."
+            )
+        if contrast_id in contrast_ids:
+            fail_runtime(f"endpoint_test.contrasts repeats id {contrast_id}.")
+        contrast_ids.add(contrast_id)
+        treatment_group = require_string(
+            item.get("treatment_group"),
+            f"endpoint_test.contrasts[{index}].treatment_group",
+        )
+        control_group = require_string(
+            item.get("control_group"),
+            f"endpoint_test.contrasts[{index}].control_group",
+        )
+        if treatment_group == control_group:
+            fail_runtime(
+                f"endpoint_test.contrasts[{index}] treatment_group and control_group must be different."
+            )
+        if treatment_group not in group_set:
+            fail_runtime(
+                f"endpoint_test.contrasts[{index}].treatment_group must reference included_groups."
+            )
+        if control_group not in group_set:
+            fail_runtime(
+                f"endpoint_test.contrasts[{index}].control_group must reference included_groups."
+            )
+        expected_direction = require_string(
+            item.get("expected_direction"),
+            f"endpoint_test.contrasts[{index}].expected_direction",
+        )
+        if expected_direction not in SUPPORTED_DIRECTIONS:
+            fail_runtime(
+                f"endpoint_test.contrasts[{index}].expected_direction must be one of {', '.join(SUPPORTED_DIRECTIONS)}."
+            )
+        contrasts.append(
+            {
+                "id": contrast_id,
+                "treatment_group": treatment_group,
+                "control_group": control_group,
+                "expected_direction": expected_direction,
+            }
+        )
+    return contrasts
+
+
+def load_endpoint_test_config(config):
+    endpoint_test = require_config_dict(config, "endpoint_test")
+    group_column = require_string(
+        endpoint_test.get("group_column"),
+        "endpoint_test.group_column",
+    )
+    value_column = require_string(
+        endpoint_test.get("value_column"),
+        "endpoint_test.value_column",
+    )
+    if group_column == value_column:
+        fail_runtime("endpoint_test.group_column and endpoint_test.value_column must be different.")
+    included_groups = require_string_list(
+        endpoint_test.get("included_groups"),
+        "endpoint_test.included_groups",
+        3,
+        20,
+    )
+    return {
+        "group_column": group_column,
+        "value_column": value_column,
+        "included_groups": included_groups,
+        "alpha": require_alpha(endpoint_test.get("alpha")),
+        "minimum_n_per_group": require_minimum_n(
+            endpoint_test.get("minimum_n_per_group")
+        ),
+        "p_adjustment": require_p_adjustment(endpoint_test.get("p_adjustment")),
+        "contrasts": load_contrasts(endpoint_test.get("contrasts"), included_groups),
+    }
+
+
+def require_observations_slot(runtime_context, role, params):
+    artifact_contract = runtime_context.get("artifact_contract")
+    if not isinstance(artifact_contract, dict):
+        fail_runtime("Runtime context is missing artifact_contract.")
+    slots = artifact_contract.get("submission")
+    if not isinstance(slots, list):
+        fail_runtime("Runtime context is missing artifact_contract.submission.")
+    for slot in slots:
+        if not isinstance(slot, dict) or slot.get("role") != role:
+            continue
+        validator = slot.get("validator")
+        if not isinstance(validator, dict) or validator.get("kind") != "csv_columns":
+            fail_runtime(
+                f"submission role {role} must use validator.kind=csv_columns for multi_group_endpoint_test@1."
+            )
+        required = validator.get("required")
+        if not isinstance(required, list):
+            fail_runtime(
+                f"submission role {role} validator.required must be an array for multi_group_endpoint_test@1."
+            )
+        required_columns = {
+            str(column).strip()
+            for column in required
+            if isinstance(column, str) and str(column).strip()
+        }
+        expected_columns = {
+            "observation_id",
+            params["group_column"],
+            params["value_column"],
+        }
+        missing = sorted(expected_columns - required_columns)
+        if missing:
+            fail_runtime(
+                f"submission role {role} validator.required must include {', '.join(missing)} for multi_group_endpoint_test@1."
+            )
+        return
+    fail_runtime(f"Runtime context is missing submission slot for role {role}.")
+
+
+def read_csv_rows(path, label):
+    try:
+        with path.open("r", encoding="utf-8", newline="") as handle:
+            reader = csv.DictReader(handle)
+            fieldnames = reader.fieldnames
+            if not fieldnames:
+                reject_submission(f"{label} must include a CSV header row.")
+            normalized = []
+            seen = set()
+            for fieldname in fieldnames:
+                if not isinstance(fieldname, str) or not fieldname.strip():
+                    reject_submission(f"{label} contains an empty CSV column name.")
+                column = fieldname.strip()
+                if column in seen:
+                    reject_submission(f"{label} contains duplicate CSV column {column!r}.")
+                seen.add(column)
+                normalized.append((fieldname, column))
+            rows = []
+            for row_index, row in enumerate(reader, start=2):
+                if None in row:
+                    reject_submission(f"{label} row {row_index} has too many columns.")
+                rows.append(
+                    {
+                        normalized_name: row.get(raw_name, "")
+                        for raw_name, normalized_name in normalized
+                    }
+                )
+    except FileNotFoundError:
+        reject_submission(f"Missing {label} at {path}.")
+    except OSError as error:
+        reject_submission(f"Unable to read {label}: {error}.")
+    return [column for _, column in normalized], rows
+
+
+def require_columns(fieldnames, required_columns, label):
+    missing = [column for column in required_columns if column not in fieldnames]
+    if missing:
+        reject_submission(f"{label} is missing required columns: {', '.join(missing)}.")
+
+
+def parse_finite_number(raw_value, label):
+    text = str(raw_value).strip() if raw_value is not None else ""
+    if not text:
+        reject_submission(f"{label} must be present.")
+    try:
+        value = float(text)
+    except ValueError:
+        reject_submission(f"{label} must be numeric, received {text!r}.")
+    if not math.isfinite(value):
+        reject_submission(f"{label} must be finite.")
+    return value
+
+
+def load_observations(path, role, params):
+    fieldnames, rows = read_csv_rows(path, f"submission artifact {role}")
+    require_columns(
+        fieldnames,
+        ["observation_id", params["group_column"], params["value_column"]],
+        f"submission artifact {role}",
+    )
+    included_groups = set(params["included_groups"])
+    seen_observation_ids = set()
+    values_by_group = {group: [] for group in params["included_groups"]}
+    for row_index, row in enumerate(rows, start=2):
+        observation_id = require_string(
+            row.get("observation_id"),
+            f"submission artifact {role} row {row_index} observation_id",
+            reject_submission,
+        )
+        if observation_id in seen_observation_ids:
+            reject_submission(
+                f"submission artifact {role} contains duplicate observation_id {observation_id!r}."
+            )
+        seen_observation_ids.add(observation_id)
+        group = require_string(
+            row.get(params["group_column"]),
+            f"submission artifact {role} row {row_index} {params['group_column']}",
+            reject_submission,
+        )
+        value = parse_finite_number(
+            row.get(params["value_column"]),
+            f"submission artifact {role} row {row_index} {params['value_column']}",
+        )
+        if group in included_groups:
+            values_by_group[group].append(value)
+    return values_by_group
+
+
+def require_group_sizes(values_by_group, minimum_n):
+    for group, values in values_by_group.items():
+        if len(values) < minimum_n:
+            reject_submission(
+                f"multi_group_endpoint_test@1 requires at least {minimum_n} observations for group {group!r}; received {len(values)}.",
+                details={"group": group, "n": len(values)},
+            )
+
+
+def sample_mean(values):
+    return sum(values) / len(values)
+
+
+def sample_variance(values, mean):
+    return sum((value - mean) ** 2 for value in values) / (len(values) - 1)
+
+
+def beta_continued_fraction(a, b, x):
+    qab = a + b
+    qap = a + 1.0
+    qam = a - 1.0
+    c = 1.0
+    d = 1.0 - (qab * x / qap)
+    if abs(d) < BETA_CONTINUED_FRACTION_MIN_FLOAT:
+        d = BETA_CONTINUED_FRACTION_MIN_FLOAT
+    d = 1.0 / d
+    h = d
+    for iteration in range(1, BETA_CONTINUED_FRACTION_MAX_ITERATIONS + 1):
+        m2 = 2 * iteration
+        aa = iteration * (b - iteration) * x / ((qam + m2) * (a + m2))
+        d = 1.0 + aa * d
+        if abs(d) < BETA_CONTINUED_FRACTION_MIN_FLOAT:
+            d = BETA_CONTINUED_FRACTION_MIN_FLOAT
+        c = 1.0 + aa / c
+        if abs(c) < BETA_CONTINUED_FRACTION_MIN_FLOAT:
+            c = BETA_CONTINUED_FRACTION_MIN_FLOAT
+        d = 1.0 / d
+        h *= d * c
+
+        aa = -((a + iteration) * (qab + iteration) * x) / (
+            (a + m2) * (qap + m2)
+        )
+        d = 1.0 + aa * d
+        if abs(d) < BETA_CONTINUED_FRACTION_MIN_FLOAT:
+            d = BETA_CONTINUED_FRACTION_MIN_FLOAT
+        c = 1.0 + aa / c
+        if abs(c) < BETA_CONTINUED_FRACTION_MIN_FLOAT:
+            c = BETA_CONTINUED_FRACTION_MIN_FLOAT
+        d = 1.0 / d
+        delta = d * c
+        h *= delta
+        if abs(delta - 1.0) < BETA_CONTINUED_FRACTION_EPSILON:
+            return h
+    fail_runtime("regularized incomplete beta calculation did not converge.")
+
+
+def regularized_incomplete_beta(x, a, b):
+    if a <= 0.0 or b <= 0.0:
+        fail_runtime("regularized incomplete beta parameters must be positive.")
+    if x < 0.0 or x > 1.0:
+        fail_runtime("regularized incomplete beta x must be in [0, 1].")
+    if x == 0.0:
+        return 0.0
+    if x == 1.0:
+        return 1.0
+
+    log_front = (
+        math.lgamma(a + b)
+        - math.lgamma(a)
+        - math.lgamma(b)
+        + (a * math.log(x))
+        + (b * math.log1p(-x))
+    )
+    front = math.exp(log_front)
+    if x < (a + 1.0) / (a + b + 2.0):
+        return front * beta_continued_fraction(a, b, x) / a
+    return 1.0 - (front * beta_continued_fraction(b, a, 1.0 - x) / b)
+
+
+def clamp_probability(value, label):
+    if not math.isfinite(value):
+        fail_runtime(f"{label} produced a non-finite result.")
+    if value < 0.0 and value > -1e-15:
+        return 0.0
+    if value > 1.0 and value < 1.0 + 1e-15:
+        return 1.0
+    if value < 0.0 or value > 1.0:
+        fail_runtime(f"{label} produced a value outside [0, 1].")
+    return value
+
+
+def two_sided_student_t_p_value(test_statistic, degrees_of_freedom):
+    if degrees_of_freedom <= 0.0 or not math.isfinite(degrees_of_freedom):
+        fail_runtime("Welch-Satterthwaite degrees of freedom must be positive and finite.")
+    t_abs = abs(test_statistic)
+    if t_abs == 0.0:
+        return 1.0
+    x = degrees_of_freedom / (degrees_of_freedom + (t_abs * t_abs))
+    return clamp_probability(
+        regularized_incomplete_beta(x, degrees_of_freedom / 2.0, 0.5),
+        "Student t p-value calculation",
+    )
+
+
+def f_distribution_survival(f_statistic, df_between, df_within):
+    if f_statistic < 0.0 or not math.isfinite(f_statistic):
+        fail_runtime("ANOVA F statistic must be finite and nonnegative.")
+    if df_between <= 0.0 or df_within <= 0.0:
+        fail_runtime("ANOVA degrees of freedom must be positive.")
+    if f_statistic == 0.0:
+        return 1.0
+    x = df_within / (df_within + (df_between * f_statistic))
+    return clamp_probability(
+        regularized_incomplete_beta(x, df_within / 2.0, df_between / 2.0),
+        "F-distribution survival calculation",
+    )
+
+
+def compute_group_stats(values_by_group):
+    stats = {}
+    for group, values in values_by_group.items():
+        mean = sample_mean(values)
+        stats[group] = {
+            "n": len(values),
+            "mean": mean,
+            "variance": sample_variance(values, mean),
+            "values": values,
+        }
+    return stats
+
+
+def compute_anova(values_by_group, group_stats):
+    all_values = [
+        value for group in values_by_group.values() for value in group
+    ]
+    total_n = len(all_values)
+    group_count = len(values_by_group)
+    grand_mean = sample_mean(all_values)
+    ss_between = 0.0
+    ss_within = 0.0
+    for group, stats in group_stats.items():
+        ss_between += stats["n"] * ((stats["mean"] - grand_mean) ** 2)
+        ss_within += sum(
+            (value - stats["mean"]) ** 2 for value in values_by_group[group]
+        )
+    df_between = group_count - 1
+    df_within = total_n - group_count
+    if df_within <= 0:
+        reject_submission("ANOVA requires positive within-group degrees of freedom.")
+    if not math.isfinite(ss_within) or ss_within <= 0.0:
+        reject_submission(
+            "One-way ANOVA requires positive within-group residual variance."
+        )
+    ms_between = ss_between / df_between
+    ms_within = ss_within / df_within
+    f_statistic = ms_between / ms_within
+    p_value = f_distribution_survival(f_statistic, df_between, df_within)
+    if not math.isfinite(p_value):
+        reject_submission("One-way ANOVA produced a non-finite p-value.")
+    return {
+        "grand_mean": grand_mean,
+        "ss_between": ss_between,
+        "ss_within": ss_within,
+        "df_between": df_between,
+        "df_within": df_within,
+        "ms_between": ms_between,
+        "ms_within": ms_within,
+        "f_statistic": f_statistic,
+        "p_value": p_value,
+    }
+
+
+def compute_welch_contrast(contrast, group_stats):
+    treatment_stats = group_stats[contrast["treatment_group"]]
+    control_stats = group_stats[contrast["control_group"]]
+    treatment_term = treatment_stats["variance"] / treatment_stats["n"]
+    control_term = control_stats["variance"] / control_stats["n"]
+    standard_error_squared = treatment_term + control_term
+    if not math.isfinite(standard_error_squared) or standard_error_squared <= 0.0:
+        reject_submission(
+            f"planned contrast {contrast['id']} requires positive within-group variance; observed zero pooled standard error."
+        )
+    effect = treatment_stats["mean"] - control_stats["mean"]
+    test_statistic = effect / math.sqrt(standard_error_squared)
+    denominator = 0.0
+    if treatment_term > 0.0:
+        denominator += (treatment_term * treatment_term) / (treatment_stats["n"] - 1)
+    if control_term > 0.0:
+        denominator += (control_term * control_term) / (control_stats["n"] - 1)
+    if denominator <= 0.0 or not math.isfinite(denominator):
+        reject_submission(
+            f"planned contrast {contrast['id']} requires positive variance in at least one group."
+        )
+    degrees_of_freedom = (standard_error_squared * standard_error_squared) / denominator
+    raw_p_value = two_sided_student_t_p_value(test_statistic, degrees_of_freedom)
+    return {
+        "id": contrast["id"],
+        "treatment_group": contrast["treatment_group"],
+        "control_group": contrast["control_group"],
+        "expected_direction": contrast["expected_direction"],
+        "treatment_mean": treatment_stats["mean"],
+        "control_mean": control_stats["mean"],
+        "n_treatment": treatment_stats["n"],
+        "n_control": control_stats["n"],
+        "effect": effect,
+        "test_statistic": test_statistic,
+        "degrees_of_freedom": degrees_of_freedom,
+        "raw_p_value": raw_p_value,
+        "direction_matched": direction_matches(effect, contrast["expected_direction"]),
+    }
+
+
+def direction_matches(effect, expected_direction):
+    if expected_direction == "treatment_lt_control":
+        return effect < 0.0
+    return effect > 0.0
+
+
+def adjust_p_values(raw_p_values, method):
+    count = len(raw_p_values)
+    if method == "bonferroni":
+        return [min(p_value * count, 1.0) for p_value in raw_p_values]
+    if method == "holm":
+        indexed = sorted(enumerate(raw_p_values), key=lambda item: item[1])
+        adjusted = [0.0] * count
+        running_max = 0.0
+        for rank, (original_index, p_value) in enumerate(indexed):
+            candidate = min((count - rank) * p_value, 1.0)
+            running_max = max(running_max, candidate)
+            adjusted[original_index] = running_max
+        return adjusted
+    fail_runtime(f"Unsupported p_adjustment {method}.")
+
+
+def compute_contrasts(params, group_stats):
+    contrast_results = [
+        compute_welch_contrast(contrast, group_stats)
+        for contrast in params["contrasts"]
+    ]
+    adjusted_p_values = adjust_p_values(
+        [result["raw_p_value"] for result in contrast_results],
+        params["p_adjustment"],
+    )
+    for result, adjusted_p_value in zip(contrast_results, adjusted_p_values):
+        result["adjusted_p_value"] = adjusted_p_value
+        result["passed"] = (
+            adjusted_p_value < params["alpha"] and result["direction_matched"]
+        )
+    return contrast_results
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    try:
+        config = load_json_file(config_path, label="compiled_config")
+    except RuntimeError as error:
+        fail_runtime(str(error))
+    if not isinstance(config, dict):
+        fail_runtime("compiled_config must be a JSON object.")
+
+    submission_role = require_string(
+        config.get("submission_role"),
+        "compiled_config.submission_role",
+    )
+    final_score_key = require_string(
+        config.get("final_score_key"),
+        "compiled_config.final_score_key",
+    )
+    objective = require_string(
+        runtime_context.get("objective"),
+        "runtime_context.objective",
+    )
+    if objective != "maximize":
+        fail_runtime("multi_group_endpoint_test@1 requires objective=maximize.")
+
+    params = load_endpoint_test_config(config)
+    require_observations_slot(runtime_context, submission_role, params)
+    observations_path = resolve_submission_artifact(runtime_context, submission_role)
+    values_by_group = load_observations(
+        observations_path,
+        submission_role,
+        params,
+    )
+    require_group_sizes(values_by_group, params["minimum_n_per_group"])
+
+    group_stats = compute_group_stats(values_by_group)
+    anova = compute_anova(values_by_group, group_stats)
+    contrast_results = compute_contrasts(params, group_stats)
+    passed_contrast_count = sum(1 for result in contrast_results if result["passed"])
+    total_contrast_count = len(contrast_results)
+    minimum_group_n = min(stats["n"] for stats in group_stats.values())
+    maximum_adjusted_p_value = max(
+        result["adjusted_p_value"] for result in contrast_results
+    )
+    omnibus_passed = anova["p_value"] < params["alpha"]
+    all_contrasts_passed = passed_contrast_count == total_contrast_count
+    score = 1.0 if omnibus_passed and all_contrasts_passed else 0.0
+    details = {
+        final_score_key: score,
+        "score": score,
+        "omnibus_p_value": anova["p_value"],
+        "omnibus_f_statistic": anova["f_statistic"],
+        "df_between": anova["df_between"],
+        "df_within": anova["df_within"],
+        "passed_contrast_count": passed_contrast_count,
+        "total_contrast_count": total_contrast_count,
+        "minimum_group_n": minimum_group_n,
+        "maximum_adjusted_p_value": maximum_adjusted_p_value,
+        "omnibus_passed": omnibus_passed,
+        "alpha": params["alpha"],
+        "minimum_n_per_group": params["minimum_n_per_group"],
+        "p_adjustment": params["p_adjustment"],
+        "group_column": params["group_column"],
+        "value_column": params["value_column"],
+        "included_groups": params["included_groups"],
+        "group_stats": {
+            group: {
+                "n": stats["n"],
+                "mean": stats["mean"],
+                "variance": stats["variance"],
+            }
+            for group, stats in group_stats.items()
+        },
+        "anova": anova,
+        "contrasts": contrast_results,
+    }
+    write_score(score=score, details=details)
+
+
+if __name__ == "__main__":
+    main()