@moleculeagora/cli 0.1.0

package/dist/python-v1/public-tool-metric.py

@@ -0,0 +1,161 @@
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    safe_extract_zip,
+    write_score,
+)
+
+
+def require_string(value, label):
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_int(value, label):
+    if not isinstance(value, int) or value <= 0:
+        fail_runtime(f"{label} must be a positive integer.")
+    return value
+
+
+def load_json_object(path, label):
+    try:
+        data = load_json_file(path, label=label)
+    except RuntimeError as error:
+        fail_runtime(str(error))
+    if not isinstance(data, dict):
+        fail_runtime(f"{label} must be a JSON object.")
+    return data
+
+
+def read_required_text(path, label):
+    try:
+        return path.read_text(encoding="utf-8")
+    except FileNotFoundError:
+        fail_runtime(f"Missing {label} at {path}.")
+    except OSError as error:
+        fail_runtime(f"Unable to read {label}: {error}.")
+
+
+def normalize_output(text, strip_trailing_whitespace):
+    if strip_trailing_whitespace:
+        return text.rstrip()
+    return text
+
+
+def run_python_solution(solution_path, stdin_text, timeout_ms, working_dir):
+    try:
+        return subprocess.run(
+            [sys.executable, str(solution_path)],
+            input=stdin_text,
+            capture_output=True,
+            text=True,
+            timeout=timeout_ms / 1000.0,
+            cwd=str(working_dir),
+            check=False,
+        )
+    except subprocess.TimeoutExpired:
+        return None
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    config = load_json_object(config_path, "compiled_config")
+    tool_role = require_string(config.get("tool_role"), "compiled_config.tool_role")
+    submission_role = require_string(
+        config.get("submission_role"), "compiled_config.submission_role"
+    )
+    final_score_key = require_string(
+        runtime_context.get("final_score_key"),
+        "runtime_context.final_score_key",
+    )
+    tool_bundle_path = resolve_scoring_asset(runtime_context, tool_role, kind="bundle")
+    solution_path = resolve_submission_artifact(runtime_context, submission_role)
+    with tempfile.TemporaryDirectory(prefix="agora-public-tool-") as working_root:
+        working_dir = Path(working_root)
+        safe_extract_zip(
+            tool_bundle_path,
+            working_dir,
+            label=f"scoring asset {tool_role}",
+        )
+        manifest_path = working_dir / "agora-harness.json"
+        harness_manifest = load_json_object(manifest_path, "agora-harness.json")
+        version = require_string(harness_manifest.get("version"), "agora-harness.version")
+        language = require_string(harness_manifest.get("language"), "agora-harness.language")
+        if version != "v1":
+            fail_runtime(f"Unsupported harness version {version}.")
+        if language != "python":
+            fail_runtime(f"Unsupported harness language {language}.")
+        timeout_ms = require_int(
+            harness_manifest.get("timeout_ms", 30000),
+            "agora-harness.timeout_ms",
+        )
+        strip_trailing_whitespace = bool(
+            harness_manifest.get("strip_trailing_whitespace", False)
+        )
+        tests = harness_manifest.get("tests")
+        if not isinstance(tests, list) or len(tests) == 0:
+            fail_runtime("agora-harness.tests must declare at least one test case.")
+        passed_tests = 0
+        for index, test_case in enumerate(tests):
+            if not isinstance(test_case, dict):
+                fail_runtime(f"agora-harness.tests[{index}] must be an object.")
+            stdin_path = working_dir / require_string(
+                test_case.get("stdin_path"),
+                f"agora-harness.tests[{index}].stdin_path",
+            )
+            expected_stdout_path = working_dir / require_string(
+                test_case.get("expected_stdout_path"),
+                f"agora-harness.tests[{index}].expected_stdout_path",
+            )
+            stdin_text = read_required_text(
+                stdin_path,
+                f"test stdin {stdin_path.name}",
+            )
+            expected_stdout = normalize_output(
+                read_required_text(
+                    expected_stdout_path,
+                    f"test expected stdout {expected_stdout_path.name}",
+                ),
+                strip_trailing_whitespace,
+            )
+            run = run_python_solution(
+                solution_path,
+                stdin_text,
+                timeout_ms,
+                working_dir,
+            )
+            if run is None:
+                continue
+            if run.returncode != 0:
+                continue
+            actual_stdout = normalize_output(run.stdout, strip_trailing_whitespace)
+            if actual_stdout == expected_stdout:
+                passed_tests += 1
+        total_tests = len(tests)
+        score = passed_tests / total_tests
+        write_score(
+            score=score,
+            details={
+                final_score_key: score,
+                "tests_passed": passed_tests,
+                "total_tests": total_tests,
+            },
+        )
+
+
+if __name__ == "__main__":
+    main()

package/dist/python-v1/ranking-metric.py

@@ -0,0 +1,426 @@
+import csv
+import math
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    reject_submission,
+    resolve_evaluation_artifact,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    write_score,
+)
+
+SUPPORTED_METRICS = ("ndcg", "mrr", "map", "top_k_recall")
+SUPPORTED_METRIC_SET = set(SUPPORTED_METRICS)
+
+
+def format_metric_list(metrics):
+    ordered = list(metrics)
+    if len(ordered) == 1:
+        return ordered[0]
+    if len(ordered) == 2:
+        return f"{ordered[0]} or {ordered[1]}"
+    return f"{', '.join(ordered[:-1])}, or {ordered[-1]}"
+
+
+def require_string(value, label):
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_policy(policies, key, allowed):
+    value = require_string(policies.get(key), f"policies.{key}")
+    if value not in allowed:
+        fail_runtime(
+            f"policies.{key} must be one of {', '.join(sorted(allowed))}."
+        )
+    return value
+
+
+def require_metric(config):
+    metric = require_string(config.get("metric"), "compiled_config.metric").lower()
+    if metric not in SUPPORTED_METRIC_SET:
+        fail_runtime(
+            f"compiled_config.metric must be one of {format_metric_list(SUPPORTED_METRICS)}."
+        )
+    return metric
+
+
+def require_metric_params(config, metric):
+    metric_params = config.get("metric_params")
+    if not isinstance(metric_params, dict):
+        fail_runtime("compiled_config.metric_params must be an object.")
+    if metric_params.get("metric") != metric:
+        fail_runtime("compiled_config.metric_params.metric must match compiled_config.metric.")
+    return metric_params
+
+
+def require_top_k_fraction(metric_params):
+    value = metric_params.get("k")
+    if (
+        isinstance(value, bool)
+        or not isinstance(value, (int, float))
+        or not math.isfinite(float(value))
+    ):
+        fail_runtime("compiled_config.metric_params.k must be a finite number.")
+    fraction = float(value)
+    if fraction <= 0.0 or fraction > 1.0:
+        fail_runtime("compiled_config.metric_params.k must be greater than 0 and at most 1.")
+    return fraction
+
+
+def find_slot(runtime_context, lane, role):
+    slot_key = f"{lane}_slots"
+    slots = runtime_context.get(slot_key)
+    if not isinstance(slots, list):
+        fail_runtime(f"Runtime context is missing {slot_key}.")
+    for slot in slots:
+        if isinstance(slot, dict) and slot.get("role") == role:
+            return slot
+    fail_runtime(f"Runtime context is missing {lane} slot for role {role}.")
+
+
+def require_csv_slot_columns(runtime_context, lane, role):
+    slot = find_slot(runtime_context, lane, role)
+    validator = slot.get("validator")
+    if not isinstance(validator, dict) or validator.get("kind") != "csv_columns":
+        fail_runtime(
+            f"{lane} role {role} must use validator.kind=csv_columns for ranking_metric."
+        )
+    record_key = require_string(
+        validator.get("record_key"),
+        f"{lane}.{role}.validator.record_key",
+    )
+    value_field = require_string(
+        validator.get("value_field"),
+        f"{lane}.{role}.validator.value_field",
+    )
+    return record_key, value_field
+
+
+def read_csv_rows(path, label, *, invalid_handler):
+    try:
+        with path.open("r", encoding="utf-8", newline="") as handle:
+            reader = csv.DictReader(handle)
+            fieldnames = reader.fieldnames
+            if not fieldnames:
+                invalid_handler(f"{label} must include a CSV header row.")
+            normalized_fieldnames = []
+            for fieldname in fieldnames:
+                if not isinstance(fieldname, str) or not fieldname.strip():
+                    invalid_handler(f"{label} contains an empty CSV column name.")
+                normalized_fieldnames.append(fieldname.strip())
+            rows = list(reader)
+    except FileNotFoundError:
+        invalid_handler(f"Missing {label} at {path}.")
+    except OSError as error:
+        invalid_handler(f"Unable to read {label}: {error}.")
+    return normalized_fieldnames, rows
+
+
+def parse_reference_relevance(raw_value, label):
+    text = raw_value.strip() if isinstance(raw_value, str) else ""
+    if not text:
+        fail_runtime(f"{label} is blank.")
+    try:
+        value = float(text)
+    except ValueError:
+        fail_runtime(f"{label} must be numeric, received {text!r}.")
+    if not math.isfinite(value):
+        fail_runtime(f"{label} must be finite.")
+    if value < 0:
+        fail_runtime(f"{label} must be non-negative for ranking metrics.")
+    return value
+
+
+def parse_submission_score(raw_value, label, invalid_value_policy):
+    text = raw_value.strip() if isinstance(raw_value, str) else ""
+    if not text:
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} is blank.")
+        return None
+    try:
+        value = float(text)
+    except ValueError:
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} must be numeric, received {text!r}.")
+        return None
+    if not math.isfinite(value):
+        if invalid_value_policy == "reject":
+            reject_submission(f"{label} must be finite.")
+        return None
+    return value
+
+
+def load_reference_relevance(path, role, record_key, value_field):
+    fieldnames, rows = read_csv_rows(
+        path,
+        f"evaluation artifact {role}",
+        invalid_handler=fail_runtime,
+    )
+    if record_key not in fieldnames:
+        fail_runtime(
+            f"evaluation artifact {role} is missing record key column {record_key}."
+        )
+    if value_field not in fieldnames:
+        fail_runtime(
+            f"evaluation artifact {role} is missing value column {value_field}."
+        )
+    values = {}
+    for row_index, row in enumerate(rows, start=2):
+        raw_key = row.get(record_key)
+        key = raw_key.strip() if isinstance(raw_key, str) else ""
+        if not key:
+            fail_runtime(
+                f"evaluation artifact {role} row {row_index} is missing {record_key}."
+            )
+        if key in values:
+            fail_runtime(
+                f"evaluation artifact {role} contains duplicate record id {key!r}."
+            )
+        values[key] = parse_reference_relevance(
+            row.get(value_field),
+            f"evaluation artifact {role} row {row_index} column {value_field}",
+        )
+    if not values:
+        fail_runtime(
+            f"evaluation artifact {role} must contain at least one ranked row."
+        )
+    return values
+
+
+def load_submission_scores(
+    path,
+    role,
+    record_key,
+    value_field,
+    duplicate_id_policy,
+    invalid_value_policy,
+):
+    fieldnames, rows = read_csv_rows(
+        path,
+        f"submission artifact {role}",
+        invalid_handler=reject_submission,
+    )
+    if record_key not in fieldnames:
+        reject_submission(
+            f"submission artifact {role} is missing record key column {record_key}."
+        )
+    if value_field not in fieldnames:
+        reject_submission(
+            f"submission artifact {role} is missing value column {value_field}."
+        )
+    values = {}
+    for row_index, row in enumerate(rows, start=2):
+        raw_key = row.get(record_key)
+        key = raw_key.strip() if isinstance(raw_key, str) else ""
+        if not key:
+            if invalid_value_policy == "reject":
+                reject_submission(
+                    f"submission artifact {role} row {row_index} is missing {record_key}."
+                )
+            continue
+        if key in values:
+            if duplicate_id_policy == "reject":
+                reject_submission(
+                    f"submission artifact {role} contains duplicate record id {key!r}."
+                )
+            continue
+        parsed_value = parse_submission_score(
+            row.get(value_field),
+            f"submission artifact {role} row {row_index} column {value_field}",
+            invalid_value_policy,
+        )
+        if parsed_value is None:
+            continue
+        values[key] = parsed_value
+    return values
+
+
+def discount(rank_index):
+    return 1.0 / math.log2(rank_index + 2.0)
+
+
+def dcg(labels):
+    return sum(((2.0 ** label) - 1.0) * discount(index) for index, label in enumerate(labels))
+
+
+def rank_submission_ids(submission_by_id):
+    return sorted(
+        submission_by_id.keys(),
+        key=lambda record_id: (-submission_by_id[record_id], record_id),
+    )
+
+
+def compute_ndcg(reference_by_id, submission_by_id):
+    ranked_ids = rank_submission_ids(submission_by_id)
+    ranked_labels = [reference_by_id[record_id] for record_id in ranked_ids]
+    ideal_labels = sorted(ranked_labels, reverse=True)
+    ideal_dcg = dcg(ideal_labels)
+    if ideal_dcg == 0.0:
+        return 1.0
+    return dcg(ranked_labels) / ideal_dcg
+
+
+def compute_mrr(reference_by_id, submission_by_id):
+    ranked_ids = rank_submission_ids(submission_by_id)
+    for index, record_id in enumerate(ranked_ids, start=1):
+        if reference_by_id[record_id] > 0:
+            return 1.0 / index
+    return 0.0
+
+
+def compute_map(reference_by_id, submission_by_id):
+    ranked_ids = rank_submission_ids(submission_by_id)
+    hits = 0
+    precision_sum = 0.0
+    for index, record_id in enumerate(ranked_ids, start=1):
+        if reference_by_id[record_id] <= 0:
+            continue
+        hits += 1
+        precision_sum += hits / index
+    if hits == 0:
+        return 0.0
+    return precision_sum / hits
+
+
+def compute_top_k_recall(reference_by_id, submission_by_id, metric_params):
+    relevant_ids = [
+        record_id for record_id, relevance in reference_by_id.items() if relevance > 0
+    ]
+    if not relevant_ids:
+        fail_runtime(
+            "ranking_metric metric top_k_recall requires at least one positive reference label."
+        )
+    ranked_ids = rank_submission_ids(submission_by_id)
+    cutoff = max(1, math.ceil(len(ranked_ids) * require_top_k_fraction(metric_params)))
+    selected_ids = set(ranked_ids[:cutoff])
+    recalled = sum(1 for record_id in relevant_ids if record_id in selected_ids)
+    return recalled / len(relevant_ids)
+
+
+def compute_metric(metric, reference_by_id, submission_by_id, metric_params):
+    if metric == "ndcg":
+        return compute_ndcg(reference_by_id, submission_by_id)
+    if metric == "mrr":
+        return compute_mrr(reference_by_id, submission_by_id)
+    if metric == "map":
+        return compute_map(reference_by_id, submission_by_id)
+    return compute_top_k_recall(reference_by_id, submission_by_id, metric_params)
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    try:
+        config = load_json_file(config_path, label="compiled_config")
+    except RuntimeError as error:
+        fail_runtime(str(error))
+    metric = require_metric(config)
+    metric_params = require_metric_params(config, metric)
+    evaluation_role = require_string(
+        config.get("evaluation_role"),
+        "compiled_config.evaluation_role",
+    )
+    submission_role = require_string(
+        config.get("submission_role"),
+        "compiled_config.submission_role",
+    )
+    objective = require_string(
+        runtime_context.get("objective"),
+        "runtime_context.objective",
+    )
+    if objective != "maximize":
+        fail_runtime(f"ranking_metric metric {metric} requires objective=maximize.")
+    final_score_key = require_string(
+        runtime_context.get("final_score_key"),
+        "runtime_context.final_score_key",
+    )
+    policies = runtime_context.get("policies")
+    if not isinstance(policies, dict):
+        fail_runtime("Runtime context is missing execution policies.")
+    coverage_policy = require_policy(
+        policies,
+        "coverage_policy",
+        {"reject", "ignore", "penalize"},
+    )
+    duplicate_id_policy = require_policy(
+        policies,
+        "duplicate_id_policy",
+        {"reject", "ignore"},
+    )
+    invalid_value_policy = require_policy(
+        policies,
+        "invalid_value_policy",
+        {"reject", "ignore"},
+    )
+    evaluation_record_key, evaluation_value_field = require_csv_slot_columns(
+        runtime_context,
+        "evaluation",
+        evaluation_role,
+    )
+    submission_record_key, submission_value_field = require_csv_slot_columns(
+        runtime_context,
+        "submission",
+        submission_role,
+    )
+    evaluation_path = resolve_evaluation_artifact(runtime_context, evaluation_role)
+    submission_path = resolve_submission_artifact(runtime_context, submission_role)
+    reference_by_id = load_reference_relevance(
+        evaluation_path,
+        evaluation_role,
+        evaluation_record_key,
+        evaluation_value_field,
+    )
+    submission_by_id = load_submission_scores(
+        submission_path,
+        submission_role,
+        submission_record_key,
+        submission_value_field,
+        duplicate_id_policy,
+        invalid_value_policy,
+    )
+    missing_ids = [
+        record_id
+        for record_id in reference_by_id
+        if record_id not in submission_by_id
+    ]
+    if missing_ids and coverage_policy == "reject":
+        reject_submission(
+            f"Submission is missing rankings for {len(missing_ids)} required rows; first missing id is {missing_ids[0]!r}."
+        )
+    scored_ids = [
+        record_id
+        for record_id in reference_by_id
+        if record_id in submission_by_id
+    ]
+    if not scored_ids:
+        reject_submission(
+            "Submission produced no ranked rows after applying runtime policies."
+        )
+    scored_reference = {record_id: reference_by_id[record_id] for record_id in scored_ids}
+    scored_submission = {record_id: submission_by_id[record_id] for record_id in scored_ids}
+    raw_metric = compute_metric(metric, scored_reference, scored_submission, metric_params)
+    normalized_score = max(0.0, min(raw_metric, 1.0))
+    if coverage_policy == "penalize":
+        normalized_score *= len(scored_ids) / len(reference_by_id)
+    write_score(
+        score=normalized_score,
+        details={
+            final_score_key: normalized_score,
+            "selected_metric": metric,
+            "selected_metric_value": raw_metric,
+            "ranked_items": len(scored_ids),
+        },
+    )
+
+
+if __name__ == "__main__":
+    main()