@moleculeagora/cli 0.1.0

package/dist/python-v1/exact-match.py

@@ -0,0 +1,60 @@
+from pathlib import Path
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_runtime_context,
+    resolve_evaluation_artifact,
+    resolve_scoring_asset,
+    resolve_submission_artifact,
+    write_score,
+)
+
+
+def require_string(config, key):
+    value = config.get(key)
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"compiled_config.{key} must be a non-empty string.")
+    return value.strip()
+
+
+def read_bytes(path: Path, label: str) -> bytes:
+    try:
+        return path.read_bytes()
+    except FileNotFoundError:
+        fail_runtime(f"Missing {label} at {path}.")
+    except OSError as error:
+        fail_runtime(f"Unable to read {label}: {error}.")
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(
+        runtime_context,
+        "compiled_config",
+        kind="config",
+    )
+    config = load_json_file(config_path, label="compiled_config")
+    evaluation_role = require_string(config, "evaluation_role")
+    submission_role = require_string(config, "submission_role")
+    score_key = require_string(config, "final_score_key")
+    evaluation_path = resolve_evaluation_artifact(runtime_context, evaluation_role)
+    submission_path = resolve_submission_artifact(runtime_context, submission_role)
+    matched = read_bytes(
+        evaluation_path,
+        f"evaluation artifact {evaluation_role}",
+    ) == read_bytes(
+        submission_path,
+        f"submission artifact {submission_role}",
+    )
+    score = 1.0 if matched else 0.0
+    write_score(
+        score=score,
+        details={
+            score_key: score,
+        },
+    )
+
+
+if __name__ == "__main__":
+    main()

package/dist/python-v1/l4-composition.py

@@ -0,0 +1,435 @@
+import json
+import math
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+from agora_runtime import (
+    fail_runtime,
+    load_json_file,
+    load_score_output,
+    load_runtime_context,
+    reject_submission,
+    resolve_scoring_asset,
+    write_score,
+)
+
+INPUT_ROOT = Path(os.environ.get("AGORA_RUNTIME_INPUT_ROOT", "/input"))
+
+
+def require_string(value, label):
+    if not isinstance(value, str) or not value.strip():
+        fail_runtime(f"{label} must be a non-empty string.")
+    return value.strip()
+
+
+def require_number(value, label):
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        fail_runtime(f"{label} must be a number.")
+    return float(value)
+
+
+def require_positive_integer(value, label):
+    if isinstance(value, bool) or not isinstance(value, int) or value < 1:
+        fail_runtime(f"{label} must be a positive integer.")
+    return value
+
+
+def validate_unit_score(value, label):
+    if not math.isfinite(value) or value < 0 or value > 1:
+        fail_runtime(f"{label} must be a finite number in [0, 1].")
+    return value
+
+
+def require_object(value, label):
+    if not isinstance(value, dict):
+        fail_runtime(f"{label} must be an object.")
+    return value
+
+
+def require_list(value, label):
+    if not isinstance(value, list):
+        fail_runtime(f"{label} must be an array.")
+    return value
+
+
+def write_json(path, value):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(value, sort_keys=True, indent=2), encoding="utf-8")
+
+
+def symlink_or_copy(source, target):
+    target.parent.mkdir(parents=True, exist_ok=True)
+    if not source.exists():
+        target.mkdir(parents=True, exist_ok=True)
+        return
+    try:
+        target.symlink_to(source, target_is_directory=source.is_dir())
+    except OSError:
+        if source.is_dir():
+            shutil.copytree(source, target, dirs_exist_ok=True)
+        else:
+            shutil.copy2(source, target)
+
+
+def asset_file_metadata(path):
+    data = path.read_bytes()
+    import hashlib
+
+    return {
+        "size_bytes": len(data),
+        "sha256": hashlib.sha256(data).hexdigest(),
+    }
+
+
+def resolve_profile_determinism_env(runtime_context):
+    profile = require_object(runtime_context.get("runtime_profile"), "runtime_profile")
+    determinism_env = require_object(
+        profile.get("determinism_env"),
+        "runtime_profile.determinism_env",
+    )
+    resolved = {}
+    for name, value in determinism_env.items():
+        resolved[require_string(name, "runtime_profile.determinism_env key")] = require_string(
+            value,
+            f"runtime_profile.determinism_env.{name}",
+        )
+    return resolved
+
+
+def assert_deterministic_child_env(env, determinism_env):
+    for name, expected in determinism_env.items():
+        if env.get(name) != expected:
+            fail_runtime(
+                f"component subprocess env {name} must be {expected!r}."
+            )
+
+
+def build_component_env(runtime_context, temp_root, temp_output):
+    determinism_env = resolve_profile_determinism_env(runtime_context)
+    env = os.environ.copy()
+    sdk_dir = str(temp_root / "scoring_assets" / "python_v1_runtime_sdk")
+    env.update(determinism_env)
+    env["PYTHONPATH"] = os.pathsep.join(
+        [sdk_dir, env.get("PYTHONPATH", "")]
+    ).strip(os.pathsep)
+    env["AGORA_RUNTIME_INPUT_ROOT"] = str(temp_root)
+    env["AGORA_RUNTIME_OUTPUT_ROOT"] = str(temp_output)
+    assert_deterministic_child_env(env, determinism_env)
+    return env
+
+
+def build_component_manifest(runtime_context, component, temp_root, program_path, config_path):
+    sdk_path = resolve_scoring_asset(
+        runtime_context,
+        "python_v1_runtime_sdk",
+        kind="document",
+    )
+    temp_program_path = temp_root / "scoring_assets" / "compiled_program" / "score.py"
+    temp_config_path = temp_root / "scoring_assets" / "compiled_config" / "score-config.json"
+    temp_sdk_path = temp_root / "scoring_assets" / "python_v1_runtime_sdk" / "agora_runtime.py"
+
+    symlink_or_copy(program_path, temp_program_path)
+    symlink_or_copy(config_path, temp_config_path)
+    symlink_or_copy(sdk_path, temp_sdk_path)
+    symlink_or_copy(INPUT_ROOT / "evaluation", temp_root / "evaluation")
+    symlink_or_copy(INPUT_ROOT / "submission", temp_root / "submission")
+
+    program_metadata = asset_file_metadata(temp_program_path)
+    config_metadata = asset_file_metadata(temp_config_path)
+    sdk_metadata = asset_file_metadata(temp_sdk_path)
+
+    return {
+        "kind": "runtime_manifest",
+        "runtime_profile": runtime_context["runtime_profile"],
+        "artifact_contract": runtime_context["artifact_contract"],
+        "evaluation_bindings": runtime_context.get("evaluation_bindings", []),
+        "artifacts": runtime_context["artifacts"],
+        "scoring_assets": [
+            {
+                "role": "compiled_program",
+                "kind": "program",
+                "artifact_id": f"component-program-{component['id']}",
+                "abi_version": "python-v1",
+                "entrypoint": "score.py",
+                "relative_path": "scoring_assets/compiled_program/score.py",
+                "file_name": "score.py",
+                **program_metadata,
+            },
+            {
+                "role": "compiled_config",
+                "kind": "config",
+                "artifact_id": f"component-config-{component['id']}",
+                "relative_path": "scoring_assets/compiled_config/score-config.json",
+                "file_name": "score-config.json",
+                **config_metadata,
+            },
+            {
+                "role": "python_v1_runtime_sdk",
+                "kind": "document",
+                "artifact_id": f"component-sdk-{component['id']}",
+                "relative_path": "scoring_assets/python_v1_runtime_sdk/agora_runtime.py",
+                "file_name": "agora_runtime.py",
+                **sdk_metadata,
+            },
+        ],
+        "objective": component["objective"],
+        "final_score_key": component["final_score_key"],
+        "scorer_result_schema": runtime_context["scorer_result_schema"],
+        "policies": runtime_context["policies"],
+    }
+
+
+def run_component(runtime_context, component):
+    program_role = require_string(component.get("program_role"), f"components.{component.get('id')}.program_role")
+    program_path = resolve_scoring_asset(runtime_context, program_role, kind="document")
+    with tempfile.TemporaryDirectory(prefix=f"agora-l4-{component['id']}-") as temp_dir:
+        temp_root = Path(temp_dir) / "input"
+        temp_output = Path(temp_dir) / "output"
+        temp_output.mkdir(parents=True, exist_ok=True)
+        config_path = temp_root / "component-config.json"
+        write_json(config_path, component["config"])
+        manifest = build_component_manifest(
+            runtime_context,
+            component,
+            temp_root,
+            program_path,
+            config_path,
+        )
+        write_json(temp_root / "runtime-manifest.json", manifest)
+        env = build_component_env(runtime_context, temp_root, temp_output)
+        completed = subprocess.run(
+            [sys.executable, str(temp_root / "scoring_assets" / "compiled_program" / "score.py")],
+            env=env,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+            timeout=300,
+            check=False,
+        )
+        output_path = temp_output / "score.json"
+        if not output_path.exists():
+            fail_runtime(
+                f"component {component['id']} did not write score.json. stderr: {completed.stderr.strip()}"
+            )
+        try:
+            payload = load_score_output(
+                output_path,
+                label=f"component {component['id']} score output",
+            )
+        except RuntimeError as error:
+            fail_runtime(str(error))
+        if completed.returncode != 0:
+            fail_runtime(
+                f"component {component['id']} failed with exit code {completed.returncode}: {payload.get('error') or completed.stderr.strip()}"
+            )
+        if payload.get("ok") is not True:
+            reject_submission(
+                f"component {component['id']} rejected submission: {payload.get('error') or 'component returned ok=false'}",
+                details={"component": component["id"]},
+            )
+        return require_number(payload.get("score"), f"component {component['id']} score")
+
+
+def normalize_components(config):
+    components = []
+    seen = set()
+    for index, value in enumerate(require_list(config.get("components"), "compiled_config.components")):
+        component = require_object(value, f"compiled_config.components[{index}]")
+        component_id = require_string(component.get("id"), f"compiled_config.components[{index}].id")
+        if component_id in seen:
+            fail_runtime(f"compiled_config.components repeats component id {component_id}.")
+        seen.add(component_id)
+        component["id"] = component_id
+        component["objective"] = require_string(component.get("objective"), f"compiled_config.components[{index}].objective")
+        component["final_score_key"] = require_string(component.get("final_score_key"), f"compiled_config.components[{index}].final_score_key")
+        component["config"] = require_object(component.get("config"), f"compiled_config.components[{index}].config")
+        components.append(component)
+    return components
+
+
+def weighted_sum_score(components_by_id, scores, aggregation):
+    terms = require_list(aggregation.get("terms"), "compiled_config.aggregation.terms")
+    weights = {}
+    for index, term_value in enumerate(terms):
+        term = require_object(term_value, f"compiled_config.aggregation.terms[{index}]")
+        component = require_string(term.get("component"), f"compiled_config.aggregation.terms[{index}].component")
+        if component not in components_by_id:
+            fail_runtime(f"compiled_config.aggregation.terms[{index}].component references unknown component {component}.")
+        if component in weights:
+            fail_runtime(f"compiled_config.aggregation.terms repeats component {component}.")
+        weights[component] = require_number(term.get("weight"), f"compiled_config.aggregation.terms[{index}].weight")
+    if set(weights) != set(components_by_id):
+        fail_runtime("compiled_config.aggregation.terms must exactly match compiled_config.components ids.")
+    if bool(aggregation.get("normalize_weights", False)):
+        total_weight = sum(abs(weight) for weight in weights.values())
+        if total_weight == 0:
+            fail_runtime("compiled_config.aggregation.terms weights must not sum to zero when normalize_weights is true.")
+        weights = {component: weight / total_weight for component, weight in weights.items()}
+    return sum(scores[component] * weight for component, weight in weights.items()), weights
+
+
+def threshold_passes(value, comparator, threshold):
+    if comparator == "gte":
+        return value >= threshold
+    if comparator == "gt":
+        return value > threshold
+    if comparator == "lte":
+        return value <= threshold
+    if comparator == "lt":
+        return value < threshold
+    fail_runtime(f"compiled_config.aggregation.comparator={comparator} is unsupported.")
+
+
+def threshold_gate_score(components_by_id, scores, aggregation):
+    score_component = require_string(
+        aggregation.get("score_component"),
+        "compiled_config.aggregation.score_component",
+    )
+    gate_component = require_string(
+        aggregation.get("gate_component"),
+        "compiled_config.aggregation.gate_component",
+    )
+    if score_component not in components_by_id:
+        fail_runtime("compiled_config.aggregation.score_component references an unknown component.")
+    if gate_component not in components_by_id:
+        fail_runtime("compiled_config.aggregation.gate_component references an unknown component.")
+    comparator = require_string(aggregation.get("comparator", "gte"), "compiled_config.aggregation.comparator")
+    threshold = require_number(aggregation.get("threshold"), "compiled_config.aggregation.threshold")
+    fail_score = require_number(aggregation.get("fail_score", 0), "compiled_config.aggregation.fail_score")
+    passed = threshold_passes(scores[gate_component], comparator, threshold)
+    return (scores[score_component] if passed else fail_score), {
+        "kind": "threshold_gate",
+        "score_component": score_component,
+        "gate_component": gate_component,
+        "gate_score": scores[gate_component],
+        "comparator": comparator,
+        "threshold": threshold,
+        "passed": passed,
+    }
+
+
+def k_of_n_score(components_by_id, scores, aggregation):
+    terms = require_list(aggregation.get("terms"), "compiled_config.aggregation.terms")
+    term_components = set()
+    term_details = []
+    for index, term_value in enumerate(terms):
+        term = require_object(term_value, f"compiled_config.aggregation.terms[{index}]")
+        component = require_string(term.get("component"), f"compiled_config.aggregation.terms[{index}].component")
+        if component not in components_by_id:
+            fail_runtime(f"compiled_config.aggregation.terms[{index}].component references unknown component {component}.")
+        if component in term_components:
+            fail_runtime(f"compiled_config.aggregation.terms repeats component {component}.")
+        term_components.add(component)
+        pass_threshold = require_number(
+            term.get("pass_threshold"),
+            f"compiled_config.aggregation.terms[{index}].pass_threshold",
+        )
+        if pass_threshold < 0 or pass_threshold > 1:
+            fail_runtime(
+                f"compiled_config.aggregation.terms[{index}].pass_threshold must be in [0, 1]."
+            )
+        component_score = scores[component]
+        passed = component_score >= pass_threshold
+        term_details.append(
+            {
+                "component": component,
+                "score": component_score,
+                "pass_threshold": pass_threshold,
+                "passed": passed,
+            }
+        )
+    if set(term_components) != set(components_by_id):
+        fail_runtime("compiled_config.aggregation.terms must exactly match compiled_config.components ids.")
+    minimum_pass_count = require_positive_integer(
+        aggregation.get("minimum_pass_count"),
+        "compiled_config.aggregation.minimum_pass_count",
+    )
+    if minimum_pass_count > len(terms):
+        fail_runtime("compiled_config.aggregation.minimum_pass_count must be less than or equal to terms.length.")
+    passed_count = sum(1 for term in term_details if term["passed"])
+    total_count = len(term_details)
+    score = 1.0 if passed_count >= minimum_pass_count else 0.0
+    validate_unit_score(score, "k_of_n final score")
+    return score, {
+        "kind": "k_of_n",
+        "passed_count": passed_count,
+        "required_count": minimum_pass_count,
+        "total_count": total_count,
+        "terms": term_details,
+    }
+
+
+def worst_group_score(components_by_id, scores, aggregation):
+    terms = require_list(aggregation.get("terms"), "compiled_config.aggregation.terms")
+    term_scores = {}
+    for index, term_value in enumerate(terms):
+        term = require_object(term_value, f"compiled_config.aggregation.terms[{index}]")
+        component = require_string(term.get("component"), f"compiled_config.aggregation.terms[{index}].component")
+        if component not in components_by_id:
+            fail_runtime(f"compiled_config.aggregation.terms[{index}].component references unknown component {component}.")
+        if component in term_scores:
+            fail_runtime(f"compiled_config.aggregation.terms repeats component {component}.")
+        term_scores[component] = scores[component]
+    if len(term_scores) < 2:
+        fail_runtime("compiled_config.aggregation.terms must include at least two components.")
+    if set(term_scores) != set(components_by_id):
+        fail_runtime("compiled_config.aggregation.terms must exactly match compiled_config.components ids.")
+    term_details = [
+        {"component": component, "score": term_scores[component]}
+        for component in sorted(term_scores)
+    ]
+    worst_term = min(term_details, key=lambda term: (term["score"], term["component"]))
+    return worst_term["score"], {
+        "kind": "worst_group",
+        "worst_component": worst_term["component"],
+        "terms": term_details,
+    }
+
+
+def main():
+    runtime_context = load_runtime_context()
+    config_path = resolve_scoring_asset(runtime_context, "compiled_config", kind="config")
+    config = load_json_file(config_path, label="compiled_config")
+    mode = require_string(config.get("mode"), "compiled_config.mode")
+    final_score_key = require_string(runtime_context.get("final_score_key"), "runtime_context.final_score_key")
+    if final_score_key != require_string(config.get("final_score_key"), "compiled_config.final_score_key"):
+        fail_runtime("compiled_config.final_score_key must match runtime_context.final_score_key.")
+    components = normalize_components(config)
+    components_by_id = {component["id"]: component for component in components}
+    scores = {
+        component["id"]: run_component(runtime_context, component)
+        for component in components
+    }
+    aggregation = require_object(config.get("aggregation"), "compiled_config.aggregation")
+    aggregation_kind = require_string(aggregation.get("kind"), "compiled_config.aggregation.kind")
+    if mode != aggregation_kind:
+        fail_runtime("compiled_config.mode must match compiled_config.aggregation.kind.")
+    if aggregation_kind == "weighted_sum":
+        score, aggregation_details = weighted_sum_score(components_by_id, scores, aggregation)
+    elif aggregation_kind == "threshold_gate":
+        score, aggregation_details = threshold_gate_score(components_by_id, scores, aggregation)
+    elif aggregation_kind == "k_of_n":
+        score, aggregation_details = k_of_n_score(components_by_id, scores, aggregation)
+    elif aggregation_kind == "worst_group":
+        score, aggregation_details = worst_group_score(components_by_id, scores, aggregation)
+    else:
+        fail_runtime(f"compiled_config.aggregation.kind={aggregation_kind} is unsupported.")
+    details = {
+        final_score_key: score,
+        "aggregation": aggregation_kind,
+        "components": scores,
+        "aggregation_details": aggregation_details,
+    }
+    if aggregation_kind == "k_of_n":
+        details["passed_count"] = aggregation_details["passed_count"]
+        details["required_count"] = aggregation_details["required_count"]
+        details["total_count"] = aggregation_details["total_count"]
+    write_score(score=score, details=details)
+
+
+if __name__ == "__main__":
+    main()