@rpamis/comet 0.4.0-beta.3 → 0.4.0-beta.4

package/eval/scaffold/python/manifests.py

@@ -0,0 +1,129 @@
+"""Eval manifest parser for generated Skill packages."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+import yaml
+
+from scaffold.python.tasks import InteractionConfig
+
+
+SHA256_HEX_RE = re.compile(r"^[a-f0-9]{64}$")
+
+
+@dataclass(frozen=True)
+class SkillEvalManifest:
+    path: Path
+    name: str
+    description: str
+    skill_name: str
+    skill_path: Path
+    profile: str | None = None
+    draft_hash: str | None = None
+    recommended_tasks: list[str] = field(default_factory=list)
+    baseline_treatments: list[str] = field(default_factory=list)
+    quality_gates: dict = field(default_factory=dict)
+    required_output_schemas: list[str] = field(default_factory=list)
+    expected_evidence: list[dict] = field(default_factory=list)
+    required_skills: list[str] = field(default_factory=list)
+    expected_artifacts: list[str] = field(default_factory=list)
+    generated_node_skills: list[str] = field(default_factory=list)
+    route_conformance_task: str | None = None
+    route_conformance_expected_node_order: list[str] = field(default_factory=list)
+    interaction: InteractionConfig = field(default_factory=InteractionConfig)
+
+
+def _require_mapping(data: dict, field_name: str) -> dict:
+    value = data.get(field_name)
+    if not isinstance(value, dict):
+        raise ValueError(f"Missing mapping field: {field_name}")
+    return value
+
+
+def _optional_string_list(data: dict, camel_name: str, snake_name: str) -> list[str]:
+    value = data.get(camel_name, data.get(snake_name))
+    if value is None:
+        return []
+    if not isinstance(value, list) or not all(isinstance(item, str) for item in value):
+        raise ValueError(f"Expected evaluation.{camel_name} to be a list of strings")
+    return list(value)
+
+
+def _optional_mapping(data: dict, camel_name: str, snake_name: str) -> dict:
+    value = data.get(camel_name, data.get(snake_name))
+    if value is None:
+        return {}
+    if not isinstance(value, dict):
+        raise ValueError(f"Expected evaluation.{camel_name} to be a mapping")
+    return dict(value)
+
+
+def _optional_dict_list(data: dict, camel_name: str, snake_name: str) -> list[dict]:
+    value = data.get(camel_name, data.get(snake_name))
+    if value is None:
+        return []
+    if not isinstance(value, list) or not all(isinstance(item, dict) for item in value):
+        raise ValueError(f"Expected evaluation.{camel_name} to be a list of mappings")
+    return list(value)
+
+
+def _optional_draft_hash(metadata: dict) -> str | None:
+    value = metadata.get("draftHash") or metadata.get("draft_hash")
+    if value is None:
+        return None
+    if not isinstance(value, str) or not SHA256_HEX_RE.match(value):
+        raise ValueError("Expected metadata.draftHash to be 64 lowercase hex characters")
+    return value
+
+
+def load_eval_manifest(path: Path | str) -> SkillEvalManifest:
+    manifest_path = Path(path).expanduser().resolve()
+    data = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
+    if data.get("apiVersion") != "comet.eval/v1alpha1":
+        raise ValueError("Expected apiVersion comet.eval/v1alpha1")
+    if data.get("kind") != "SkillEvalManifest":
+        raise ValueError("Expected kind SkillEvalManifest")
+
+    metadata = _require_mapping(data, "metadata")
+    skill = _require_mapping(data, "skill")
+    evaluation = data.get("evaluation") or {}
+    interaction_data = data.get("interaction") or {}
+    skill_source = skill.get("source", "..")
+    route_conformance = evaluation.get("routeConformance") or {}
+
+    return SkillEvalManifest(
+        path=manifest_path,
+        name=str(metadata.get("name") or skill.get("name")),
+        description=str(metadata.get("description") or ""),
+        skill_name=str(skill.get("name") or metadata.get("name")),
+        skill_path=(manifest_path.parent / skill_source).resolve(),
+        profile=skill.get("profile"),
+        draft_hash=_optional_draft_hash(metadata),
+        recommended_tasks=list(evaluation.get("recommendedTasks") or []),
+        baseline_treatments=_optional_string_list(
+            evaluation, "baselineTreatments", "baseline_treatments"
+        ),
+        quality_gates=_optional_mapping(evaluation, "qualityGates", "quality_gates"),
+        required_output_schemas=_optional_string_list(
+            evaluation, "requiredOutputSchemas", "required_output_schemas"
+        ),
+        expected_evidence=_optional_dict_list(
+            evaluation, "expectedEvidence", "expected_evidence"
+        ),
+        required_skills=list(evaluation.get("requiredSkills") or []),
+        expected_artifacts=list(evaluation.get("expectedArtifacts") or []),
+        generated_node_skills=list(evaluation.get("generatedNodeSkills") or []),
+        route_conformance_task=route_conformance.get("task"),
+        route_conformance_expected_node_order=list(
+            route_conformance.get("expectedNodeOrder") or []
+        ),
+        interaction=InteractionConfig(
+            mode=interaction_data.get("mode", "none"),
+            max_turns=int(interaction_data.get("maxTurns", interaction_data.get("max_turns", 12))),
+            simulator_prompt=interaction_data.get("simulatorPrompt")
+            or interaction_data.get("simulator_prompt"),
+        ),
+    )

package/eval/scaffold/python/paper_charts.py

@@ -0,0 +1,25 @@
+"""Python chart backend for paper-style eval report figures."""
+
+from __future__ import annotations
+
+import json
+import sys
+from typing import Any
+
+from scaffold.python.report_outputs import _render_paper_figures_inline
+
+
+def render_from_payload(payload: dict[str, Any]) -> str:
+    return _render_paper_figures_inline(payload, backend="python")
+
+
+def main() -> int:
+    payload = json.loads(sys.stdin.read() or "{}")
+    if not isinstance(payload, dict):
+        raise ValueError("chart payload must be an object")
+    sys.stdout.write(render_from_payload(payload))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/eval/scaffold/python/pass_at_k.py

@@ -0,0 +1,107 @@
+"""pass@k / pass^k metrics for the comet eval.
+
+These two metrics capture complementary aspects of skill quality:
+
+- **pass@k** (HumanEval): the probability that at least one of k independent
+  attempts succeeds. Measures the *capability ceiling* — "can the agent do it
+  at all, given k tries?". Uses the unbiased estimator
+  ``1 - C(n-c, k) / C(n, k)`` where n = total runs, c = successful runs.
+
+- **pass^k** (Cons@k / reliability): the probability that *all* k attempts
+  succeed. Measures the *reliability floor* — "does the agent do it
+  consistently, every time?". Equals 1 iff c == n, else 0 (for the observed
+  sample).
+
+The gap ``pass@k − pass^k`` quantifies instability: a skill with high pass@k
+but low pass^k "can do it but can't be trusted to do it every time" — a
+critical distinction for a workflow skill users run repeatedly.
+
+"Pass" is defined at the task level: a run passes when its task-specific
+validator reports zero failures (``checks_failed == []``), i.e. the feature was
+actually implemented correctly.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+from typing import Sequence
+
+
+def _comb(n: int, k: int) -> int:
+    """Integer binomial coefficient C(n, k), with C(n,k)=0 for k<0 or k>n."""
+    if k < 0 or k > n:
+        return 0
+    return math.comb(n, k)
+
+
+def pass_at_k(n: int, c: int, k: int) -> float:
+    """Unbiased pass@k estimator (HumanEval definition).
+
+    Parameters
+    ----------
+    n : total number of runs sampled.
+    c : number of successful runs (0 <= c <= n).
+    k : the "k" in pass@k — number of attempts given.
+
+    Returns the probability that at least one of k attempts (drawn without
+    replacement from the n observed runs) is a success. When ``n - c < k`` it
+    is impossible to draw k all-failures, so pass@k = 1.0.
+    """
+    if n - c < k:
+        return 1.0
+    # 1 - C(n-c, k) / C(n, k)
+    return 1.0 - _comb(n - c, k) / _comb(n, k)
+
+
+def pass_pow_k(n: int, c: int, k: int) -> float:
+    """pass^k (reliability): 1.0 iff all n runs passed (so any k-subset is
+    all-pass), else 0.0.
+
+    For the observed sample this is a lower bound on the true "all k succeed"
+    probability. With n >= k runs all passing, pass^k = 1.0.
+    """
+    if c >= n and n >= 1:
+        return 1.0
+    return 0.0
+
+
+def compute_pass_metrics(results: Sequence[bool], k: int = 1) -> dict[str, float | int]:
+    """Compute pass@k and pass^k for a sequence of per-run pass/fail booleans.
+
+    Parameters
+    ----------
+    results : list of bool, one per run (True = passed).
+    k : the k for pass@k / pass^k (default 1, i.e. single-attempt pass rate).
+
+    Returns ``{"pass_at_k": float, "pass_pow_k": float, "n": int, "c": int}``.
+    """
+    n = len(results)
+    c = sum(1 for r in results if r)
+    if n == 0:
+        return {"pass_at_k": 0.0, "pass_pow_k": 0.0, "n": 0, "c": 0}
+    # pass@k is meaningful for k <= n; clamp k to n.
+    k_eff = min(k, n)
+    return {
+        "pass_at_k": pass_at_k(n, c, k_eff),
+        "pass_pow_k": pass_pow_k(n, c, k_eff),
+        "n": n,
+        "c": c,
+        "k": k_eff,
+    }
+
+
+def pass_metrics_table(
+    runs_by_treatment: dict[str, Sequence[bool]],
+    ks: Sequence[int] = (1, 2, 5),
+) -> dict[str, dict[int, dict[str, float]]]:
+    """Compute pass@k / pass^k for multiple treatments and k values.
+
+    Returns ``{treatment: {k: {"pass_at_k", "pass_pow_k", "n", "c"}}}``.
+    """
+    out: dict[str, dict[int, dict[str, float]]] = {}
+    for treatment, results in runs_by_treatment.items():
+        out[treatment] = {}
+        for k in ks:
+            out[treatment][k] = compute_pass_metrics(results, k)
+    return out

package/eval/scaffold/python/paths.py

@@ -0,0 +1,56 @@
+"""Path helpers for eval suites."""
+
+import os
+from pathlib import Path
+
+
+EVAL_ROOT = Path(__file__).resolve().parents[2]
+SUITE_NAMES = {"local", "langsmith"}
+
+
+def get_suite_root() -> Path:
+    """Return the active eval suite root.
+
+    Resolution order:
+    1. BENCH_SUITE_ROOT, for explicit automation
+    2. Current working directory when it looks like a suite root
+    3. Any parent of the current working directory named local/langsmith
+    4. eval/local as the default local suite
+    """
+    configured = os.environ.get("BENCH_SUITE_ROOT")
+    if configured:
+        return Path(configured).resolve()
+
+    cwd = Path.cwd().resolve()
+    if (cwd / "tasks").exists() and (cwd / "treatments").exists():
+        return cwd
+
+    for parent in (cwd, *cwd.parents):
+        if parent.name in SUITE_NAMES:
+            return parent
+
+    return EVAL_ROOT / "local"
+
+
+def get_suite_name() -> str:
+    return get_suite_root().name
+
+
+def get_tasks_dir() -> Path:
+    configured = os.environ.get("BENCH_TASKS_DIR")
+    return Path(configured).resolve() if configured else get_suite_root() / "tasks"
+
+
+def get_treatments_dir() -> Path:
+    configured = os.environ.get("BENCH_TREATMENTS_DIR")
+    return Path(configured).resolve() if configured else get_suite_root() / "treatments"
+
+
+def get_skills_dir() -> Path:
+    configured = os.environ.get("BENCH_SKILLS_DIR")
+    return Path(configured).resolve() if configured else get_suite_root() / "skills"
+
+
+def get_logs_dir() -> Path:
+    configured = os.environ.get("BENCH_LOGS_DIR")
+    return Path(configured).resolve() if configured else get_suite_root() / "logs"

package/eval/scaffold/python/profiles.py

@@ -0,0 +1,127 @@
+"""Evaluation profile registry for local and LangSmith suites."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from scaffold.python.tasks import InteractionConfig, Task
+from scaffold.python.validation.core import ValidatorFn
+from scaffold.python.validation.generic_rubric import (
+    GENERIC_RUBRIC_DIMENSIONS,
+    generic_rubric_validator,
+)
+from scaffold.python.validation.authoring_rubric import (
+    AUTHORING_RUBRIC_DIMENSIONS,
+    authoring_skill_rubric_validator,
+)
+
+GENERIC_PROFILE = "generic"
+COMET_WORKFLOW_PROFILE = "comet-workflow"
+AUTHORING_SKILL_PROFILE = "authoring-skill"
+
+COMET_SIMULATOR_PROMPT = (
+    "You are simulating a developer user in an automated eval. The AI assistant "
+    "below is running the Comet development workflow and has paused to ask you "
+    "something. Read its message and reply with a SHORT (1-3 sentences) response "
+    "that approves reasonable plans, picks sensible defaults, and only asks for "
+    "clarification when the requested outcome is unclear."
+)
+
+GENERIC_SIMULATOR_PROMPT = (
+    "You are simulating a concise developer user in an automated eval. Answer "
+    "the assistant's question in 1-3 sentences, choose reasonable defaults, and "
+    "keep the task moving."
+)
+
+
+@dataclass(frozen=True)
+class ProfileSpec:
+    name: str
+    rubric_dimensions: tuple[str, ...]
+    default_interaction: InteractionConfig
+    rubric: ValidatorFn
+
+
+def _build_profiles() -> dict[str, ProfileSpec]:
+    from scaffold.python.validation.rubric import RUBRIC_DIMENSIONS, comet_rubric_validator
+
+    return {
+        GENERIC_PROFILE: ProfileSpec(
+            name=GENERIC_PROFILE,
+            rubric_dimensions=GENERIC_RUBRIC_DIMENSIONS,
+            default_interaction=InteractionConfig(
+                mode="none",
+                max_turns=12,
+                simulator_prompt=GENERIC_SIMULATOR_PROMPT,
+            ),
+            rubric=generic_rubric_validator,
+        ),
+        COMET_WORKFLOW_PROFILE: ProfileSpec(
+            name=COMET_WORKFLOW_PROFILE,
+            rubric_dimensions=tuple(RUBRIC_DIMENSIONS),
+            default_interaction=InteractionConfig(
+                mode="auto_user",
+                max_turns=12,
+                simulator_prompt=COMET_SIMULATOR_PROMPT,
+            ),
+            rubric=comet_rubric_validator,
+        ),
+        AUTHORING_SKILL_PROFILE: ProfileSpec(
+            name=AUTHORING_SKILL_PROFILE,
+            rubric_dimensions=AUTHORING_RUBRIC_DIMENSIONS,
+            default_interaction=InteractionConfig(
+                mode="auto_user",
+                max_turns=8,
+                simulator_prompt=GENERIC_SIMULATOR_PROMPT,
+            ),
+            rubric=authoring_skill_rubric_validator,
+        ),
+    }
+
+
+def list_profiles() -> list[str]:
+    return sorted(_build_profiles())
+
+
+def get_profile(name: str) -> ProfileSpec:
+    profiles = _build_profiles()
+    if name not in profiles:
+        raise KeyError(f"Profile not found: {name}. Available: {list_profiles()}")
+    return profiles[name]
+
+
+def resolve_profile_name(
+    task: Task,
+    override: str | None = None,
+    target_profile: str | None = None,
+) -> str:
+    if override:
+        get_profile(override)
+        return override
+    if target_profile:
+        get_profile(target_profile)
+        return target_profile
+    if task.config.evaluation.profile:
+        get_profile(task.config.evaluation.profile)
+        return task.config.evaluation.profile
+    return GENERIC_PROFILE
+
+
+def run_profile_rubric(
+    profile_name: str,
+    test_dir: Path,
+    outputs: dict[str, Any],
+) -> tuple[list[str], list[str]]:
+    profile = get_profile(profile_name)
+    return profile.rubric(test_dir, outputs)
+
+
+def all_rubric_dimensions() -> tuple[str, ...]:
+    seen: list[str] = []
+    for profile in _build_profiles().values():
+        for dim in profile.rubric_dimensions:
+            if dim not in seen:
+                seen.append(dim)
+    return tuple(seen)