@rpamis/comet 0.4.0-beta.6 → 0.4.0-beta.8

package/eval/scaffold/python/generic_llm_judge.py

@@ -18,7 +18,6 @@ Usage::
 
 from __future__ import annotations
 
-import os
 import re
 from pathlib import Path
 from typing import Any
@@ -47,6 +46,7 @@ _SKIP_DIRS = {
     ".venv",
     "env",
 }
+_SKIP_FILES = {"_test_context.json", "_test_results.json"}
 
 # Max chars per file, total budget across all files.
 _PER_FILE_LIMIT = 3000
@@ -74,6 +74,8 @@ def _collect_workspace_artifacts(test_dir: Path) -> str:
     for path in sorted(test_dir.rglob("*")):
         if not path.is_file():
             continue
+        if path.name in _SKIP_FILES:
+            continue
         # Skip hidden paths.
         if any(part.startswith(".") for part in path.relative_to(test_dir).parts):
             continue

package/eval/scaffold/python/llm_judge.py

@@ -18,11 +18,8 @@ Usage from the rubric validator or rescore tool::
 
 from __future__ import annotations
 
-import json
-import os
 import re
 from pathlib import Path
-from typing import Any
 
 from scaffold.python.judge_config import build_judge_invocation, run_judge_prompt
 

package/eval/scaffold/python/logging.py

@@ -1,6 +1,7 @@
 """Output parsing, event extraction, and experiment logging for Claude CLI."""
 
 import json
+import math
 import re
 from collections.abc import Callable
 from dataclasses import dataclass, field
@@ -68,6 +69,7 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
         "commands_run": [],
         "skills_invoked": [],
         "duration_seconds": None,
+        "subject_invocations": 0,
         "num_turns": None,
         "input_tokens": None,
         "output_tokens": None,
@@ -76,32 +78,99 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
         "total_tokens": None,
         "total_cost_usd": None,
         "model_usage": {},
+        "peak_context_input_tokens": None,
+        "p95_context_input_tokens": None,
+        "average_context_input_tokens": None,
+        "peak_context_window_tokens": None,
+        "peak_context_occupancy_pct": None,
     }
 
     # Map tool_use_id -> index in tool_calls list for matching outputs
     tool_id_to_index = {}
+    duration_ms_total = 0.0
+    duration_observed = False
+    additive_metrics = {
+        "num_turns": 0,
+        "input_tokens": 0,
+        "output_tokens": 0,
+        "cache_read_input_tokens": 0,
+        "cache_creation_input_tokens": 0,
+        "total_cost_usd": 0.0,
+    }
+    observed_metrics: set[str] = set()
+    context_by_message: dict[str, tuple[int, str | None]] = {}
+
+    def add_metric(key: str, value: Any) -> None:
+        if isinstance(value, (int, float)) and not isinstance(value, bool):
+            additive_metrics[key] += value
+            observed_metrics.add(key)
+
+    def merge_model_usage(value: Any) -> None:
+        if not isinstance(value, dict):
+            return
+        additive = {
+            "inputTokens",
+            "outputTokens",
+            "cacheReadInputTokens",
+            "cacheCreationInputTokens",
+            "webSearchRequests",
+            "costUSD",
+        }
+        capacities = {"contextWindow", "maxOutputTokens"}
+        for model, raw_usage in value.items():
+            if not isinstance(model, str) or not isinstance(raw_usage, dict):
+                continue
+            target = events["model_usage"].setdefault(model, {})
+            for key, item in raw_usage.items():
+                if isinstance(item, bool) or not isinstance(item, (int, float)):
+                    continue
+                if key in additive:
+                    target[key] = target.get(key, 0) + item
+                elif key in capacities:
+                    target[key] = max(target.get(key, 0), item)
 
     for msg in parsed.get("messages", []):
         if msg.get("type") == "result":
-            events["duration_seconds"] = msg.get("duration_ms", 0) / 1000
-            events["num_turns"] = msg.get("num_turns")
+            events["subject_invocations"] += 1
+            duration_ms = msg.get("duration_ms")
+            if isinstance(duration_ms, (int, float)) and not isinstance(duration_ms, bool):
+                duration_ms_total += duration_ms
+                duration_observed = True
+
+            # A loop concatenates one result per subject invocation. Claude's
+            # result telemetry is invocation-local, so task totals must add all
+            # initial, answer, and cold-resume invocations instead of retaining
+            # only the final result event.
+            add_metric("num_turns", msg.get("num_turns"))
             usage = msg.get("usage") or {}
-            input_tokens = usage.get("input_tokens")
-            output_tokens = usage.get("output_tokens")
-            cache_read = usage.get("cache_read_input_tokens")
-            cache_creation = usage.get("cache_creation_input_tokens")
-            events["input_tokens"] = input_tokens
-            events["output_tokens"] = output_tokens
-            events["cache_read_input_tokens"] = cache_read
-            events["cache_creation_input_tokens"] = cache_creation
-            token_parts = [input_tokens, output_tokens, cache_read, cache_creation]
-            if any(v is not None for v in token_parts):
-                events["total_tokens"] = sum(v or 0 for v in token_parts)
-            events["total_cost_usd"] = msg.get("total_cost_usd")
-            events["model_usage"] = msg.get("modelUsage") or {}
+            add_metric("input_tokens", usage.get("input_tokens"))
+            add_metric("output_tokens", usage.get("output_tokens"))
+            add_metric("cache_read_input_tokens", usage.get("cache_read_input_tokens"))
+            add_metric("cache_creation_input_tokens", usage.get("cache_creation_input_tokens"))
+            add_metric("total_cost_usd", msg.get("total_cost_usd"))
+            merge_model_usage(msg.get("modelUsage"))
 
         if msg.get("type") == "assistant":
-            for item in msg.get("message", {}).get("content", []):
+            message = msg.get("message", {})
+            message_id = message.get("id")
+            usage = message.get("usage") or {}
+            context_parts = (
+                usage.get("input_tokens"),
+                usage.get("cache_read_input_tokens"),
+                usage.get("cache_creation_input_tokens"),
+            )
+            if isinstance(message_id, str):
+                context_tokens = sum(
+                    value
+                    for value in context_parts
+                    if isinstance(value, (int, float)) and not isinstance(value, bool)
+                )
+                if context_tokens > 0:
+                    previous = context_by_message.get(message_id)
+                    if previous is None or context_tokens > previous[0]:
+                        context_by_message[message_id] = (context_tokens, message.get("model"))
+
+            for item in message.get("content", []):
                 if item.get("type") == "tool_use":
                     tool, inp = item.get("name", ""), item.get("input", {})
                     tool_id = item.get("id")
@@ -143,6 +212,35 @@ def extract_events(parsed: dict[str, Any]) -> dict[str, Any]:
                             )
                         events["tool_calls"][idx]["output"] = content
 
+    if duration_observed:
+        events["duration_seconds"] = duration_ms_total / 1000
+
+    for key, value in additive_metrics.items():
+        if key in observed_metrics:
+            events[key] = value
+    token_keys = (
+        "input_tokens",
+        "output_tokens",
+        "cache_read_input_tokens",
+        "cache_creation_input_tokens",
+    )
+    if any(key in observed_metrics for key in token_keys):
+        events["total_tokens"] = sum(additive_metrics[key] for key in token_keys)
+
+    if context_by_message:
+        samples = sorted(value for value, _model in context_by_message.values())
+        peak = samples[-1]
+        p95 = samples[max(0, math.ceil(len(samples) * 0.95) - 1)]
+        peak_model = next(model for value, model in context_by_message.values() if value == peak)
+        model_usage = events["model_usage"].get(peak_model, {}) if peak_model else {}
+        context_window = model_usage.get("contextWindow")
+        events["peak_context_input_tokens"] = peak
+        events["p95_context_input_tokens"] = p95
+        events["average_context_input_tokens"] = sum(samples) / len(samples)
+        if isinstance(context_window, (int, float)) and not isinstance(context_window, bool):
+            events["peak_context_window_tokens"] = context_window
+            events["peak_context_occupancy_pct"] = peak / context_window * 100
+
     return events
 
 
@@ -374,6 +472,7 @@ def rubric_columns(dimensions: tuple[str, ...] | list[str] | None = None) -> lis
     cols.append(rubric_total_column())
     return cols
 
+
 def _checks_aggregate(runs: list) -> str:
     """Aggregate checks passed across runs."""
     total_passed = sum(len(r.checks_passed) for r in runs)

package/eval/scaffold/python/manifests.py

@@ -125,5 +125,7 @@ def load_eval_manifest(path: Path | str) -> SkillEvalManifest:
             max_turns=int(interaction_data.get("maxTurns", interaction_data.get("max_turns", 12))),
             simulator_prompt=interaction_data.get("simulatorPrompt")
             or interaction_data.get("simulator_prompt"),
+            fresh_resume_marker=interaction_data.get("freshResumeMarker")
+            or interaction_data.get("fresh_resume_marker"),
         ),
     )

package/eval/scaffold/python/native_eval.py

@@ -0,0 +1,138 @@
+"""Adapt canonical Comet task checks to Native workflow semantics."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from scaffold.python.validation.native_workflow import validate_native_workflow
+
+
+NATIVE_CLARIFICATION_MODES = {
+    "COMET_NATIVE_SEQUENTIAL": "sequential",
+    "COMET_NATIVE_BATCH": "batch",
+}
+NATIVE_TREATMENTS = {"COMET_NATIVE_PHASE1", *NATIVE_CLARIFICATION_MODES}
+CONTROL_BUSINESS_ONLY_TREATMENTS = {"CONTROL"}
+CLASSIC_WORKFLOW_CHECK_PREFIXES = (
+    "openspec_artifacts",
+    "comet_state",
+    "workflow_phases",
+    "tests_written",
+    "tests_exist",
+)
+NATIVE_WORKFLOW_CHECK_PREFIXES = (
+    "native_skill_invocation",
+    "native_artifacts",
+    "native_state",
+    "native_trajectory",
+    "native_isolation",
+)
+COMET_WORKFLOW_CHECK_PREFIXES = (
+    *CLASSIC_WORKFLOW_CHECK_PREFIXES,
+    *NATIVE_WORKFLOW_CHECK_PREFIXES,
+)
+
+NATIVE_PROMPT_PREFIX = """[COMET NATIVE TREATMENT]
+Invoke /comet-native as the only Skill. Do not invoke /comet or any other Skill.
+Preserve every business requirement in the task below, but interpret legacy
+references to the comet workflow and its Open, Design, Build, Verify, and Archive
+phases as the Native Shape, Build, Verify, and Archive workflow. Use only Native's
+bundled runtime and initialize artifact_root `docs`. {terminal_instruction}
+{clarification_instruction}
+Do not create OpenSpec, Classic, Superpowers, or hidden `.comet` artifacts.
+
+[ORIGINAL BUSINESS TASK]
+"""
+
+
+def _is_classic_workflow_check(check: str) -> bool:
+    return any(check.startswith(prefix) for prefix in CLASSIC_WORKFLOW_CHECK_PREFIXES)
+
+
+def _is_comet_workflow_check(check: str) -> bool:
+    return any(check.startswith(prefix) for prefix in COMET_WORKFLOW_CHECK_PREFIXES)
+
+
+def is_control_business_only_run(profile_name: str, treatment_name: str) -> bool:
+    """Return whether workflow checks are non-applicable for this CONTROL run."""
+    return profile_name == "comet-workflow" and treatment_name in CONTROL_BUSINESS_ONLY_TREATMENTS
+
+
+def filter_control_workflow_checks(
+    profile_name: str,
+    treatment_name: str,
+    passed: list[str],
+    failed: list[str],
+) -> tuple[list[str], list[str]]:
+    """Remove Classic and Native workflow checks from business-only CONTROL."""
+    if not is_control_business_only_run(profile_name, treatment_name):
+        return passed, failed
+    return (
+        [check for check in passed if not _is_comet_workflow_check(check)],
+        [check for check in failed if not _is_comet_workflow_check(check)],
+    )
+
+
+def split_comet_completion_checks(
+    passed: list[str],
+    failed: list[str],
+) -> dict[str, dict[str, list[str]]]:
+    """Split business checks from both Classic and Native workflow checks."""
+    return {
+        "business_completion": {
+            "passed": [check for check in passed if not _is_comet_workflow_check(check)],
+            "failed": [check for check in failed if not _is_comet_workflow_check(check)],
+        },
+        "workflow_completion": {
+            "passed": [check for check in passed if _is_comet_workflow_check(check)],
+            "failed": [check for check in failed if _is_comet_workflow_check(check)],
+        },
+    }
+
+
+def adapt_prompt_for_native(
+    prompt: str,
+    treatment_name: str,
+    terminal_mode: str = "archive",
+) -> str:
+    """Give canonical Classic-worded tasks an explicit Native treatment contract."""
+    if treatment_name not in NATIVE_TREATMENTS:
+        return prompt
+    terminal_instruction = {
+        "active": "Leave the task-requested Native changes active; do not force an Archive.",
+        "active-blocked": (
+            "If the task intentionally exercises a runtime stop, leave the change active at its "
+            "runtime-enforced blocked state."
+        ),
+    }.get(terminal_mode, "Leave a verified terminal Native archive.")
+    clarification_mode = NATIVE_CLARIFICATION_MODES.get(treatment_name)
+    clarification_instruction = (
+        f"Configure native.clarification_mode `{clarification_mode}` before asking any product "
+        "question and do not change it during the run."
+        if clarification_mode
+        else ""
+    )
+    return (
+        f"{NATIVE_PROMPT_PREFIX.format(terminal_instruction=terminal_instruction, clarification_instruction=clarification_instruction)}"
+        f"{prompt}"
+    )
+
+
+def adapt_checks_for_native(
+    test_dir: Path,
+    outputs: dict[str, Any],
+    passed: list[str],
+    failed: list[str],
+) -> tuple[list[str], list[str]]:
+    """Replace Classic-only checks with Native's equivalent hard contract."""
+    if outputs.get("treatment_name") not in NATIVE_TREATMENTS:
+        return passed, failed
+    kept_passed = [check for check in passed if not _is_classic_workflow_check(check)]
+    kept_failed = [check for check in failed if not _is_classic_workflow_check(check)]
+    native_passed, native_failed = validate_native_workflow(
+        test_dir,
+        outputs,
+        terminal_mode=outputs.get("native_terminal", "archive"),
+    )
+    return kept_passed + native_passed, kept_failed + native_failed

package/eval/scaffold/python/pass_at_k.py

@@ -24,7 +24,6 @@ actually implemented correctly.
 from __future__ import annotations
 
 import math
-from collections import defaultdict
 from typing import Sequence