@bastani/atomic 0.6.8 → 0.7.0-2

package/.agents/skills/context-compression/scripts/compression_evaluator.py

@@ -1,862 +0,0 @@
-"""
-Context Compression Evaluation
-
-Public API for evaluating context compression quality using probe-based
-assessment. This module provides three composable components:
-
-- **ProbeGenerator**: Extracts factual claims, file operations, and decisions
-  from conversation history, then generates typed probes for evaluation.
-  Use when: building a compression evaluation pipeline and needing to
-  automatically derive test questions from raw conversation history.
-
-- **CompressionEvaluator**: Scores probe responses against a multi-dimensional
-  rubric (accuracy, context awareness, artifact trail, completeness,
-  continuity, instruction following). Use when: comparing compression methods
-  or validating that a compression strategy preserves critical information.
-
-- **StructuredSummarizer**: Implements anchored iterative summarization with
-  explicit sections for session intent, file tracking, decisions, and next
-  steps. Use when: compressing long-running coding sessions where file
-  tracking and decision rationale must survive compression.
-
-Top-level convenience function:
-- **evaluate_compression_quality**: End-to-end pipeline that generates probes,
-  collects model responses, evaluates them, and returns a scored summary with
-  recommendations. Use when: running a one-shot compression quality check
-  without wiring up individual components.
-
-PRODUCTION NOTES:
-- The LLM judge calls are stubbed for demonstration. Production systems
-  should implement actual API calls to a frontier model.
-- Token estimation uses simplified heuristics. Production systems should
-  use model-specific tokenizers.
-- Ground truth extraction uses pattern matching. Production systems may
-  benefit from more sophisticated fact extraction.
-"""
-
-from dataclasses import dataclass, field
-from typing import List, Dict, Optional, Callable
-from enum import Enum
-import json
-import re
-
-__all__ = [
-    "ProbeType",
-    "Probe",
-    "CriterionResult",
-    "EvaluationResult",
-    "RUBRIC_CRITERIA",
-    "ProbeGenerator",
-    "CompressionEvaluator",
-    "StructuredSummarizer",
-    "evaluate_compression_quality",
-]
-
-
-class ProbeType(Enum):
-    """Types of evaluation probes for compression quality assessment."""
-    RECALL = "recall"
-    ARTIFACT = "artifact"
-    CONTINUATION = "continuation"
-    DECISION = "decision"
-
-
-@dataclass
-class Probe:
-    """A probe question for evaluating compression quality.
-
-    Use when: constructing evaluation inputs for CompressionEvaluator.
-    Each probe targets a specific information category that compression
-    may have lost.
-    """
-    probe_type: ProbeType
-    question: str
-    ground_truth: Optional[str] = None
-    context_reference: Optional[str] = None
-
-
-@dataclass
-class CriterionResult:
-    """Result for a single evaluation criterion."""
-    criterion_id: str
-    score: float
-    reasoning: str
-
-
-@dataclass
-class EvaluationResult:
-    """Complete evaluation result for a probe response.
-
-    Contains per-criterion scores, per-dimension aggregates, and an
-    overall aggregate score.
-    """
-    probe: Probe
-    response: str
-    criterion_results: List[CriterionResult]
-    aggregate_score: float
-    dimension_scores: Dict[str, float] = field(default_factory=dict)
-
-
-# Evaluation Rubrics
-
-RUBRIC_CRITERIA: Dict[str, List[Dict]] = {
-    "accuracy": [
-        {
-            "id": "accuracy_factual",
-            "question": "Are facts, file paths, and technical details correct?",
-            "weight": 0.6
-        },
-        {
-            "id": "accuracy_technical",
-            "question": "Are code references and technical concepts correct?",
-            "weight": 0.4
-        }
-    ],
-    "context_awareness": [
-        {
-            "id": "context_conversation_state",
-            "question": "Does the response reflect current conversation state?",
-            "weight": 0.5
-        },
-        {
-            "id": "context_artifact_state",
-            "question": "Does the response reflect which files/artifacts were accessed?",
-            "weight": 0.5
-        }
-    ],
-    "artifact_trail": [
-        {
-            "id": "artifact_files_created",
-            "question": "Does the agent know which files were created?",
-            "weight": 0.3
-        },
-        {
-            "id": "artifact_files_modified",
-            "question": "Does the agent know which files were modified?",
-            "weight": 0.4
-        },
-        {
-            "id": "artifact_key_details",
-            "question": "Does the agent remember function names, variable names, error messages?",
-            "weight": 0.3
-        }
-    ],
-    "completeness": [
-        {
-            "id": "completeness_coverage",
-            "question": "Does the response address all parts of the question?",
-            "weight": 0.6
-        },
-        {
-            "id": "completeness_depth",
-            "question": "Is sufficient detail provided?",
-            "weight": 0.4
-        }
-    ],
-    "continuity": [
-        {
-            "id": "continuity_work_state",
-            "question": "Can the agent continue without re-fetching information?",
-            "weight": 0.4
-        },
-        {
-            "id": "continuity_todo_state",
-            "question": "Does the agent maintain awareness of pending tasks?",
-            "weight": 0.3
-        },
-        {
-            "id": "continuity_reasoning",
-            "question": "Does the agent retain rationale behind previous decisions?",
-            "weight": 0.3
-        }
-    ],
-    "instruction_following": [
-        {
-            "id": "instruction_format",
-            "question": "Does the response follow the requested format?",
-            "weight": 0.5
-        },
-        {
-            "id": "instruction_constraints",
-            "question": "Does the response respect stated constraints?",
-            "weight": 0.5
-        }
-    ]
-}
-
-
-class ProbeGenerator:
-    """Generate typed probes from conversation history.
-
-    Use when: automatically deriving evaluation questions from raw
-    conversation history at compression points. Extracts facts, file
-    operations, and decisions via pattern matching, then produces
-    one probe per category.
-
-    For production systems, replace the regex-based extraction with
-    an LLM-based extractor for higher recall.
-    """
-
-    def __init__(self, conversation_history: str) -> None:
-        self.history = conversation_history
-        self.extracted_facts = self._extract_facts()
-        self.extracted_files = self._extract_files()
-        self.extracted_decisions = self._extract_decisions()
-
-    def generate_probes(self) -> List[Probe]:
-        """Generate all probe types for evaluation.
-
-        Use when: preparing evaluation inputs at a compression point.
-        Returns one probe per category (recall, artifact, continuation,
-        decision) based on extractable content from the history.
-        """
-        probes: List[Probe] = []
-
-        # Recall probes
-        if self.extracted_facts:
-            probes.append(Probe(
-                probe_type=ProbeType.RECALL,
-                question="What was the original error or issue that started this session?",
-                ground_truth=self.extracted_facts.get("original_error"),
-                context_reference="session_start"
-            ))
-
-        # Artifact probes
-        if self.extracted_files:
-            probes.append(Probe(
-                probe_type=ProbeType.ARTIFACT,
-                question="Which files have we modified? Describe what changed in each.",
-                ground_truth=json.dumps(self.extracted_files),
-                context_reference="file_operations"
-            ))
-
-        # Continuation probes
-        probes.append(Probe(
-            probe_type=ProbeType.CONTINUATION,
-            question="What should we do next?",
-            ground_truth=self.extracted_facts.get("next_steps"),
-            context_reference="task_state"
-        ))
-
-        # Decision probes
-        if self.extracted_decisions:
-            probes.append(Probe(
-                probe_type=ProbeType.DECISION,
-                question="What key decisions did we make and why?",
-                ground_truth=json.dumps(self.extracted_decisions),
-                context_reference="decision_points"
-            ))
-
-        return probes
-
-    def _extract_facts(self) -> Dict[str, str]:
-        """Extract factual claims from history."""
-        facts: Dict[str, str] = {}
-
-        # Extract error patterns
-        error_patterns = [
-            r"error[:\s]+(.+?)(?:\n|$)",
-            r"(\d{3})\s+(Unauthorized|Not Found|Internal Server Error)",
-            r"exception[:\s]+(.+?)(?:\n|$)"
-        ]
-
-        for pattern in error_patterns:
-            match = re.search(pattern, self.history, re.IGNORECASE)
-            if match:
-                facts["original_error"] = match.group(0).strip()
-                break
-
-        # Extract next steps
-        next_step_patterns = [
-            r"next[:\s]+(.+?)(?:\n|$)",
-            r"TODO[:\s]+(.+?)(?:\n|$)",
-            r"remaining[:\s]+(.+?)(?:\n|$)"
-        ]
-
-        for pattern in next_step_patterns:
-            match = re.search(pattern, self.history, re.IGNORECASE)
-            if match:
-                facts["next_steps"] = match.group(0).strip()
-                break
-
-        return facts
-
-    def _extract_files(self) -> List[Dict[str, str]]:
-        """Extract file operations from history."""
-        files: List[Dict[str, str]] = []
-
-        # Common file patterns
-        file_patterns = [
-            r"(?:modified|changed|updated|edited)\s+([^\s]+\.[a-z]+)",
-            r"(?:created|added)\s+([^\s]+\.[a-z]+)",
-            r"(?:read|examined|opened)\s+([^\s]+\.[a-z]+)"
-        ]
-
-        for pattern in file_patterns:
-            matches = re.findall(pattern, self.history, re.IGNORECASE)
-            for match in matches:
-                if match not in [f["path"] for f in files]:
-                    files.append({
-                        "path": match,
-                        "operation": "modified" if "modif" in pattern else "created" if "creat" in pattern else "read"
-                    })
-
-        return files
-
-    def _extract_decisions(self) -> List[Dict[str, str]]:
-        """Extract decision points from history."""
-        decisions: List[Dict[str, str]] = []
-
-        decision_patterns = [
-            r"decided to\s+(.+?)(?:\n|$)",
-            r"chose\s+(.+?)(?:\n|$)",
-            r"going with\s+(.+?)(?:\n|$)",
-            r"will use\s+(.+?)(?:\n|$)"
-        ]
-
-        for pattern in decision_patterns:
-            matches = re.findall(pattern, self.history, re.IGNORECASE)
-            for match in matches:
-                decisions.append({
-                    "decision": match.strip(),
-                    "context": pattern.split("\\s+")[0]
-                })
-
-        return decisions[:5]  # Limit to 5 decisions
-
-
-class CompressionEvaluator:
-    """Evaluate compression quality using probes and LLM judge.
-
-    Use when: comparing compression methods or validating that a specific
-    compression pass preserved critical information. Scores responses
-    across six dimensions (accuracy, context awareness, artifact trail,
-    completeness, continuity, instruction following) and produces an
-    aggregate quality score.
-
-    The evaluate() method is the primary entry point. Call it once per
-    probe, then call get_summary() to retrieve aggregated results.
-    """
-
-    def __init__(self, model: str = "gpt-5.2") -> None:
-        self.model = model
-        self.results: List[EvaluationResult] = []
-
-    def evaluate(self,
-                 probe: Probe,
-                 response: str,
-                 compressed_context: str) -> EvaluationResult:
-        """Evaluate a single probe response against the rubric.
-
-        Use when: scoring how well a model's response (given compressed
-        context) answers a probe question. Returns per-criterion scores,
-        per-dimension aggregates, and an overall score.
-
-        Args:
-            probe: The probe question with expected ground truth.
-            response: The model's response to evaluate.
-            compressed_context: The compressed context that was provided
-                to the model when generating the response.
-
-        Returns:
-            EvaluationResult with scores and reasoning across all
-            applicable dimensions.
-        """
-        # Get relevant criteria based on probe type
-        criteria = self._get_criteria_for_probe(probe.probe_type)
-
-        # Evaluate each criterion
-        criterion_results: List[CriterionResult] = []
-        for criterion in criteria:
-            result = self._evaluate_criterion(
-                criterion,
-                probe,
-                response,
-                compressed_context
-            )
-            criterion_results.append(result)
-
-        # Calculate dimension scores
-        dimension_scores = self._calculate_dimension_scores(criterion_results)
-
-        # Calculate aggregate score
-        aggregate_score = sum(dimension_scores.values()) / len(dimension_scores) if dimension_scores else 0.0
-
-        result = EvaluationResult(
-            probe=probe,
-            response=response,
-            criterion_results=criterion_results,
-            aggregate_score=aggregate_score,
-            dimension_scores=dimension_scores
-        )
-
-        self.results.append(result)
-        return result
-
-    def get_summary(self) -> Dict:
-        """Get summary of all evaluation results.
-
-        Use when: all probes have been evaluated and an aggregate
-        report is needed to compare methods or make a go/no-go
-        decision on a compression strategy.
-
-        Returns:
-            Dictionary with total evaluations, average score,
-            per-dimension averages, and weakest/strongest dimensions.
-        """
-        if not self.results:
-            return {"error": "No evaluations performed"}
-
-        avg_score = sum(r.aggregate_score for r in self.results) / len(self.results)
-
-        # Average dimension scores
-        dimension_totals: Dict[str, float] = {}
-        dimension_counts: Dict[str, int] = {}
-
-        for result in self.results:
-            for dim, score in result.dimension_scores.items():
-                dimension_totals[dim] = dimension_totals.get(dim, 0) + score
-                dimension_counts[dim] = dimension_counts.get(dim, 0) + 1
-
-        avg_dimensions = {
-            dim: dimension_totals[dim] / dimension_counts[dim]
-            for dim in dimension_totals
-        }
-
-        return {
-            "total_evaluations": len(self.results),
-            "average_score": avg_score,
-            "dimension_averages": avg_dimensions,
-            "weakest_dimension": min(avg_dimensions, key=avg_dimensions.get) if avg_dimensions else None,
-            "strongest_dimension": max(avg_dimensions, key=avg_dimensions.get) if avg_dimensions else None,
-        }
-
-    def _get_criteria_for_probe(self, probe_type: ProbeType) -> List[Dict]:
-        """Get relevant criteria for probe type."""
-        criteria: List[Dict] = []
-
-        # All probes get accuracy and completeness
-        criteria.extend(RUBRIC_CRITERIA["accuracy"])
-        criteria.extend(RUBRIC_CRITERIA["completeness"])
-
-        # Add type-specific criteria
-        if probe_type == ProbeType.ARTIFACT:
-            criteria.extend(RUBRIC_CRITERIA["artifact_trail"])
-        elif probe_type == ProbeType.CONTINUATION:
-            criteria.extend(RUBRIC_CRITERIA["continuity"])
-        elif probe_type == ProbeType.RECALL:
-            criteria.extend(RUBRIC_CRITERIA["context_awareness"])
-        elif probe_type == ProbeType.DECISION:
-            criteria.extend(RUBRIC_CRITERIA["context_awareness"])
-            criteria.extend(RUBRIC_CRITERIA["continuity"])
-
-        criteria.extend(RUBRIC_CRITERIA["instruction_following"])
-
-        return criteria
-
-    def _evaluate_criterion(self,
-                            criterion: Dict,
-                            probe: Probe,
-                            response: str,
-                            context: str) -> CriterionResult:
-        """
-        Evaluate a single criterion using LLM judge.
-
-        PRODUCTION NOTE: This is a stub implementation.
-        Production systems should call the actual LLM API:
-
-        ```python
-        result = openai.chat.completions.create(
-            model="gpt-5.2",
-            messages=[
-                {"role": "system", "content": JUDGE_SYSTEM_PROMPT},
-                {"role": "user", "content": self._format_judge_input(criterion, probe, response, context)}
-            ]
-        )
-        return self._parse_judge_output(result)
-        ```
-        """
-        # Stub implementation - in production, call LLM judge
-        score = self._heuristic_score(criterion, response, probe.ground_truth)
-        reasoning = f"Evaluated {criterion['id']} based on response content."
-
-        return CriterionResult(
-            criterion_id=criterion["id"],
-            score=score,
-            reasoning=reasoning
-        )
-
-    def _heuristic_score(self,
-                         criterion: Dict,
-                         response: str,
-                         ground_truth: Optional[str]) -> float:
-        """
-        Heuristic scoring for demonstration.
-
-        Production systems should use LLM judge instead.
-        """
-        score = 3.0  # Base score
-
-        # Adjust based on response length and content
-        if len(response) < 50:
-            score -= 1.0  # Too short
-        elif len(response) > 500:
-            score += 0.5  # Detailed
-
-        # Check for technical content
-        if any(ext in response for ext in [".ts", ".py", ".js", ".md"]):
-            score += 0.5  # Contains file references
-
-        overlap_ratio = self._ground_truth_overlap_ratio(response, ground_truth)
-        if overlap_ratio >= 0.75:
-            score += 1.0
-        elif overlap_ratio >= 0.4:
-            score += 0.5
-        elif ground_truth:
-            score -= 0.5
-
-        return min(5.0, max(0.0, score))
-
-    def _ground_truth_overlap_ratio(self,
-                                    response: str,
-                                    ground_truth: Optional[str]) -> float:
-        if not ground_truth:
-            return 0.0
-
-        terms = self._extract_ground_truth_terms(ground_truth)
-        if not terms:
-            return 1.0 if ground_truth.lower() in response.lower() else 0.0
-
-        response_lower = response.lower()
-        matches = sum(1 for term in terms if term in response_lower)
-        return matches / len(terms)
-
-    def _extract_ground_truth_terms(self, ground_truth: str) -> List[str]:
-        try:
-            parsed = json.loads(ground_truth)
-        except json.JSONDecodeError:
-            return [ground_truth.lower()] if ground_truth.strip() else []
-
-        terms: List[str] = []
-
-        def collect(value) -> None:
-            if isinstance(value, str):
-                normalized = value.strip().lower()
-                if normalized:
-                    terms.append(normalized)
-            elif isinstance(value, dict):
-                for nested in value.values():
-                    collect(nested)
-            elif isinstance(value, list):
-                for nested in value:
-                    collect(nested)
-
-        collect(parsed)
-        return list(dict.fromkeys(terms))
-
-    def _calculate_dimension_scores(self,
-                                    criterion_results: List[CriterionResult]) -> Dict[str, float]:
-        """Calculate dimension scores from criterion results."""
-        dimension_scores: Dict[str, float] = {}
-
-        for dimension, criteria in RUBRIC_CRITERIA.items():
-            criterion_ids = [c["id"] for c in criteria]
-            relevant_results = [
-                r for r in criterion_results
-                if r.criterion_id in criterion_ids
-            ]
-
-            if relevant_results:
-                # Weighted average
-                total_weight = sum(
-                    c["weight"] for c in criteria
-                    if c["id"] in [r.criterion_id for r in relevant_results]
-                )
-                weighted_sum = sum(
-                    r.score * next(c["weight"] for c in criteria if c["id"] == r.criterion_id)
-                    for r in relevant_results
-                )
-                dimension_scores[dimension] = weighted_sum / total_weight if total_weight > 0 else 0.0
-
-        return dimension_scores
-
-
-class StructuredSummarizer:
-    """Generate structured summaries with explicit sections.
-
-    Use when: implementing anchored iterative summarization for
-    long-running coding sessions. Maintains a persistent summary
-    with dedicated sections for session intent, file modifications,
-    decisions, current state, and next steps.
-
-    Call update_from_span() each time a new content span is truncated.
-    The summarizer merges new information into existing sections rather
-    than regenerating, preventing cumulative detail loss.
-    """
-
-    TEMPLATE = """## Session Intent
-{intent}
-
-## Files Modified
-{files_modified}
-
-## Files Read (Not Modified)
-{files_read}
-
-## Decisions Made
-{decisions}
-
-## Current State
-{current_state}
-
-## Next Steps
-{next_steps}
-"""
-
-    def __init__(self) -> None:
-        self.sections: Dict = {
-            "intent": "",
-            "files_modified": [],
-            "files_read": [],
-            "decisions": [],
-            "current_state": "",
-            "next_steps": []
-        }
-
-    def update_from_span(self, new_content: str) -> str:
-        """Update summary from newly truncated content span.
-
-        Use when: a compression trigger fires and a portion of
-        conversation history is about to be discarded. Pass the
-        content that will be truncated; the summarizer extracts
-        structured information and merges it with prior state.
-
-        Args:
-            new_content: The conversation span being truncated.
-
-        Returns:
-            Formatted summary string with all sections populated.
-        """
-        # Extract information from new content
-        new_info = self._extract_from_content(new_content)
-
-        # Merge with existing sections
-        self._merge_sections(new_info)
-
-        # Generate formatted summary
-        return self._format_summary()
-
-    def _extract_from_content(self, content: str) -> Dict:
-        """Extract structured information from content."""
-        extracted: Dict = {
-            "intent": "",
-            "files_modified": [],
-            "files_read": [],
-            "decisions": [],
-            "current_state": "",
-            "next_steps": []
-        }
-
-        # Extract file modifications
-        mod_pattern = r"(?:modified|changed|updated|fixed)\s+([^\s]+\.[a-z]+)[:\s]*(.+?)(?:\n|$)"
-        for match in re.finditer(mod_pattern, content, re.IGNORECASE):
-            extracted["files_modified"].append({
-                "path": match.group(1),
-                "change": match.group(2).strip()[:100]
-            })
-
-        # Extract file reads
-        read_pattern = r"(?:read|examined|opened|checked)\s+([^\s]+\.[a-z]+)"
-        for match in re.finditer(read_pattern, content, re.IGNORECASE):
-            file_path = match.group(1)
-            if file_path not in [f["path"] for f in extracted["files_modified"]]:
-                extracted["files_read"].append(file_path)
-
-        # Extract decisions
-        decision_pattern = r"(?:decided|chose|going with|will use)\s+(.+?)(?:\n|$)"
-        for match in re.finditer(decision_pattern, content, re.IGNORECASE):
-            extracted["decisions"].append(match.group(1).strip()[:150])
-
-        return extracted
-
-    def _merge_sections(self, new_info: Dict) -> None:
-        """Merge new information with existing sections."""
-        # Update intent if empty
-        if new_info["intent"] and not self.sections["intent"]:
-            self.sections["intent"] = new_info["intent"]
-
-        # Merge file lists (deduplicate by path)
-        existing_mod_paths = [f["path"] for f in self.sections["files_modified"]]
-        for file_info in new_info["files_modified"]:
-            if file_info["path"] not in existing_mod_paths:
-                self.sections["files_modified"].append(file_info)
-
-        # Merge read files
-        for file_path in new_info["files_read"]:
-            if file_path not in self.sections["files_read"]:
-                self.sections["files_read"].append(file_path)
-
-        # Append decisions
-        self.sections["decisions"].extend(new_info["decisions"])
-
-        # Update current state (latest wins)
-        if new_info["current_state"]:
-            self.sections["current_state"] = new_info["current_state"]
-
-        # Merge next steps
-        self.sections["next_steps"].extend(new_info["next_steps"])
-
-    def _format_summary(self) -> str:
-        """Format sections into summary string."""
-        files_modified_str = "\n".join(
-            f"- {f['path']}: {f['change']}"
-            for f in self.sections["files_modified"]
-        ) or "None"
-
-        files_read_str = "\n".join(
-            f"- {f}" for f in self.sections["files_read"]
-        ) or "None"
-
-        decisions_str = "\n".join(
-            f"- {d}" for d in self.sections["decisions"][-5:]  # Keep last 5
-        ) or "None"
-
-        next_steps_str = "\n".join(
-            f"{i+1}. {s}" for i, s in enumerate(self.sections["next_steps"][-5:])
-        ) or "None"
-
-        return self.TEMPLATE.format(
-            intent=self.sections["intent"] or "Not specified",
-            files_modified=files_modified_str,
-            files_read=files_read_str,
-            decisions=decisions_str,
-            current_state=self.sections["current_state"] or "In progress",
-            next_steps=next_steps_str
-        )
-
-
-def evaluate_compression_quality(
-    original_history: str,
-    compressed_context: str,
-    model_response_fn: Callable[[str, str], str],
-) -> Dict:
-    """Evaluate compression quality for a conversation end-to-end.
-
-    Use when: running a one-shot quality check on a compression pass.
-    Generates probes from original history, collects model responses
-    using the compressed context, evaluates each response, and returns
-    a scored summary with actionable recommendations.
-
-    Args:
-        original_history: The full conversation before compression.
-        compressed_context: The compressed version to evaluate.
-        model_response_fn: Callable that takes (compressed_context, question)
-            and returns the model's response string.
-
-    Returns:
-        Dictionary with total evaluations, average score, per-dimension
-        averages, weakest/strongest dimensions, and recommendations list.
-    """
-    # Generate probes
-    generator = ProbeGenerator(original_history)
-    probes = generator.generate_probes()
-
-    # Evaluate each probe
-    evaluator = CompressionEvaluator()
-
-    for probe in probes:
-        # Get model response using compressed context
-        response = model_response_fn(compressed_context, probe.question)
-
-        # Evaluate response
-        evaluator.evaluate(probe, response, compressed_context)
-
-    # Get summary
-    summary = evaluator.get_summary()
-
-    # Add recommendations
-    summary["recommendations"] = []
-
-    if summary.get("weakest_dimension") == "artifact_trail":
-        summary["recommendations"].append(
-            "Consider implementing separate artifact tracking outside compression"
-        )
-
-    if summary.get("average_score", 0) < 3.5:
-        summary["recommendations"].append(
-            "Compression quality is below threshold - consider less aggressive compression"
-        )
-
-    return summary
-
-
-if __name__ == "__main__":
-    # Demo: generate probes and evaluate a sample compression
-
-    sample_history = """
-    User reported error: 401 Unauthorized on /api/auth/login endpoint.
-    Examined auth.controller.ts - JWT generation looks correct.
-    Examined middleware/cors.ts - no issues found.
-    Modified config/redis.ts: Fixed connection pooling configuration.
-    Modified services/session.service.ts: Added retry logic for transient failures.
-    Decided to use Redis connection pool instead of per-request connections.
-    Modified tests/auth.test.ts: Updated mock setup for new config.
-    14 tests passing, 2 failing (mock setup issues).
-    Next: Fix remaining test failures in session service mocks.
-    """
-
-    sample_compressed = """
-    ## Session Intent
-    Debug 401 Unauthorized on /api/auth/login.
-
-    ## Root Cause
-    Stale Redis connection in session store.
-
-    ## Files Modified
-    - config/redis.ts: Fixed connection pooling
-    - services/session.service.ts: Added retry logic
-    - tests/auth.test.ts: Updated mock setup
-
-    ## Test Status
-    14 passing, 2 failing
-
-    ## Next Steps
-    1. Fix remaining test failures
-    """
-
-    # Stub model response function
-    def mock_model_response(context: str, question: str) -> str:
-        if "error" in question.lower():
-            return "The original error was a 401 Unauthorized on /api/auth/login."
-        if "files" in question.lower():
-            return "Modified config/redis.ts, services/session.service.ts, tests/auth.test.ts."
-        if "next" in question.lower():
-            return "Fix remaining test failures in session service mocks."
-        if "decision" in question.lower():
-            return "Decided to use Redis connection pool instead of per-request connections."
-        return "No specific information available."
-
-    # Run evaluation
-    result = evaluate_compression_quality(
-        original_history=sample_history,
-        compressed_context=sample_compressed,
-        model_response_fn=mock_model_response,
-    )
-
-    print("=== Compression Quality Evaluation ===")
-    print(f"Total evaluations: {result['total_evaluations']}")
-    print(f"Average score: {result['average_score']:.2f}")
-    print()
-    print("Dimension averages:")
-    for dim, score in result.get("dimension_averages", {}).items():
-        print(f"  {dim}: {score:.2f}")
-    print()
-    print(f"Weakest dimension: {result.get('weakest_dimension')}")
-    print(f"Strongest dimension: {result.get('strongest_dimension')}")
-    print()
-    if result.get("recommendations"):
-        print("Recommendations:")
-        for rec in result["recommendations"]:
-            print(f"  - {rec}")
-    else:
-        print("No recommendations - compression quality looks acceptable.")