@microsoft/m365-copilot-eval 1.0.1-preview.1

package/src/clients/cli/generate_report.py

@@ -0,0 +1,251 @@
+import json
+import markdown
+import pandas as pd
+from pathlib import Path
+
+def calculate_aggregate_statistics(results):
+    """Calculate aggregate statistics across all evaluation results."""
+    if not results:
+        return {}
+    
+    # Extract all metrics from the first result to know what metrics we have
+    first_result = results[0]
+    metrics = first_result.get('results', {})
+    
+    aggregates = {}
+    
+    for metric_key in metrics.keys():
+        if not metric_key.endswith('_score'):
+            continue
+            
+        metric_name = metric_key[:-6]  # Remove '_score' suffix
+        metric_display_name = metric_name.replace('_', ' ').title()
+        
+        scores = []
+        pass_count = 0
+        fail_count = 0
+        threshold_value = None
+        
+        for result in results:
+            metric_data = result.get('results', {}).get(metric_key)
+            if metric_data:
+                try:
+                    # Parse the JSON string to get the actual data
+                    parsed_data = json.loads(metric_data) if isinstance(metric_data, str) else metric_data
+                    
+                    # Extract score, result, and threshold
+                    score = parsed_data.get('score')
+                    if score is None:
+                        score = parsed_data.get(metric_name) 
+                    if score is None:
+                        score = parsed_data.get(f'{metric_name}_score')
+
+                    result_status = parsed_data.get('result')
+                    if result_status is None:
+                        result_status = parsed_data.get(f'{metric_name}_result')
+
+                    threshold = parsed_data.get('threshold')
+                    if threshold is None:
+                        threshold = parsed_data.get(f'{metric_name}_threshold')
+                    
+                    if score is not None:
+                        scores.append(float(score))
+                    
+                    if result_status:
+                        if str(result_status).lower() == 'pass':
+                            pass_count += 1
+                        elif str(result_status).lower() == 'fail':
+                            fail_count += 1
+                    
+                    # Capture threshold (should be consistent across all results)
+                    if threshold is not None and threshold_value is None:
+                        threshold_value = threshold
+                            
+                except (json.JSONDecodeError, ValueError, TypeError):
+                    continue
+        
+        if scores:
+            avg_score = sum(scores) / len(scores)
+            total_evaluated = pass_count + fail_count
+            pass_rate = (pass_count / total_evaluated * 100) if total_evaluated > 0 else 0
+            
+            aggregates[metric_display_name] = {
+                'total_prompts': len(results),
+                'total_evaluated': total_evaluated,
+                'pass_count': pass_count,
+                'fail_count': fail_count,
+                'pass_rate': pass_rate,
+                'avg_score': avg_score,
+                'threshold': threshold_value,
+                'scores': scores
+            }
+    
+    return aggregates
+
+def parse_score(score_str):
+    try:
+        return json.loads(score_str)
+    except Exception:
+        return {}
+
+def format_score(score):
+  try:
+    val = float(score)
+  except (TypeError, ValueError):
+    return score
+  if val.is_integer():
+    return str(int(val))
+  s = f"{val:.3f}".rstrip('0').rstrip('.')
+  return s or "0"
+
+def extract_metric_rows(entry):
+    """
+    Build generic metric rows from any `*_score` keys on an entry.
+    Each row has: Metric, Result, Score, Threshold, Reason.
+    Supports metrics under entry['results'] and falls back to top-level for backward compatibility.
+    """
+    rows = []
+
+    def pick(d, candidates):
+        for k in candidates:
+            if k in d and d[k] not in (None, ''):
+                return d[k]
+        return ''
+
+    def iter_score_fields(e):
+        container = e.get('results') if isinstance(e, dict) else None
+        if isinstance(container, dict):
+            for k, v in container.items():
+                if isinstance(k, str) and k.endswith('_score'):
+                    yield k, v
+            return
+        # fallback to top-level flat structure
+        for k, v in e.items():
+            if isinstance(k, str) and k.endswith('_score'):
+                yield k, v
+
+    for key, raw in iter_score_fields(entry):
+        metric_id = key[:-6]  # strip "_score"
+        metric_obj = parse_score(raw) if isinstance(raw, (str, bytes)) else (raw or {})
+
+        display_name = metric_id.replace('_', ' ').title()
+
+        # Candidate key patterns inside the parsed metric object
+        score_val = pick(metric_obj, [metric_id, f'{metric_id}_score', 'score', 'value'])
+        result_val = pick(metric_obj, [f'{metric_id}_result', 'result', 'status'])
+        threshold_val = pick(metric_obj, [f'{metric_id}_threshold', 'threshold', 'min_threshold', 'expected'])
+        reason_val = pick(metric_obj, [f'{metric_id}_reason', 'reason', 'rationale', 'explanation'])
+
+        rows.append({
+            'Metric': display_name,
+            'Result': str(result_val).lower() if isinstance(result_val, str) else result_val,
+            'Score': format_score(score_val),
+            'Threshold': format_score(threshold_val),
+            'Reason': reason_val
+        })
+    return rows
+
+def generate_html_report(results):
+    html = [
+      '<!DOCTYPE html>',
+      '<html lang="en">',
+      '<head>',
+      '    <meta charset="UTF-8">',
+      '    <title>M365 Copilot Agents Evaluation Scores Report</title>',
+      '    <style>',
+      '        body { font-family: Arial, sans-serif; margin: 2em; padding: 1.5em; background: #fafafa; }',
+      '        h1 { margin-top: 0; color: #2c3e50; }',
+      '        h2 { color: #34495e; margin-top: 2em; }',
+      '        table { border-collapse: collapse; width: 100%; margin: 1.5em 0; }',
+      '        th, td { border: 1px solid #ccc; padding: 10px 12px; text-align: left; vertical-align: top; }',
+      '        th { background: #f4f4f4; font-weight: 600; }',
+      '        details { padding: 0.75em 1em; border: 1px solid #ddd; border-radius: 6px; background: #fff; }',
+      '        details summary { cursor: pointer; font-weight: 600; margin: -0.75em -1em 0.75em; padding: 0.75em 1em; background: #eef2f5; border-bottom: 1px solid #ddd; }',
+      '        .pass { background: #d4edda; color: #155724; }',
+      '        .fail { background: #f8d7da; color: #721c24; }',
+      '        .score-details { font-size: 0.95em; color: #333; background: #f9f9f9; }',
+      '        .score-details table { margin-top: 0.5em !important; }',
+      '        .score-details th { background: #e9ecef; }',
+      '        .aggregate-section { margin: 2em 0; }',
+      '        .aggregate-section h2 { color: #34495e; margin-top: 0; }',
+      '        .pass-rate-excellent { background: #d1f2eb; color: #186a3b; font-weight: bold; }',
+      '        .pass-rate-good { background: #fef9e7; color: #7d6608; font-weight: bold; }',
+      '        .pass-rate-poor { background: #fadbd8; color: #b03a2e; font-weight: bold; }',
+      '        .individual-results { margin-top: 3em; }',
+      '    </style>',
+      '</head>',
+      '<body>',
+      '    <h1> M365 Copilot Agents Evaluation Report</h1>',
+    ]
+
+    # Add aggregate statistics if multiple results
+    if len(results) > 1:
+        aggregates = calculate_aggregate_statistics(results)
+        if aggregates:
+            html.append('<div class="aggregate-section">')
+            html.append(f'<h2> Aggregate Statistics ({len(results)} prompts evaluated)</h2>')
+            
+            # Create aggregate table with same style as individual results
+            html.append('<table>')
+            html.append('<tr><th>Metric</th><th>Pass Rate</th><th>Passed</th><th>Failed</th><th>Avg Score</th><th>Threshold</th></tr>')
+            
+            for metric_name, stats in aggregates.items():
+                pass_rate_class = 'pass-rate-excellent' if stats['pass_rate'] >= 80 else 'pass-rate-good' if stats['pass_rate'] >= 60 else 'pass-rate-poor'
+                threshold_display = stats.get('threshold', 'N/A')
+                html.append(f'''
+                <tr>
+                    <td><strong>{metric_name}</strong></td>
+                    <td class="{pass_rate_class}">{stats['pass_rate']:.1f}%</td>
+                    <td class="pass">{stats['pass_count']}</td>
+                    <td class="fail">{stats['fail_count']}</td>
+                    <td>{stats['avg_score']:.2f}</td>
+                    <td>{threshold_display}</td>
+                </tr>
+                ''')
+            
+            html.append('</table>')
+            html.append('</div>')
+
+    # Individual results section
+    html.append('<div class="individual-results">')
+    html.append('<h2> Individual Results</h2>')
+
+    for idx, entry in enumerate(results, 1):
+        html.append(f'<h3>Prompt {idx}: {entry.get("prompt", "")}</h3>')
+        html.append('<table>')
+        html.append('<tr><th>Response</th><td>{}</td></tr>'.format(markdown.markdown(entry.get("response", ""))))
+        html.append('<tr><th>Expected Response</th><td>{}</td></tr>'.format(markdown.markdown(entry.get("expected_response", ""))))
+        html.append('</table>')
+
+        score_rows = extract_metric_rows(entry)
+        score_df = pd.DataFrame(score_rows)
+
+        def highlight_result(val):
+            lv = str(val).lower()
+            if lv == 'pass':
+                return 'background-color: #d4edda; color: #155724;'
+            elif lv == 'fail':
+                return 'background-color: #f8d7da; color: #721c24;'
+            return ''
+
+        score_html = (
+            score_df.style
+            .map(highlight_result, subset=['Result'])
+            .set_table_attributes('style="margin-top:1em;"')
+            .hide(axis="index")
+            .to_html()
+        )
+
+        html.append('<details open class="score-details"><summary>Show/Hide Evaluation Metric Scores</summary>')
+        html.append(score_html)
+        html.append('</details>')
+
+        html.append(f'<p><em>Entry {idx} of {len(results)}</em></p>')
+        if idx < len(results):  # Don't add HR after last item
+            html.append('<hr>')
+
+    html.append('</div>')  # Close individual-results
+    html.append('</body></html>')
+    html.append('<p><small>Generated by M365 Copilot Agents Evaluation CLI.</small></p>')
+
+    return '\n'.join(html)