@cohortapp/agent-sdk 2.3.0

package/scripts/self-optimization/compute-metrics.py

@@ -0,0 +1,398 @@
+#!/usr/bin/env python3
+"""
+Self-Optimization Metric Computation
+Computes current values for all scoring rubrics from operational logs.
+
+Usage:
+    python3 scripts/self-optimization/compute-metrics.py [--date YYYY-MM-DD] [--append]
+
+    --date    Date to compute metrics for (default: today)
+    --append  Append snapshot to self-optimization/metric-snapshots.yaml
+
+Reads from:
+    logs/audit/{date}-actions.jsonl
+    logs/audit/{date}-validation.jsonl
+    logs/workflows/{date}-inbox-processor.jsonl
+    logs/daemon/{date}-responses.jsonl
+    logs/sessions/{date}-sessions.jsonl
+    state/queues/*.yaml
+
+Outputs metric snapshot to stdout (YAML format) or appends to metric-snapshots.yaml.
+"""
+
+import json
+import os
+import sys
+import glob
+import argparse
+from datetime import datetime, timezone, timedelta
+from pathlib import Path
+from collections import defaultdict
+
+# Repo root — script expects to be run from repo root or adjusts via __file__
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+
+
+def _load_principal_slug():
+    """Load the principal's slug from config/agent.json (SOT).
+
+    Falls back to "principal" if agent.json is unavailable; metrics tagged
+    with the literal string "principal" then no-op gracefully rather than
+    being matched against a hardcoded name.
+    """
+    try:
+        with open(REPO_ROOT / "config" / "agent.json") as fh:
+            agent = json.load(fh)
+    except Exception:
+        return "principal"
+    p = agent.get("principal") or {}
+    full = p.get("fullName") or f"{p.get('firstName', '')} {p.get('lastName', '')}".strip()
+    return full.lower().replace(" ", "-") if full else "principal"
+
+
+PRINCIPAL_SLUG = _load_principal_slug()
+
+
+def parse_jsonl(filepath):
+    """Parse a JSONL file, skipping malformed lines."""
+    entries = []
+    if not filepath.exists():
+        return entries
+    with open(filepath, 'r') as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entries.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+    return entries
+
+
+def count_queue_items(queues_dir, statuses=None):
+    """Count items across all queue YAML files by status."""
+    counts = defaultdict(int)
+    total = 0
+    if not queues_dir.exists():
+        return counts, total
+
+    for qfile in queues_dir.glob('*.yaml'):
+        try:
+            with open(qfile, 'r') as f:
+                content = f.read()
+        except Exception:
+            continue
+
+        # Simple YAML parsing for queue items — avoids pyyaml dependency
+        # Looks for "status: <value>" lines under "items:" sections
+        in_items = False
+        for line in content.split('\n'):
+            stripped = line.strip()
+            if stripped == 'items:':
+                in_items = True
+                continue
+            if in_items and stripped.startswith('status:'):
+                status = stripped.split(':', 1)[1].strip().strip('"').strip("'")
+                counts[status] += 1
+                total += 1
+            # Reset if we hit a top-level key
+            if in_items and not line.startswith(' ') and not line.startswith('\t') and ':' in line and not stripped.startswith('-') and not stripped.startswith('status:'):
+                if stripped != 'items:':
+                    in_items = False
+
+    return counts, total
+
+
+def compute_metrics(date_str):
+    """Compute all scoring rubric metrics for a given date."""
+    logs_audit = REPO_ROOT / 'logs' / 'audit'
+    logs_workflows = REPO_ROOT / 'logs' / 'workflows'
+    logs_daemon = REPO_ROOT / 'logs' / 'daemon'
+    logs_sessions = REPO_ROOT / 'logs' / 'sessions'
+    queues_dir = REPO_ROOT / 'state' / 'queues'
+
+    # Load log files
+    actions = parse_jsonl(logs_audit / f'{date_str}-actions.jsonl')
+    validations = parse_jsonl(logs_audit / f'{date_str}-validation.jsonl')
+    inbox_logs = parse_jsonl(logs_workflows / f'{date_str}-inbox-processor.jsonl')
+    responses = parse_jsonl(logs_daemon / f'{date_str}-responses.jsonl')
+    sessions = parse_jsonl(logs_sessions / f'{date_str}-sessions.jsonl')
+
+    # Also check prior days for multi-day metrics
+    prev_date = (datetime.strptime(date_str, '%Y-%m-%d') - timedelta(days=1)).strftime('%Y-%m-%d')
+    prev_actions = parse_jsonl(logs_audit / f'{prev_date}-actions.jsonl')
+
+    metrics = {}
+
+    # --- Responsiveness ---
+
+    # inbox_to_action_time: Average time from inbox item to first action
+    # Proxy: count responses with duration_ms from daemon response logs
+    response_times_ms = [r.get('duration_ms', 0) for r in responses if r.get('duration_ms')]
+    if response_times_ms:
+        avg_response_ms = sum(response_times_ms) / len(response_times_ms)
+        metrics['inbox_to_action_time'] = {
+            'value': round(avg_response_ms / 1000, 1),
+            'unit': 'seconds',
+            'sample_size': len(response_times_ms),
+            'note': 'Average daemon response generation time (proxy for inbox-to-action)'
+        }
+    else:
+        metrics['inbox_to_action_time'] = {'value': None, 'unit': 'seconds', 'note': 'No response data available'}
+
+    # holding_note_time: Measured from actions log where action_type contains "holding"
+    holding_actions = [a for a in actions if 'holding' in str(a.get('action_type', '')).lower()
+                       or 'holding' in str(a.get('result', '')).lower()]
+    metrics['holding_note_time'] = {
+        'value': len(holding_actions),
+        'unit': 'count',
+        'note': f'{len(holding_actions)} holding notes sent today'
+    }
+
+    # --- Throughput ---
+
+    # queue_throughput: Items resolved/closed today
+    queue_counts, queue_total = count_queue_items(queues_dir)
+    resolved_closed = queue_counts.get('resolved', 0) + queue_counts.get('closed', 0)
+    metrics['queue_throughput'] = {
+        'value': resolved_closed,
+        'unit': 'items',
+        'total_items': queue_total,
+        'breakdown': dict(queue_counts),
+        'note': f'{resolved_closed} resolved/closed out of {queue_total} total queue items'
+    }
+
+    # queue_age: computed from queue items with created dates — would need YAML parsing
+    open_items = queue_counts.get('open', 0) + queue_counts.get('in_progress', 0)
+    metrics['queue_age'] = {
+        'value': open_items,
+        'unit': 'open_items',
+        'note': f'{open_items} items currently open/in_progress (age computation requires YAML date parsing)'
+    }
+
+    # --- Accuracy ---
+
+    # classification_accuracy: from inbox processor logs
+    total_classified = 0
+    classifications = {}
+    for entry in inbox_logs:
+        if 'classifications' in entry:
+            cls = entry['classifications']
+            if isinstance(cls, dict):
+                # Format: {"action_required": 9, "fyi": 15, "ignore": 7}
+                for category, count in cls.items():
+                    classifications[category] = classifications.get(category, 0) + count
+                    total_classified += count
+            elif isinstance(cls, list):
+                # Format: [{"id": ..., "class": "fyi", ...}, ...]
+                for item in cls:
+                    cat = item.get('class', 'unknown')
+                    classifications[cat] = classifications.get(cat, 0) + 1
+                    total_classified += 1
+        if 'items_processed' in entry and 'classifications' not in entry:
+            total_classified += entry.get('items_processed', 0)
+
+    metrics['classification_accuracy'] = {
+        'value': total_classified,
+        'unit': 'items_classified',
+        'breakdown': classifications,
+        'note': f'{total_classified} items classified today. Accuracy requires reclassification tracking (not yet instrumented).'
+    }
+
+    # duplicate_reply_rate: from validation logs — count entries with issues
+    total_validations = len(validations)
+    issues_found = sum(1 for v in validations if v.get('issues_found', 0) > 0)
+    blockers_found = sum(1 for v in validations if v.get('blockers', 0) > 0)
+    metrics['duplicate_reply_rate'] = {
+        'value': blockers_found,
+        'unit': 'blocked_sends',
+        'total_validations': total_validations,
+        'issues_found': issues_found,
+        'note': f'{blockers_found} sends blocked, {issues_found} issues out of {total_validations} validations'
+    }
+
+    # factual_accuracy: requires manual tracking — report as unmeasured
+    metrics['factual_accuracy'] = {
+        'value': None,
+        'unit': 'corrections',
+        'note': 'Requires manual tracking of corrections from recipients'
+    }
+
+    # --- Follow-through ---
+
+    # follow_through_rate & sla_breach_rate: from queue SLA fields
+    metrics['follow_through_rate'] = {
+        'value': None,
+        'unit': 'percentage',
+        'note': 'Requires SLA deadline comparison (not yet instrumented in metric script)'
+    }
+
+    metrics['sla_breach_rate'] = {
+        'value': None,
+        'unit': 'percentage',
+        'note': 'Requires SLA deadline comparison (not yet instrumented in metric script)'
+    }
+
+    # --- CEO Satisfaction ---
+
+    # ceo_correction_rate & ceo_positive_signal_rate: from interaction patterns.
+    # Principal slug resolved from config/agent.json (SOT) — never hardcoded.
+    ceo_responses = [r for r in responses if r.get('sender') == PRINCIPAL_SLUG]
+    metrics['ceo_correction_rate'] = {
+        'value': len(ceo_responses),
+        'unit': 'ceo_interactions',
+        'note': f'{len(ceo_responses)} CEO interactions today. Correction/positive signal classification not yet instrumented.'
+    }
+    metrics['ceo_positive_signal_rate'] = {
+        'value': None,
+        'unit': 'percentage',
+        'note': 'Requires sentiment classification of CEO responses'
+    }
+
+    # --- System Health ---
+
+    # error_rate: from workflow logs — count entries with error/failed status
+    all_workflow_logs = []
+    for wf_file in logs_workflows.glob(f'{date_str}-*.jsonl'):
+        all_workflow_logs.extend(parse_jsonl(wf_file))
+
+    total_workflow_entries = len(all_workflow_logs)
+    error_entries = sum(1 for e in all_workflow_logs
+                       if e.get('status') in ('error', 'failed')
+                       or 'error' in str(e.get('status', '')).lower())
+    metrics['error_rate'] = {
+        'value': round(error_entries / max(total_workflow_entries, 1) * 100, 1),
+        'unit': 'percentage',
+        'errors': error_entries,
+        'total_entries': total_workflow_entries,
+        'note': f'{error_entries} errors in {total_workflow_entries} workflow log entries'
+    }
+
+    # harness_complexity: line count across key config files
+    complexity_dirs = [
+        REPO_ROOT / 'agents',
+        REPO_ROOT / 'workflows',
+        REPO_ROOT / 'policies',
+        REPO_ROOT / 'schedules',
+    ]
+    total_lines = 0
+    file_count = 0
+    for d in complexity_dirs:
+        if d.exists():
+            for f in d.rglob('*'):
+                if f.is_file() and f.suffix in ('.md', '.yaml', '.yml'):
+                    try:
+                        total_lines += sum(1 for _ in open(f))
+                        file_count += 1
+                    except Exception:
+                        pass
+
+    metrics['harness_complexity'] = {
+        'value': total_lines,
+        'unit': 'lines',
+        'file_count': file_count,
+        'dirs': ['agents/', 'workflows/', 'policies/', 'schedules/'],
+        'note': f'{total_lines} total lines across {file_count} config files'
+    }
+
+    # context_recall_accuracy & entity_index_coverage: require manual assessment
+    metrics['context_recall_accuracy'] = {
+        'value': None,
+        'unit': 'percentage',
+        'note': 'Requires manual assessment during weekly review'
+    }
+
+    # entity_index_coverage: check memory indexes
+    entity_index = REPO_ROOT / 'memory' / 'indexes' / 'entity-relationships.yaml'
+    profiles_dir = REPO_ROOT / 'memory' / 'profiles' / 'users'
+    profile_count = len(list(profiles_dir.glob('*.yaml'))) if profiles_dir.exists() else 0
+    metrics['entity_index_coverage'] = {
+        'value': profile_count,
+        'unit': 'user_profiles',
+        'index_exists': entity_index.exists(),
+        'note': f'{profile_count} user profiles in memory. Index exists: {entity_index.exists()}'
+    }
+
+    # --- Summary stats ---
+    total_sends = 0
+    send_counter = REPO_ROOT / 'logs' / 'audit' / 'send-counter.yaml'
+    if send_counter.exists():
+        with open(send_counter, 'r') as f:
+            for line in f:
+                if 'total:' in line and 'totals' not in line:
+                    try:
+                        total_sends = int(line.split(':')[1].strip())
+                    except (ValueError, IndexError):
+                        pass
+
+    metrics['_summary'] = {
+        'total_actions_logged': len(actions),
+        'total_validations': total_validations,
+        'total_workflow_entries': total_workflow_entries,
+        'total_daemon_responses': len(responses),
+        'total_sessions': len(sessions),
+        'total_sends_today': total_sends,
+        'total_queue_items': queue_total,
+    }
+
+    return metrics
+
+
+def format_yaml(metrics, date_str):
+    """Format metrics as YAML snapshot."""
+    lines = [f'- date: "{date_str}"']
+    lines.append(f'  computed_at: "{datetime.now(timezone.utc).isoformat()}"')
+    lines.append('  metrics:')
+
+    for key, val in metrics.items():
+        if key == '_summary':
+            lines.append('  summary:')
+            for sk, sv in val.items():
+                lines.append(f'    {sk}: {sv}')
+            continue
+
+        lines.append(f'    {key}:')
+        for vk, vv in val.items():
+            if isinstance(vv, dict):
+                lines.append(f'      {vk}:')
+                for dk, dv in vv.items():
+                    lines.append(f'        {dk}: {json.dumps(dv) if isinstance(dv, str) else dv}')
+            elif isinstance(vv, str):
+                lines.append(f'      {vk}: "{vv}"')
+            elif vv is None:
+                lines.append(f'      {vk}: null')
+            else:
+                lines.append(f'      {vk}: {vv}')
+
+    return '\n'.join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Compute self-optimization metrics')
+    parser.add_argument('--date', default=datetime.now().strftime('%Y-%m-%d'),
+                        help='Date to compute metrics for (YYYY-MM-DD)')
+    parser.add_argument('--append', action='store_true',
+                        help='Append to metric-snapshots.yaml')
+    args = parser.parse_args()
+
+    metrics = compute_metrics(args.date)
+    yaml_output = format_yaml(metrics, args.date)
+
+    if args.append:
+        snapshots_path = REPO_ROOT / 'self-optimization' / 'metric-snapshots.yaml'
+        mode = 'a' if snapshots_path.exists() else 'w'
+        with open(snapshots_path, mode) as f:
+            if mode == 'w':
+                f.write('# Self-Optimization Metric Snapshots\n')
+                f.write('# Computed by: scripts/self-optimization/compute-metrics.py\n')
+                f.write('# Read by: evening wrap, weekly review, experiment evaluator\n\n')
+                f.write('snapshots:\n')
+            f.write('\n' + yaml_output + '\n')
+        print(f'Snapshot appended to {snapshots_path}', file=sys.stderr)
+    else:
+        print(yaml_output)
+
+
+if __name__ == '__main__':
+    main()