@rm0nroe/coach-claw 1.0.6

package/coach/tests/test_aggregate_facets.py

@@ -0,0 +1,379 @@
+"""Unit tests for coach/bin/aggregate_facets.py.
+
+Mock facets/*.json sidecar fixtures, assert threshold-based emit shape
+matches what merge.py expects on its --detections input.
+"""
+from __future__ import annotations
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import aggregate_facets
+
+
+def _write_facet(dir_: Path, name: str, payload: dict) -> Path:
+    p = dir_ / f"{name}.json"
+    p.write_text(json.dumps(payload))
+    return p
+
+
+def _make_session(
+    *,
+    friction: dict | None = None,
+    primary_success: str | None = None,
+    friction_detail: str = "",
+    brief_summary: str = "",
+    session_id: str = "test-session",
+) -> dict:
+    out: dict = {"session_id": session_id}
+    if friction is not None:
+        out["friction_counts"] = friction
+    if primary_success is not None:
+        out["primary_success"] = primary_success
+    if friction_detail:
+        out["friction_detail"] = friction_detail
+    if brief_summary:
+        out["brief_summary"] = brief_summary
+    return out
+
+
+def test_friction_counts_emits_negative_detection(tmp_path: Path) -> None:
+    """friction_counts.misunderstood_request in 4/5 sessions → emit negative."""
+    for i in range(4):
+        _write_facet(
+            tmp_path,
+            f"s{i}",
+            _make_session(
+                friction={"misunderstood_request": 2},
+                friction_detail=f"session {i} got off-track on the first attempt",
+                session_id=f"s{i}",
+            ),
+        )
+    _write_facet(tmp_path, "s4", _make_session(session_id="s4"))
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    ids = [d["id"] for d in dets]
+    assert "misunderstood-request" in ids
+    det = next(d for d in dets if d["id"] == "misunderstood-request")
+    assert det["direction"] == "negative"
+    assert det["ratio"] == 0.8
+    assert det["n_sessions"] == 5
+
+
+def test_primary_success_emits_positive_detection(tmp_path: Path) -> None:
+    """6/10 sessions with primary_success=good_debugging → emit positive."""
+    for i in range(6):
+        _write_facet(
+            tmp_path,
+            f"s{i}",
+            _make_session(
+                primary_success="good_debugging",
+                brief_summary=f"session {i}: drove the bug to root cause",
+                session_id=f"s{i}",
+            ),
+        )
+    for i in range(6, 10):
+        _write_facet(tmp_path, f"s{i}", _make_session(session_id=f"s{i}"))
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    ids = [d["id"] for d in dets]
+    assert "good-debugging" in ids
+    det = next(d for d in dets if d["id"] == "good-debugging")
+    assert det["direction"] == "positive"
+    assert det["ratio"] == 0.6
+
+
+def test_below_threshold_drops_detection(tmp_path: Path) -> None:
+    """Friction in 2/10 sessions (<25%) → not emitted."""
+    for i in range(2):
+        _write_facet(
+            tmp_path,
+            f"s{i}",
+            _make_session(friction={"buggy_code": 1}, session_id=f"s{i}"),
+        )
+    for i in range(2, 10):
+        _write_facet(tmp_path, f"s{i}", _make_session(session_id=f"s{i}"))
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    assert "buggy-code" not in [d["id"] for d in dets]
+
+
+def test_strength_threshold_higher_than_negative(tmp_path: Path) -> None:
+    """5/10 (50%) primary_success does NOT emit; needs ≥60%."""
+    for i in range(5):
+        _write_facet(
+            tmp_path,
+            f"s{i}",
+            _make_session(primary_success="multi_file_changes", session_id=f"s{i}"),
+        )
+    for i in range(5, 10):
+        _write_facet(tmp_path, f"s{i}", _make_session(session_id=f"s{i}"))
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    assert "multi-file-changes" not in [d["id"] for d in dets]
+
+
+def test_strength_at_threshold_emits(tmp_path: Path) -> None:
+    """Exactly 60% (6/10) primary_success → emits."""
+    for i in range(6):
+        _write_facet(
+            tmp_path,
+            f"s{i}",
+            _make_session(primary_success="multi_file_changes", session_id=f"s{i}"),
+        )
+    for i in range(6, 10):
+        _write_facet(tmp_path, f"s{i}", _make_session(session_id=f"s{i}"))
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    assert "multi-file-changes" in [d["id"] for d in dets]
+
+
+def test_id_kebab_normalization(tmp_path: Path) -> None:
+    """friction_counts underscore keys emit kebab-case ids."""
+    for i in range(3):
+        _write_facet(
+            tmp_path,
+            f"s{i}",
+            _make_session(
+                friction={"misunderstood_request": 1, "wrong_approach": 1},
+                session_id=f"s{i}",
+            ),
+        )
+    _write_facet(tmp_path, "s3", _make_session(session_id="s3"))
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    ids = {d["id"] for d in dets}
+    # Both 3/4 = 75% > 25% → both should emit.
+    assert "misunderstood-request" in ids
+    assert "wrong-approach" in ids
+    # No underscores in any id.
+    for d in dets:
+        assert "_" not in d["id"]
+
+
+def test_examples_capped_and_redacted(tmp_path: Path) -> None:
+    """5 friction_detail strings → capped at 3, each ≤120 chars; file paths
+    redacted."""
+    raw_examples = [
+        "Edited /Users/foo/project/src/main.py and broke the build for an hour",
+        "The settings.py change cascaded into a migration regression",
+        "Wrong approach in /tmp/bar/src/handler.go before we caught it on PR",
+        "Went down the wrong rabbit hole on test_runner.ts for 30 minutes",
+        "Misread the spec — the README.md said the opposite of what I assumed",
+    ]
+    for i, detail in enumerate(raw_examples):
+        _write_facet(
+            tmp_path,
+            f"s{i}",
+            _make_session(
+                friction={"misunderstood_request": 1},
+                friction_detail=detail,
+                session_id=f"s{i}",
+            ),
+        )
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    det = next(d for d in dets if d["id"] == "misunderstood-request")
+    assert len(det["examples"]) == 3
+    for ex in det["examples"]:
+        assert len(ex) <= 120
+        # File-path tokens redacted.
+        assert "/Users/foo/" not in ex
+        assert "/tmp/bar/" not in ex
+        # File-extension tokens redacted.
+        assert "settings.py" not in ex
+        assert "handler.go" not in ex
+        assert "test_runner.ts" not in ex
+        assert "README.md" not in ex
+
+
+def test_window_filtering(tmp_path: Path) -> None:
+    """Facets older than the window are dropped."""
+    import os
+    import time as _time
+
+    # Stale: mtime 14d ago.
+    stale_payload = _make_session(friction={"misunderstood_request": 5}, session_id="stale")
+    p_stale = _write_facet(tmp_path, "stale", stale_payload)
+    stale_ts = _time.time() - 14 * 86400
+    os.utime(p_stale, (stale_ts, stale_ts))
+
+    # Fresh: today.
+    for i in range(3):
+        _write_facet(
+            tmp_path,
+            f"fresh{i}",
+            _make_session(friction={"buggy_code": 1}, session_id=f"fresh{i}"),
+        )
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    ids = [d["id"] for d in dets]
+    # Stale not counted: only 3 sessions in window, all with buggy_code → emit it.
+    assert "buggy-code" in ids
+    # Stale's misunderstood_request should NOT emit.
+    assert "misunderstood-request" not in ids
+    det = next(d for d in dets if d["id"] == "buggy-code")
+    assert det["n_sessions"] == 3
+
+
+def test_missing_facets_dir_returns_empty(tmp_path: Path) -> None:
+    """Nonexistent facets dir → empty list, no crash."""
+    nonexistent = tmp_path / "nope"
+    dets = aggregate_facets.aggregate(nonexistent, window_days=7, cap=8)
+    assert dets == []
+
+
+def test_empty_facets_dir_returns_empty(tmp_path: Path) -> None:
+    """Existing but empty facets dir → empty list."""
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    assert dets == []
+
+
+def test_malformed_json_skipped(tmp_path: Path) -> None:
+    """Malformed JSON files are skipped silently."""
+    (tmp_path / "broken.json").write_text("not valid json {{{")
+    for i in range(3):
+        _write_facet(
+            tmp_path,
+            f"s{i}",
+            _make_session(friction={"buggy_code": 1}, session_id=f"s{i}"),
+        )
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    assert "buggy-code" in [d["id"] for d in dets]
+
+
+def test_cap_enforced(tmp_path: Path) -> None:
+    """cap=2 limits output to 2 detections, highest ratio first."""
+    keys = ["misunderstood_request", "wrong_approach", "buggy_code", "edge_case"]
+    # All four hit 100%.
+    for i in range(5):
+        _write_facet(
+            tmp_path,
+            f"s{i}",
+            _make_session(
+                friction={k: 1 for k in keys},
+                session_id=f"s{i}",
+            ),
+        )
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=2)
+    assert len(dets) == 2
+
+
+def test_schema_shape_matches_merge_input(tmp_path: Path) -> None:
+    """Detection objects must carry the fields merge.py reads."""
+    for i in range(3):
+        _write_facet(
+            tmp_path,
+            f"s{i}",
+            _make_session(
+                friction={"misunderstood_request": 1},
+                friction_detail=f"detail {i}",
+                session_id=f"s{i}",
+            ),
+        )
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    assert dets, "expected at least one detection"
+    for d in dets:
+        assert "id" in d and isinstance(d["id"], str) and d["id"]
+        assert d["direction"] in ("positive", "negative")
+        assert "name" in d
+        assert "nudge" in d
+        assert "examples" in d and isinstance(d["examples"], list)
+        assert d.get("source") == "insights-weekly"
+
+
+def test_zero_count_friction_not_emitted(tmp_path: Path) -> None:
+    """friction_counts entries with count=0 are not treated as present."""
+    for i in range(5):
+        _write_facet(
+            tmp_path,
+            f"s{i}",
+            _make_session(friction={"misunderstood_request": 0}, session_id=f"s{i}"),
+        )
+
+    dets = aggregate_facets.aggregate(tmp_path, window_days=7, cap=8)
+    assert "misunderstood-request" not in [d["id"] for d in dets]
+
+
+# --- CLI-level evidence gate (v0.5.1 P1 #1b) -------------------------------
+# `aggregate()` continues to return [] for empty/missing dirs (those tests
+# above stay green). The CLI `main()` adds an "evidence gate": if
+# n_sessions == 0 in the requested window, exit 3 (EXIT_NO_EVIDENCE) and
+# print no JSON to stdout. The wrapper translates that to its own exit 7.
+# Reasoning: empty detections WITH n_sessions > 0 is valid (clean week,
+# merges normally); empty detections WITH n_sessions == 0 is no evidence
+# and must NOT advance absence-based streaks.
+
+SCRIPT = Path(__file__).resolve().parent.parent / "bin" / "aggregate_facets.py"
+
+
+def test_no_sessions_in_window_returns_3(tmp_path: Path) -> None:
+    """Empty facets dir → CLI exits 3, prints no JSON to stdout, prints
+    a clear stderr message naming the window. Pinned by
+    aggregate_facets.EXIT_NO_EVIDENCE."""
+    empty = tmp_path / "empty-facets"
+    empty.mkdir()
+
+    result = subprocess.run(
+        [sys.executable, str(SCRIPT), "--facets-dir", str(empty), "--window-days", "7"],
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == aggregate_facets.EXIT_NO_EVIDENCE == 3, (
+        f"expected exit 3, got {result.returncode}\nstderr={result.stderr}"
+    )
+    assert "no sessions in last 7 days" in result.stderr
+    assert "refusing to emit detections" in result.stderr
+    # Stdout MUST be empty so a caller piping stdout into merge gets a
+    # parse error instead of a silent `[]` merge.
+    assert result.stdout.strip() == "", (
+        f"stdout should be empty when bailing on no-evidence: {result.stdout!r}"
+    )
+
+
+def test_nonexistent_facets_dir_cli_returns_3(tmp_path: Path) -> None:
+    """Same gate fires when --facets-dir doesn't exist (the function
+    returns []; the CLI catches it via the n_sessions recount)."""
+    nonexistent = tmp_path / "does-not-exist"
+
+    result = subprocess.run(
+        [sys.executable, str(SCRIPT), "--facets-dir", str(nonexistent)],
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 3
+    assert "no sessions" in result.stderr
+    assert result.stdout.strip() == ""
+
+
+def test_session_with_no_detections_still_exits_0(tmp_path: Path) -> None:
+    """The gate fires on n_sessions==0, NOT on detections==0. A clean
+    session (no friction, no primary_success) with n_sessions=1 emits
+    no detections but is a legitimate clean signal and must merge as
+    `[]` — exit 0, NOT exit 3. This pins the asymmetry from
+    `_session_with_no_detections_still_exits_0` vs the no-evidence
+    gate."""
+    _write_facet(tmp_path, "s0", _make_session(session_id="s0"))
+
+    result = subprocess.run(
+        [sys.executable, str(SCRIPT), "--facets-dir", str(tmp_path)],
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 0, (
+        f"single-session-no-friction should exit 0, got {result.returncode}\n"
+        f"stderr={result.stderr}"
+    )
+    assert "n_sessions=1" in result.stderr
+    assert "detections=0" in result.stderr
+    # Empty detections list, NOT empty stdout — wrapper merges this as
+    # a clean week.
+    assert json.loads(result.stdout) == []

package/coach/tests/test_analyze_aggregate.py

@@ -0,0 +1,153 @@
+"""analyze.py aggregator: per-project skill invocation breakdown.
+
+Locks in the data shape that flows through insights.sh → merge.py and
+ultimately becomes the rolling accumulator the inventory inference
+reads. Pre-2026-04-24 the aggregator collapsed all projects into a
+flat `skills_used` Counter and lost the project association — this
+file guards that the new `skills_by_project` field stays correct
+across the canonical cases.
+"""
+from __future__ import annotations
+
+import analyze
+
+
+# --- _project_name_from_slug -----------------------------------------------
+
+def test_project_name_from_simple_slug():
+    """The common case: `~/Desktop/dev/widget` becomes the slug
+    `-Users-alice-Desktop-dev-widget`. Last segment wins."""
+    assert analyze._project_name_from_slug(
+        "-Users-alice-Desktop-dev-widget") == "widget"
+
+
+def test_project_name_from_hyphenated_slug_collapses_to_last_segment():
+    """Documented limitation: hyphens in original project names
+    collide with the slash-to-dash separator. `acme-app` becomes
+    `app`. The hook tokenizer compensates by splitting cwd anchors
+    on dashes too, so the partial still matches at filter time."""
+    assert analyze._project_name_from_slug(
+        "-Users-r-Desktop-dev-acme-app") == "app"
+
+
+def test_project_name_from_empty_or_garbage():
+    assert analyze._project_name_from_slug("") == ""
+    assert analyze._project_name_from_slug(None or "") == ""
+    # Trailing dashes stripped before split.
+    assert analyze._project_name_from_slug("-Users-x-foo--") == "foo"
+
+
+def test_project_name_lowercases():
+    """Anchor-token comparison is lowercase on the hook side; emit
+    lowercase at source so they line up without the consumer needing
+    to re-normalize."""
+    assert analyze._project_name_from_slug(
+        "-Users-r-Desktop-dev-MyProject") == "myproject"
+
+
+# --- aggregate(): skills_by_project shape ----------------------------------
+
+def _make_session(*, project: str, skills: dict[str, int],
+                  assistant_turns: int = 5) -> dict:
+    """Minimal session shape sufficient for aggregate() to consume.
+    aggregate() only reads project, skills_invoked, and a few fields
+    from the detection branches; we provide the per-skill counts and
+    fill the rest with neutral defaults."""
+    return {
+        "project": project,
+        "skills_invoked": dict(skills),
+        "session_hash": "abcd1234",
+        "tool_counts": {},
+        "user_turns": 1,
+        "assistant_turns": assistant_turns,
+        "first_ts": None,
+        "last_ts": None,
+        "first_user_ts": None,
+        "first_edit_ts": None,
+        "first_plan_ts": None,
+        "task_create_count": 0,
+        "exit_plan_count": 0,
+        "edit_count": 0,
+        "write_count": 0,
+        "bash_count": 0,
+        "commit_count": 0,
+        "test_run_count": 0,
+        "has_any_commit": False,
+        "has_any_test_run": False,
+        "bash_rm_rf_count": 0,
+        "read_count": 0,
+        "grep_count": 0,
+        "glob_count": 0,
+        "agent_count": 0,
+        "skill_count": sum(skills.values()),
+        "sec_first_user_to_first_edit": None,
+        "plan_before_edit": False,
+    }
+
+
+def test_aggregate_emits_skills_by_project():
+    """The new emit. Single session, single project, single skill —
+    smallest case that proves the pipe is open."""
+    sessions = [_make_session(
+        project="-Users-r-Desktop-dev-service",
+        skills={"deploy-staging": 3})]
+    _detections, summary = analyze.aggregate(sessions)
+    assert summary["skills_by_project"] == {"service": {"deploy-staging": 3}}
+
+
+def test_aggregate_sums_across_sessions_in_same_project():
+    sessions = [
+        _make_session(project="-Users-r-Desktop-dev-service",
+                      skills={"deploy-staging": 2}),
+        _make_session(project="-Users-r-Desktop-dev-service",
+                      skills={"deploy-staging": 1, "design": 1}),
+    ]
+    _detections, summary = analyze.aggregate(sessions)
+    assert summary["skills_by_project"]["service"]["deploy-staging"] == 3
+    assert summary["skills_by_project"]["service"]["design"] == 1
+
+
+def test_aggregate_separates_distinct_projects():
+    sessions = [
+        _make_session(project="-Users-r-Desktop-dev-service",
+                      skills={"deploy-staging": 2}),
+        _make_session(project="-Users-r-Desktop-dev-widget",
+                      skills={"widget-build": 4}),
+    ]
+    _detections, summary = analyze.aggregate(sessions)
+    sbp = summary["skills_by_project"]
+    assert sbp["service"] == {"deploy-staging": 2}
+    assert sbp["widget"] == {"widget-build": 4}
+
+
+def test_aggregate_skips_sessions_with_no_project():
+    """If a session somehow lacks a project (corrupted state, edge
+    case from a transcript without a parent dir), it must not crash
+    or attribute its invocations to an empty-string project key."""
+    sessions = [
+        _make_session(project="", skills={"deploy-staging": 1}),
+        _make_session(project="-Users-r-Desktop-dev-service",
+                      skills={"deploy-staging": 1}),
+    ]
+    _detections, summary = analyze.aggregate(sessions)
+    assert "" not in summary["skills_by_project"]
+    assert summary["skills_by_project"] == {"service": {"deploy-staging": 1}}
+
+
+def test_aggregate_keeps_skills_used_in_sync_with_skills_by_project():
+    """The flat skills_used Counter and the per-project breakdown are
+    derived from the same source. Their totals must match — drift
+    here would be a sign of a bookkeeping error in aggregate()."""
+    sessions = [
+        _make_session(project="-Users-r-Desktop-dev-service",
+                      skills={"deploy-staging": 2, "design": 1}),
+        _make_session(project="-Users-r-Desktop-dev-widget",
+                      skills={"design": 3}),
+    ]
+    _detections, summary = analyze.aggregate(sessions)
+    flat_total = summary["skills_used"]
+    by_proj_totals: dict[str, int] = {}
+    for proj_skills in summary["skills_by_project"].values():
+        for sid, count in proj_skills.items():
+            by_proj_totals[sid] = by_proj_totals.get(sid, 0) + count
+    assert flat_total == by_proj_totals

package/coach/tests/test_analyze_redaction.py

@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import analyze
+import redact
+
+
+def _assistant_record(*, command: str = "") -> dict:
+    content = []
+    if command:
+        content.append({
+            "type": "tool_use",
+            "name": "Bash",
+            "input": {"command": command},
+        })
+    return {
+        "type": "assistant",
+        "message": {"role": "assistant", "content": content},
+    }
+
+
+def test_analyze_redacts_each_record_before_json_parsing(tmp_path, monkeypatch):
+    secret = "sk-" + ("A" * 40)
+    transcript = tmp_path / "session.jsonl"
+    transcript.write_text(
+        json.dumps(_assistant_record(command=f"echo {secret}; pytest")) + "\n"
+    )
+
+    real_loads = analyze.json.loads
+    parsed_inputs: list[str] = []
+
+    def loads_spy(text, *args, **kwargs):
+        parsed_inputs.append(text)
+        assert secret not in text
+        return real_loads(text, *args, **kwargs)
+
+    monkeypatch.setattr(analyze.json, "loads", loads_spy)
+
+    sig = analyze.analyze_session(transcript)
+
+    assert sig is not None
+    assert sig["test_run_count"] == 1
+    assert parsed_inputs
+    assert "[REDACTED:openai-key]" in parsed_inputs[0]
+
+
+def test_analyze_streams_transcript_without_reading_whole_file(tmp_path, monkeypatch):
+    transcript = tmp_path / "large.jsonl"
+    transcript.write_text(
+        "".join(json.dumps(_assistant_record()) + "\n" for _ in range(5000))
+    )
+
+    def fail_read_text(*args, **kwargs):
+        raise AssertionError("analyze_session must not read whole transcripts")
+
+    monkeypatch.setattr(Path, "read_text", fail_read_text)
+
+    sig = analyze.analyze_session(transcript)
+
+    assert sig is not None
+    assert sig["assistant_turns"] == 5000
+
+
+# --- redact.py pattern coverage -------------------------------------------
+# Each test passes a bare token in prose context (not a `KEY=value`
+# assignment) so we know the token-shape pattern itself catches it, not the
+# `.env`-style fallback.
+
+
+def test_redact_stripe_live_key():
+    text = "we use sk_live_" + "a1B2c3D4e5F6g7H8i9J0k1L2" + " for prod"
+    out = redact.redact(text)
+    assert "sk_live_" not in out
+    assert "[REDACTED:stripe-live-key]" in out
+
+
+def test_redact_stripe_test_key():
+    text = "test creds: sk_test_" + ("A" * 30)
+    out = redact.redact(text)
+    assert "sk_test_" not in out
+    assert "[REDACTED:stripe-test-key]" in out
+
+
+def test_redact_huggingface_token():
+    text = "use hf_" + ("a" * 35) + " to download"
+    out = redact.redact(text)
+    assert "hf_a" not in out
+    assert "[REDACTED:huggingface-token]" in out
+
+
+def test_redact_npm_publish_token():
+    text = "npm_" + ("X" * 36) + " is the publish token"
+    out = redact.redact(text)
+    assert "npm_X" not in out
+    assert "[REDACTED:npm-token]" in out
+
+
+def test_redact_does_not_collapse_short_lookalikes():
+    """Don't redact short fragments that happen to start with these
+    prefixes — minimum length thresholds matter."""
+    text = "sk_live_short hf_short npm_short"
+    out = redact.redact(text)
+    assert "REDACTED" not in out