@cohortapp/agent-sdk 2.3.0

package/scripts/test-llm-email-dedup.py

@@ -0,0 +1,251 @@
+#!/usr/bin/env python3
+"""Tests for llm_email_dedup.py — LLM-based email deduplication.
+
+Tests the module in two modes:
+  1. Unit tests (no network) — test internal logic, parsing, audit logging
+  2. Integration test (with mocked IMAP + real LLM) — test the full flow
+
+Run: python3 scripts/test-llm-email-dedup.py
+"""
+import json
+import os
+import sys
+import time
+from unittest.mock import patch, MagicMock
+from datetime import datetime, timezone
+
+# Add scripts dir to path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+from llm_email_dedup import (
+    llm_dedup_check,
+    fetch_recent_thread_messages,
+    _extract_text_body,
+    _call_haiku,
+    _log_audit,
+    AUDIT_DIR,
+)
+
+PASS = 0
+FAIL = 0
+
+
+def assert_eq(label, actual, expected):
+    global PASS, FAIL
+    if actual == expected:
+        PASS += 1
+        print(f"  PASS: {label}")
+    else:
+        FAIL += 1
+        print(f"  FAIL: {label}")
+        print(f"    expected: {expected!r}")
+        print(f"    actual:   {actual!r}")
+
+
+def assert_true(label, value):
+    assert_eq(label, bool(value), True)
+
+
+def assert_false(label, value):
+    assert_eq(label, bool(value), False)
+
+
+# ─── Test 1: No recent messages → proceed ───────────────────────────
+
+print("\n=== LLM dedup: no recent messages → proceed ===")
+
+with patch("llm_email_dedup.fetch_recent_thread_messages", return_value=[]):
+    result = llm_dedup_check("alice@example.com", "Hello", "Test body")
+    assert_false("no messages → skip=False", result["skip"])
+    assert_eq("no messages → recent_messages=0", result["recent_messages"], 0)
+    assert_eq("no messages → confidence=high", result["confidence"], "high")
+
+
+# ─── Test 2: LLM says PROCEED ────────────────────────────────────────
+
+print("\n=== LLM dedup: LLM says PROCEED ===")
+
+mock_messages = [
+    {
+        "from": "alice@example.com",
+        "to": "robin@example.com",
+        "date": "Mon, 6 Apr 2026 10:00:00 +0000",
+        "subject": "Re: Project Alpha",
+        "body_preview": "Thanks for the update. Can you send me the latest timeline?",
+        "folder": "INBOX",
+    },
+]
+
+with patch("llm_email_dedup.fetch_recent_thread_messages", return_value=mock_messages):
+    with patch("llm_email_dedup._call_haiku", return_value="PROCEED: This is a new response to the recipient's question about the timeline, not a duplicate."):
+        result = llm_dedup_check("alice@example.com", "Re: Project Alpha", "Here is the updated timeline...")
+        assert_false("LLM says proceed → skip=False", result["skip"])
+        assert_eq("recent_messages=1", result["recent_messages"], 1)
+        assert_true("has reason", len(result["reason"]) > 0)
+
+
+# ─── Test 3: LLM says SKIP ───────────────────────────────────────────
+
+print("\n=== LLM dedup: LLM says SKIP ===")
+
+mock_messages_with_sent = [
+    {
+        "from": "robin@example.com",
+        "to": "bob@example.com",
+        "date": "Mon, 6 Apr 2026 14:00:00 +0000",
+        "subject": "Weekly Strategic Review Brief",
+        "body_preview": "Hi Bob, please find the weekly strategic review brief attached.",
+        "folder": "[Gmail]/Sent Mail",
+    },
+    {
+        "from": "bob@example.com",
+        "to": "robin@example.com",
+        "date": "Mon, 6 Apr 2026 10:00:00 +0000",
+        "subject": "Re: Weekly Status",
+        "body_preview": "Thanks for the brief last week. Can you send the updated one?",
+        "folder": "INBOX",
+    },
+]
+
+with patch("llm_email_dedup.fetch_recent_thread_messages", return_value=mock_messages_with_sent):
+    with patch("llm_email_dedup._call_haiku", return_value="SKIP: A weekly strategic review brief was already sent to this recipient today at 14:00. The proposed email covers the same content."):
+        result = llm_dedup_check("bob@example.com", "Weekly Strategic Review Brief — 7 April 2026", "Hi Bob, the weekly brief is attached.")
+        assert_true("LLM says skip → skip=True", result["skip"])
+        assert_eq("recent_messages=2", result["recent_messages"], 2)
+        assert_true("has reason", len(result["reason"]) > 0)
+        assert_true("reason mentions brief", "brief" in result["reason"].lower())
+
+
+# ─── Test 4: LLM call fails → fail-open ──────────────────────────────
+
+print("\n=== LLM dedup: LLM fails → fail-open ===")
+
+with patch("llm_email_dedup.fetch_recent_thread_messages", return_value=mock_messages):
+    with patch("llm_email_dedup._call_haiku", return_value=""):
+        result = llm_dedup_check("alice@example.com", "Important", "Body text")
+        assert_false("LLM fails → skip=False (fail-open)", result["skip"])
+        assert_eq("confidence=low on failure", result["confidence"], "low")
+
+
+# ─── Test 5: Ambiguous LLM response → fail-open ──────────────────────
+
+print("\n=== LLM dedup: ambiguous response → fail-open ===")
+
+with patch("llm_email_dedup.fetch_recent_thread_messages", return_value=mock_messages):
+    with patch("llm_email_dedup._call_haiku", return_value="I'm not sure whether this is a duplicate. It could go either way."):
+        result = llm_dedup_check("alice@example.com", "Unclear topic", "Some body")
+        assert_false("ambiguous → skip=False (fail-open)", result["skip"])
+        assert_eq("confidence=low on ambiguity", result["confidence"], "low")
+
+
+# ─── Test 6: Return structure ─────────────────────────────────────────
+
+print("\n=== LLM dedup: return structure ===")
+
+with patch("llm_email_dedup.fetch_recent_thread_messages", return_value=[]):
+    result = llm_dedup_check("test@example.com", "Test", "Body")
+    assert_true("has 'skip' key", "skip" in result)
+    assert_true("has 'reason' key", "reason" in result)
+    assert_true("has 'confidence' key", "confidence" in result)
+    assert_true("has 'recent_messages' key", "recent_messages" in result)
+    assert_true("skip is bool", isinstance(result["skip"], bool))
+    assert_true("reason is str", isinstance(result["reason"], str))
+    assert_true("confidence is str", isinstance(result["confidence"], str))
+    assert_true("recent_messages is int", isinstance(result["recent_messages"], int))
+
+
+# ─── Test 7: Audit logging ────────────────────────────────────────────
+
+print("\n=== LLM dedup: audit logging ===")
+
+date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+audit_file = os.path.join(AUDIT_DIR, f"{date_str}-actions.jsonl")
+
+# Count lines before
+lines_before = 0
+if os.path.exists(audit_file):
+    with open(audit_file) as f:
+        lines_before = sum(1 for _ in f)
+
+# Trigger a dedup check that will log
+with patch("llm_email_dedup.fetch_recent_thread_messages", return_value=[]):
+    llm_dedup_check("audit-test@example.com", "Audit Test Subject", "Audit test body")
+
+# Count lines after
+lines_after = 0
+if os.path.exists(audit_file):
+    with open(audit_file) as f:
+        lines_after = sum(1 for _ in f)
+
+assert_true("audit log entry written", lines_after > lines_before)
+
+
+# ─── Test 8: text body extraction from email ──────────────────────────
+
+print("\n=== Text body extraction ===")
+
+# Test plain text extraction
+from email.mime.text import MIMEText
+plain_msg = MIMEText("Hello this is plain text", "plain")
+extracted = _extract_text_body(plain_msg)
+assert_eq("plain text extracted", extracted, "Hello this is plain text")
+
+# Test HTML stripping
+html_msg = MIMEText("<html><body><p>Hello <b>world</b></p></body></html>", "html")
+extracted = _extract_text_body(html_msg)
+assert_true("HTML tags stripped", "<" not in extracted)
+assert_true("text content preserved", "Hello" in extracted and "world" in extracted)
+
+
+# ─── Test 9: sender_email and sender_password passthrough ─────────────
+
+print("\n=== Sender credential passthrough ===")
+
+# Verify that custom sender credentials are passed through to IMAP fetch
+with patch("llm_email_dedup.fetch_recent_thread_messages", return_value=[]) as mock_fetch:
+    llm_dedup_check("test@example.com", "Test", "Body",
+                      sender_email="alex@example.com",
+                      sender_password="fake-password")
+    call_args = mock_fetch.call_args
+    assert_eq("sender_email passed", call_args[1].get("sender_email", call_args[0][2] if len(call_args[0]) > 2 else ""), "alex@example.com")
+
+
+# ─── Test 10: Multiple SENT messages — higher confidence ──────────────
+
+print("\n=== LLM dedup: multiple sent messages → higher confidence ===")
+
+multi_sent = [
+    {
+        "from": "robin@example.com",
+        "to": "carol@example.com",
+        "date": "Mon, 6 Apr 2026 14:00:00 +0000",
+        "subject": "Brief v2",
+        "body_preview": "Updated brief attached.",
+        "folder": "[Gmail]/Sent Mail",
+    },
+    {
+        "from": "robin@example.com",
+        "to": "carol@example.com",
+        "date": "Mon, 6 Apr 2026 10:00:00 +0000",
+        "subject": "Brief v1",
+        "body_preview": "First version of brief attached.",
+        "folder": "[Gmail]/Sent Mail",
+    },
+]
+
+with patch("llm_email_dedup.fetch_recent_thread_messages", return_value=multi_sent):
+    with patch("llm_email_dedup._call_haiku", return_value="SKIP: Two versions of this brief have already been sent today."):
+        result = llm_dedup_check("carol@example.com", "Brief v3", "Third version.")
+        assert_true("skip with multiple sent", result["skip"])
+        assert_eq("high confidence with >=2 messages", result["confidence"], "high")
+
+
+# ─── Summary ──────────────────────────────────────────────────────────
+
+print(f"\n{'='*50}")
+print(f"Results: {PASS} passed, {FAIL} failed, {PASS + FAIL} total")
+if FAIL > 0:
+    print("SOME TESTS FAILED")
+    sys.exit(1)
+else:
+    print("ALL TESTS PASSED")
+    sys.exit(0)

package/scripts/test-pre-draft-integration.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""Test pre-draft context integration — Memory Enhancement Phase 7.
+
+Verifies that:
+  1. pre_draft_lookup module imports correctly
+  2. Context lookup returns data for known entities
+  3. Audit log entries are written
+  4. All send scripts can import the module without errors
+  5. Fail-open behaviour works (bad recipient doesn't crash)
+
+Usage:
+  python3 scripts/test-pre-draft-integration.py
+
+Created: 2026-04-04 — Memory Enhancement Phase 7
+"""
+import importlib
+import importlib.util
+import json
+import os
+import sys
+import time
+from datetime import datetime, timezone
+from pathlib import Path
+
+SCRIPT_DIR = Path(__file__).resolve().parent
+REPO_ROOT = SCRIPT_DIR.parent
+AUDIT_LOG_DIR = REPO_ROOT / "logs" / "audit"
+
+# Ensure scripts dir is on path
+sys.path.insert(0, str(SCRIPT_DIR))
+
+passed = 0
+failed = 0
+total = 0
+
+
+def test(name, condition, detail=""):
+    global passed, failed, total
+    total += 1
+    if condition:
+        passed += 1
+        print(f"  PASS: {name}")
+    else:
+        failed += 1
+        print(f"  FAIL: {name}" + (f" — {detail}" if detail else ""))
+
+
+def section(title):
+    print(f"\n{'='*60}")
+    print(f"  {title}")
+    print(f"{'='*60}")
+
+
+# ─────────────────────────────────────────────────────
+# Test 1: Module import
+# ─────────────────────────────────────────────────────
+section("1. Module Import")
+
+try:
+    from pre_draft_lookup import pre_draft_lookup, format_context_brief
+    test("pre_draft_lookup imports successfully", True)
+except ImportError as e:
+    test("pre_draft_lookup imports successfully", False, str(e))
+    print("\nFATAL: Cannot continue without the module. Exiting.")
+    sys.exit(1)
+
+# Verify the underlying pre-draft-context module loaded
+try:
+    from pre_draft_lookup import _pdc_module
+    test("pre-draft-context.py module loaded", _pdc_module is not None)
+except Exception as e:
+    test("pre-draft-context.py module loaded", False, str(e))
+
+
+# ─────────────────────────────────────────────────────
+# Test 2: Known entity lookup
+# ─────────────────────────────────────────────────────
+section("2. Known Entity Lookup")
+
+# Test with a known entity from the entity index
+ctx = pre_draft_lookup("Alex Chen", message_type="email")
+test("Lookup 'Alex Chen' returns context", ctx is not None)
+if ctx:
+    test("Context has recipient.name containing 'Alex'",
+         "Alex" in (ctx.get("recipient", {}).get("name") or ""))
+    test("Context has relationship type", ctx.get("relationship", {}).get("type", "") != "")
+    test("Context has entity_id", ctx.get("recipient", {}).get("entity_id", "") != "")
+
+# Test brief format
+if ctx:
+    brief = format_context_brief(ctx)
+    test("Brief format is non-empty", len(brief) > 0)
+    test("Brief contains recipient name", "Alex" in brief)
+
+# Test with another known entity
+ctx2 = pre_draft_lookup("Sam Rivera", message_type="email")
+test("Lookup 'Sam Rivera' returns context", ctx2 is not None)
+
+# Test with email lookup
+ctx3 = pre_draft_lookup("alex@example.com", message_type="email")
+test("Lookup by email returns context", ctx3 is not None)
+
+
+# ─────────────────────────────────────────────────────
+# Test 3: Unknown entity (fail-open)
+# ─────────────────────────────────────────────────────
+section("3. Fail-Open Behaviour")
+
+ctx_unknown = pre_draft_lookup("Completely Unknown Person ZZZZZ", message_type="email")
+test("Unknown recipient returns None (not crash)", ctx_unknown is None)
+
+ctx_empty = pre_draft_lookup("", message_type="email")
+test("Empty recipient returns None (not crash)", ctx_empty is None)
+
+
+# ─────────────────────────────────────────────────────
+# Test 4: Audit log written
+# ─────────────────────────────────────────────────────
+section("4. Audit Log")
+
+date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")
+log_path = AUDIT_LOG_DIR / f"{date_str}-pre-draft-lookups.jsonl"
+test("Audit log file exists", log_path.exists())
+
+if log_path.exists():
+    lines = log_path.read_text(encoding="utf-8").strip().split("\n")
+    lines = [l for l in lines if l.strip()]
+    test("Audit log has entries", len(lines) >= 1)
+
+    # Check the last few entries for our test lookups
+    recent = []
+    for line in lines[-6:]:
+        try:
+            recent.append(json.loads(line))
+        except json.JSONDecodeError:
+            pass
+
+    found_alex = any("Alex" in (r.get("entity_name") or "") for r in recent)
+    test("Audit log contains Alex lookup", found_alex)
+
+    found_unknown = any(r.get("status") == "not_found" for r in recent)
+    test("Audit log contains not_found entry", found_unknown)
+
+    # Verify log schema
+    if recent:
+        sample = recent[-1]
+        test("Log entry has timestamp", "timestamp" in sample)
+        test("Log entry has status", "status" in sample)
+        test("Log entry has lookup_ms", "lookup_ms" in sample)
+        test("Log entry has message_type", "message_type" in sample)
+
+
+# ─────────────────────────────────────────────────────
+# Test 5: Send script imports
+# ─────────────────────────────────────────────────────
+section("5. Send Script Import Compatibility")
+
+# Check that the import line in each send script would work
+# (We test the import mechanism, not the actual send functionality)
+
+for script_name in ["send-email-threaded.py", "send-email-as-principal.py", "send-email-with-attachment.py"]:
+    script_path = SCRIPT_DIR / script_name
+    if script_path.exists():
+        content = script_path.read_text(encoding="utf-8")
+        has_import = "from pre_draft_lookup import pre_draft_lookup" in content
+        test(f"{script_name} has pre_draft_lookup import", has_import)
+        has_call = "pre_draft_lookup(" in content and "PRE_DRAFT" in content
+        test(f"{script_name} has pre_draft_lookup call in send_email()", has_call)
+    else:
+        test(f"{script_name} exists", False, "file not found")
+
+# Check slack-send.sh
+slack_path = SCRIPT_DIR / "slack-send.sh"
+if slack_path.exists():
+    content = slack_path.read_text(encoding="utf-8")
+    has_call = "pre_draft_lookup.py" in content
+    test("slack-send.sh has pre_draft_lookup.py call", has_call)
+    has_comment = "Phase 7" in content
+    test("slack-send.sh has Phase 7 comment", has_comment)
+else:
+    test("slack-send.sh exists", False)
+
+
+# ─────────────────────────────────────────────────────
+# Test 6: Performance
+# ─────────────────────────────────────────────────────
+section("6. Performance")
+
+# Measure lookup time — should be < 500ms
+t0 = time.time()
+pre_draft_lookup("Alex Chen", message_type="email")
+elapsed_ms = (time.time() - t0) * 1000
+test(f"Lookup completes in < 500ms (actual: {elapsed_ms:.1f}ms)", elapsed_ms < 500)
+
+
+# ─────────────────────────────────────────────────────
+# Summary
+# ─────────────────────────────────────────────────────
+print(f"\n{'='*60}")
+print(f"  RESULTS: {passed}/{total} passed, {failed} failed")
+print(f"{'='*60}\n")
+
+sys.exit(0 if failed == 0 else 1)