kyp-mem 0.4.4 → 0.5.1

package/kyp_mem/hooks.py

@@ -10,6 +10,67 @@ SESSION_DIR = Path.home() / ".kyp-mem" / "sessions"
 CURRENT_SESSION = SESSION_DIR / "current.jsonl"
 
 MIN_ACTIONS = 3
+CHARS_PER_TOKEN = 4
+
+COMMAND_OUTPUT_ESTIMATES = {
+    "search": 2000,
+    "explore": 1000,
+    "read_cmd": 3000,
+    "git_inspect": 3000,
+    "test": 2000,
+    "build": 500,
+    "run": 200,
+    "git_write": 200,
+    "api_test": 1000,
+    "other": 300,
+}
+
+
+def _load_token_stats():
+    from .config import STATS_FILE
+    if STATS_FILE.exists():
+        try:
+            return json.loads(STATS_FILE.read_text())
+        except (json.JSONDecodeError, OSError):
+            pass
+    return {"sessions": [], "injections": []}
+
+
+def _save_token_stats(stats):
+    from .config import STATS_FILE
+    STATS_FILE.parent.mkdir(parents=True, exist_ok=True)
+    STATS_FILE.write_text(json.dumps(stats, indent=2) + "\n")
+
+
+def _record_session_tokens(session_id, project, exploration_tokens,
+                           files_read_count, files_read_chars,
+                           commands_run, commands_chars,
+                           files_edited, files_created):
+    stats = _load_token_stats()
+    stats["sessions"].append({
+        "id": session_id,
+        "project": project,
+        "ts": datetime.now().isoformat(),
+        "exploration_tokens": exploration_tokens,
+        "files_read": files_read_count,
+        "files_read_chars": files_read_chars,
+        "commands_run": commands_run,
+        "commands_chars": commands_chars,
+        "files_edited": files_edited,
+        "files_created": files_created,
+    })
+    _save_token_stats(stats)
+
+
+def _record_injection(project, chars):
+    stats = _load_token_stats()
+    stats["injections"].append({
+        "ts": datetime.now().isoformat(),
+        "project": project,
+        "chars": chars,
+        "tokens": chars // CHARS_PER_TOKEN,
+    })
+    _save_token_stats(stats)
 
 
 def handle_session_start():
@@ -82,7 +143,12 @@ def handle_session_start():
 
         parts.append("Use `kyp_project_context` for full details. Use `kyp_session_search` to search past sessions.")
 
-        print("\n".join(parts))
+        output = "\n".join(parts)
+        try:
+            _record_injection(project_name, len(output))
+        except Exception:
+            pass
+        print(output)
     except Exception:
         pass
 
@@ -129,6 +195,10 @@ def handle_post_tool_use():
     cwd = os.environ.get("CLAUDE_PROJECT_DIR", os.getcwd())
     entry["cwd"] = cwd
 
+    # Measure response size for token economics
+    tool_response = data.get("tool_response", "")
+    response_chars = len(str(tool_response)) if tool_response else 0
+
     if tool_name == "Edit":
         entry["action"] = "edit"
         entry["file"] = tool_input.get("file_path", "")
@@ -138,9 +208,16 @@ def handle_post_tool_use():
     elif tool_name == "Read":
         entry["action"] = "read"
         entry["file"] = tool_input.get("file_path", "")
+        if response_chars == 0:
+            try:
+                response_chars = Path(tool_input.get("file_path", "")).stat().st_size
+            except OSError:
+                pass
+        entry["response_chars"] = response_chars
     elif tool_name == "Bash":
         entry["action"] = "command"
         entry["command"] = tool_input.get("command", "")
+        entry["response_chars"] = response_chars
     else:
         return
 
@@ -340,6 +417,58 @@ def _build_next_steps(files_edited, files_created, commands_classified):
     return items
 
 
+def _summarize_with_claude(raw_note, project_name):
+    """Use Claude to rewrite session sections in plain, human-readable language."""
+    try:
+        from .config import get_session_model
+        import anthropic
+
+        model = get_session_model()
+        client = anthropic.Anthropic()
+
+        prompt = f"""Summarize this coding session for "{project_name}" in plain English. A future AI agent will read this to understand what happened — write for that audience.
+
+You have: user prompts (what was asked), a timeline of file edits/reads/commands (what happened), and raw section data. Synthesize these into a coherent narrative.
+
+Rules:
+- Summary: 2-3 sentences. State the objective (from prompts), what was done, and the outcome. Be specific: "Fixed navigation bug where clicking sessions broke the back button" not "Modified files and ran commands."
+- INVESTIGATED: What was explored and WHY. "Examined the session hook pipeline to understand why summaries were empty" not "Searched for `session-view`". Max 4 bullets.
+- LEARNED: Insights or discoveries. "The config CLI command was defined but never wired to the dispatcher" not "Investigated and modified: `cli.py`". Max 4 bullets.
+- COMPLETED: Concrete deliverables. "Added AI-powered session summarization using Claude Haiku" not "Modified `hooks.py`". Max 5 bullets.
+- NEXT STEPS: What should happen next session. Infer from context — unfinished work, unfixed bugs, natural follow-ups. Max 3 bullets.
+
+NEVER include raw grep patterns, CSS class names, file paths, or command output. Write like you're telling a teammate what you did today.
+
+Return ONLY this format (no preamble):
+
+## Summary
+<text>
+
+## INVESTIGATED
+- <item>
+
+## LEARNED
+- <item>
+
+## COMPLETED
+- <item>
+
+## NEXT STEPS
+- <item>
+
+Raw session data:
+{raw_note}"""
+
+        response = client.messages.create(
+            model=model,
+            max_tokens=1024,
+            messages=[{"role": "user", "content": prompt}],
+        )
+        return response.content[0].text.strip()
+    except Exception:
+        return None
+
+
 def handle_stop():
     if not CURRENT_SESSION.exists():
         return
@@ -417,49 +546,130 @@ def handle_stop():
     completed = _build_completed(files_edited, files_created, commands_classified, project_dir)
     next_steps = _build_next_steps(files_edited, files_created, commands_classified)
 
+    # Build raw note for Claude summarization
+    raw_parts = []
+    raw_parts.append("## Summary")
+    raw_parts.append(", ".join(summary_items) + f" in `{project_name}`." if summary_items else "")
+    raw_parts.append("")
+    raw_parts.append("## INVESTIGATED")
+    if investigated:
+        raw_parts.extend(investigated)
+    raw_parts.append("")
+    raw_parts.append("## LEARNED")
+    if learned:
+        raw_parts.extend(learned)
+    raw_parts.append("")
+    raw_parts.append("## COMPLETED")
+    if completed:
+        raw_parts.extend(completed)
+    raw_parts.append("")
+    raw_parts.append("## NEXT STEPS")
+    if next_steps:
+        raw_parts.extend(next_steps)
+
+    # Prompts go FIRST — they define the session's objective
+    if prompts:
+        raw_parts.insert(0, "## USER PROMPTS (what was asked)")
+        for i, p in enumerate(prompts):
+            raw_parts.insert(i + 1, f"- [{p['ts']}] {p['text'][:300]}")
+        raw_parts.insert(len(prompts) + 1, "")
+
+    # Full timeline gives Claude the narrative arc
+    if timeline:
+        raw_parts.append("")
+        raw_parts.append("## TIMELINE (what happened, chronological)")
+        for line in timeline[:50]:
+            raw_parts.append(line)
+
+    raw_note = "\n".join(raw_parts)
+
+    # Compute exploration tokens from captured response sizes
+    files_read_chars = 0
+    commands_chars = 0
+    files_read_count = len(files_read)
+    commands_run_count = len(commands)
+
+    for e in entries:
+        rc = e.get("response_chars", 0)
+        if e.get("action") == "read":
+            if rc > 0:
+                files_read_chars += rc
+            else:
+                try:
+                    files_read_chars += Path(e.get("file", "")).stat().st_size
+                except OSError:
+                    pass
+        elif e.get("action") == "command":
+            if rc > 0:
+                commands_chars += rc
+            else:
+                cls, _ = _classify_command(e.get("command", ""))
+                commands_chars += COMMAND_OUTPUT_ESTIMATES.get(cls, 300)
+
+    exploration_chars = files_read_chars + commands_chars
+    exploration_tokens = exploration_chars // CHARS_PER_TOKEN
+
+    try:
+        _record_session_tokens(
+            session_id, project_name, exploration_tokens,
+            files_read_count, files_read_chars,
+            commands_run_count, commands_chars,
+            len(files_edited), len(files_created),
+        )
+    except Exception:
+        pass
+
+    # Try Claude summarization, fall back to raw sections
+    summarized = _summarize_with_claude(raw_note, project_name)
+
     parts = [f"# Session {session_id}", ""]
     parts.append(f"**Project:** `{project_dir}`")
     parts.append(f"**Actions:** {len(entries)} total, {len(write_actions)} substantive")
+    parts.append(f"**Exploration:** ~{exploration_tokens:,} tokens ({files_read_count} reads, {commands_run_count} commands)")
     parts.append("")
 
-    parts.append("## Summary")
-    parts.append(", ".join(summary_items) + f" in `{project_name}`." if summary_items else "")
-    parts.append("")
-
-    parts.append("## PROMPTS")
-    if prompts:
-        for i, p in enumerate(prompts, 1):
-            parts.append(f"### {i}. [{p['ts']}]")
-            parts.append(f"> {p['text']}")
-            parts.append("")
-    parts.append("")
+    if summarized:
+        # Insert prompts section before the Claude-rewritten sections
+        parts.append("## PROMPTS")
+        if prompts:
+            for i, p in enumerate(prompts, 1):
+                parts.append(f"### {i}. [{p['ts']}]")
+                parts.append(f"> {p['text']}")
+                parts.append("")
+        parts.append("")
+        parts.append(summarized)
+    else:
+        parts.append("## Summary")
+        parts.append(", ".join(summary_items) + f" in `{project_name}`." if summary_items else "")
+        parts.append("")
 
-    parts.append("## INVESTIGATED")
-    if investigated:
-        parts.extend(investigated)
-    parts.append("")
+        parts.append("## PROMPTS")
+        if prompts:
+            for i, p in enumerate(prompts, 1):
+                parts.append(f"### {i}. [{p['ts']}]")
+                parts.append(f"> {p['text']}")
+                parts.append("")
+        parts.append("")
 
-    parts.append("## LEARNED")
-    if learned:
-        parts.extend(learned)
-    parts.append("")
+        parts.append("## INVESTIGATED")
+        if investigated:
+            parts.extend(investigated)
+        parts.append("")
 
-    parts.append("## COMPLETED")
-    if completed:
-        parts.extend(completed)
-    parts.append("")
+        parts.append("## LEARNED")
+        if learned:
+            parts.extend(learned)
+        parts.append("")
 
-    parts.append("## NEXT STEPS")
-    if next_steps:
-        parts.extend(next_steps)
-    parts.append("")
+        parts.append("## COMPLETED")
+        if completed:
+            parts.extend(completed)
+        parts.append("")
 
-    if timeline:
-        parts.append("## Timeline")
-        for line in timeline[:40]:
-            parts.append(line)
-        if len(timeline) > 40:
-            parts.append(f"  ... and {len(timeline) - 40} more actions")
+        parts.append("## NEXT STEPS")
+        if next_steps:
+            parts.extend(next_steps)
+        parts.append("")
 
     content = "\n".join(parts)
     tags = ["session", "auto-captured", project_name]

package/kyp_mem/server.py

@@ -237,16 +237,39 @@ def kyp_recent(limit: int = 10) -> str:
 
 @mcp.tool()
 def kyp_stats() -> str:
-    """Get vault statistics — note count, folders, tags, links."""
+    """Get vault statistics — note count, folders, tags, links, and token economics (exploration cost vs memory injection cost)."""
     s = vault.get_stats()
-    return (
-        f"Vault stats:\n"
-        f"  Notes: {s['notes']}\n"
-        f"  Folders: {s['folders']}\n"
-        f"  Tags: {s['tags']}\n"
-        f"  Links: {s['links']}\n"
-        f"  Backlinks: {s['backlinks']}"
-    )
+    lines = [
+        "Vault stats:",
+        f"  Notes: {s['notes']}",
+        f"  Folders: {s['folders']}",
+        f"  Tags: {s['tags']}",
+        f"  Links: {s['links']}",
+        f"  Backlinks: {s['backlinks']}",
+    ]
+
+    try:
+        from .config import STATS_FILE
+        raw = json.loads(STATS_FILE.read_text()) if STATS_FILE.exists() else {}
+        sessions = raw.get("sessions", [])
+        injections = raw.get("injections", [])
+        if sessions:
+            total_exploration = sum(s.get("exploration_tokens", 0) for s in sessions)
+            avg_exploration = total_exploration // len(sessions)
+            latest_inj = injections[-1].get("tokens", 0) if injections else 0
+            lines.append("")
+            lines.append("Token economics:")
+            lines.append(f"  Total exploration: ~{total_exploration:,}t across {len(sessions)} sessions")
+            lines.append(f"  Avg per session: ~{avg_exploration:,}t (cold-start cost)")
+            lines.append(f"  Memory injection: ~{latest_inj:,}t (session start)")
+            if latest_inj > 0 and avg_exploration > 0:
+                ratio = round(avg_exploration / latest_inj, 1)
+                pct = round((1 - latest_inj / avg_exploration) * 100, 1)
+                lines.append(f"  Compression: {ratio}x — {pct}% smaller than re-exploring")
+    except Exception:
+        pass
+
+    return "\n".join(lines)
 
 
 @mcp.tool()